From f9e32e306a9cb79dfef834307858550cb4f9ecd1 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 11:50:04 +0300
Subject: [PATCH 01/44] sdk: add memexport tracing

---
 rexglue-sdk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rexglue-sdk b/rexglue-sdk
index 7db90c7..c0d7fb9 160000
--- a/rexglue-sdk
+++ b/rexglue-sdk
@@ -1 +1 @@
-Subproject commit 7db90c73dea7eb79e8cdfe3061150fa312b82ed0
+Subproject commit c0d7fb98a4c8e70eec1751d8bbbecb011be0201e

From 7fa5d1e9029b64cac6b3d044dc65beafab64ccf0 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 11:54:23 +0300
Subject: [PATCH 02/44] sdk: trace resolve readback stalls

---
 rexglue-sdk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rexglue-sdk b/rexglue-sdk
index c0d7fb9..d846a43 160000
--- a/rexglue-sdk
+++ b/rexglue-sdk
@@ -1 +1 @@
-Subproject commit c0d7fb98a4c8e70eec1751d8bbbecb011be0201e
+Subproject commit d846a4310369323e054f683141145ee96cc6f982

From ed788ae3437f0f63ed7203508542dd3ef5426e66 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 13:14:49 +0300
Subject: [PATCH 03/44] gpu: add LEGO shader AOT pipeline

---
 tools/gpu-aot/README.md                       |  60 ++
 tools/gpu-aot/patches/xenosrecomp-lego.patch  | 671 ++++++++++++++++++
 tools/gpu-shader-extract/.gitignore           |   3 +
 tools/gpu-shader-extract/Program.cs           | 126 ++++
 tools/gpu-shader-extract/README.md            |  15 +
 .../gpu-shader-extract.csproj                 |  20 +
 6 files changed, 895 insertions(+)
 create mode 100644 tools/gpu-aot/README.md
 create mode 100644 tools/gpu-aot/patches/xenosrecomp-lego.patch
 create mode 100644 tools/gpu-shader-extract/.gitignore
 create mode 100644 tools/gpu-shader-extract/Program.cs
 create mode 100644 tools/gpu-shader-extract/README.md
 create mode 100644 tools/gpu-shader-extract/gpu-shader-extract.csproj

diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
new file mode 100644
index 0000000..1e63ff5
--- /dev/null
+++ b/tools/gpu-aot/README.md
@@ -0,0 +1,60 @@
+# LEGO Dimensions GPU AOT pipeline
+
+This directory keeps the reproducible, project-owned part of the native D3D12
+shader pipeline. Generated archives, HLSL, DXIL, and build trees stay under
+`rexlego/out/native-gpu` and are intentionally not committed.
+
+## Inputs
+
+- Read-only game archives from the complete Xbox 360 disc set.
+- `tools/gpu-shader-extract`, which extracts likely shader assets without
+  changing the archives.
+- The pinned XenosRecomp checkout under `research/reblue/thirdparty`.
+- `patches/xenosrecomp-lego.patch`, applied on top of that checkout.
+
+The patch is required because the upstream fork assumes Blue Dragon's narrower
+resource layout. The LEGO adaptation currently includes:
+
+- the four-byte separator between virtual and physical shader-container data;
+- bounds-checked microcode decoding and deterministic sequential DXC calls;
+- the complete Xenos float, integer, temporary-register, boolean, MRT, and
+  32-slot texture/sampler banks;
+- vertex-stage texture fetches using explicit LOD;
+- D3D12-only cache generation and partial-cache output on failures;
+- bounded diagnostics plus single-hash repro modes.
+
+The verified D3D12-only corpus result is **3,481 / 3,481 unique shader
+containers (100.00%)**. The generated `lego-dxil-cache.cpp` is about 13 MiB
+compressed C++ source in the current build tree.
+
+## Reproduction
+
+Build the extractor with its local project, then:
+
+```powershell
+gpu-shader-extract inventory <game-directory>
+gpu-shader-extract extract <game-directory> rexlego/out/native-gpu/shader-assets
+```
+
+Apply the adaptation to the pinned XenosRecomp checkout and build it in an
+x64 MSVC/Windows SDK environment. Generate the shipping D3D12 cache with:
+
+```powershell
+$env:XENOS_RECOMP_DXIL_ONLY = '1'
+XenosRecomp.exe `
+  rexlego/out/native-gpu/shader-assets `
+  rexlego/out/native-gpu/lego-dxil-cache.cpp `
+  research/reblue/thirdparty/XenosRecomp/XenosRecomp/shader_common.h
+```
+
+For a failing shader, set `XENOS_RECOMP_ONLY_HASH` to its `0x...` hash and
+`XENOS_RECOMP_VERBOSE_DXC=1`. `XENOS_RECOMP_HLSL_DUMP` may name a directory
+for generated HLSL.
+
+## Integration boundary
+
+The generated cache is only the shader half of the native renderer. Shipping
+still requires the D3D12 host device, guest resource wrappers, draw/state
+hooks, constant and descriptor upload, render-target/resolve handling, and a
+safe fallback to the existing `xenos` plugin until native coverage is proven
+in gameplay.
diff --git a/tools/gpu-aot/patches/xenosrecomp-lego.patch b/tools/gpu-aot/patches/xenosrecomp-lego.patch
new file mode 100644
index 0000000..40fbe1d
--- /dev/null
+++ b/tools/gpu-aot/patches/xenosrecomp-lego.patch
@@ -0,0 +1,671 @@
+warning: in the working copy of 'XenosRecomp/dxc_compiler.cpp', LF will be replaced by CRLF the next time Git touches it
+warning: in the working copy of 'XenosRecomp/main.cpp', LF will be replaced by CRLF the next time Git touches it
+warning: in the working copy of 'XenosRecomp/shader_common.h', LF will be replaced by CRLF the next time Git touches it
+warning: in the working copy of 'XenosRecomp/shader_recompiler.cpp', LF will be replaced by CRLF the next time Git touches it
+warning: in the working copy of 'XenosRecomp/shader_recompiler.h', LF will be replaced by CRLF the next time Git touches it
+diff --git a/XenosRecomp/dxc_compiler.cpp b/XenosRecomp/dxc_compiler.cpp
+index 4a851c2..ceffb3f 100644
+--- a/XenosRecomp/dxc_compiler.cpp
++++ b/XenosRecomp/dxc_compiler.cpp
+@@ -1,5 +1,7 @@
+ #include "dxc_compiler.h"
+ 
++static std::atomic<uint32_t> g_printedCompilerDiagnostics = 0;
++
+ DxcCompiler::DxcCompiler()
+ {
+     HRESULT hr = DxcCreateInstance(CLSID_DxcCompiler, IID_PPV_ARGS(&dxcCompiler));
+@@ -82,7 +84,19 @@ IDxcBlob* DxcCompiler::compile(const std::string& shaderSource, bool compilePixe
+                 hr = result->GetOutput(DXC_OUT_ERRORS, IID_PPV_ARGS(&errors), nullptr);
+                 assert(SUCCEEDED(hr) && errors != nullptr);
+ 
+-                fputs(errors->GetStringPointer(), stderr);
++                const bool verboseDiagnostics = std::getenv("XENOS_RECOMP_VERBOSE_DXC") != nullptr;
++                const uint32_t diagnosticIndex = g_printedCompilerDiagnostics.fetch_add(1);
++                if (verboseDiagnostics || diagnosticIndex < 3)
++                {
++                    if (verboseDiagnostics)
++                        fputs(errors->GetStringPointer(), stderr);
++                    else
++                        fmt::println(stderr, "DXC compilation failed (diagnostic {}); rerun the hash with XENOS_RECOMP_VERBOSE_DXC=1", diagnosticIndex + 1);
++                }
++                else if (diagnosticIndex == 3)
++                {
++                    fputs("Further DXC diagnostics suppressed; see the failure summary.\n", stderr);
++                }
+ 
+                 errors->Release();
+             }
+diff --git a/XenosRecomp/main.cpp b/XenosRecomp/main.cpp
+index 754198c..5ee2ca9 100644
+--- a/XenosRecomp/main.cpp
++++ b/XenosRecomp/main.cpp
+@@ -31,6 +31,8 @@ struct RecompiledShader
+     std::vector<uint8_t> spirv;
+     uint32_t specConstantsMask = 0;
+     std::string sourceName;
++    std::string sourcePath;
++    size_t sourceOffset = 0;
+ };
+ 
+ // Per-shader recompile failures, collected from the parallel loop and reported before exit.
+@@ -83,6 +85,8 @@ int main(int argc, char** argv)
+     else if (const char* env = std::getenv("XENOS_RECOMP_HLSL_DUMP"))
+         hlslDumpDir = env;
+ 
++    const bool dxilOnly = std::getenv("XENOS_RECOMP_DXIL_ONLY") != nullptr;
++
+     size_t includeSize = 0;
+     auto includeData = readAllBytes(includeInput, includeSize);
+     std::string_view include(reinterpret_cast<const char*>(includeData.get()), includeSize);
+@@ -107,7 +111,8 @@ int main(int argc, char** argv)
+             for (size_t i = 0; fileSize > sizeof(ShaderContainer) && i < fileSize - sizeof(ShaderContainer) - 1;)
+             {
+                 auto shaderContainer = reinterpret_cast<const ShaderContainer*>(fileData.get() + i);
+-                size_t dataSize = shaderContainer->virtualSize + shaderContainer->physicalSize;
++                const size_t virtualSize = shaderContainer->virtualSize;
++                const size_t dataSize = virtualSize + sizeof(uint32_t) + shaderContainer->physicalSize;
+ 
+                 if ((shaderContainer->flags & 0xFFFFFF00) == 0x102A1100 &&
+                     dataSize <= (fileSize - i) &&
+@@ -123,6 +128,8 @@ int main(int argc, char** argv)
+                         if (containerIndex > 0)
+                             stem += fmt::format(".{}", containerIndex);
+                         shader.first->second.sourceName = std::move(stem);
++                        shader.first->second.sourcePath = file.path().string();
++                        shader.first->second.sourceOffset = i;
+                         foundAny = true;
+                     }
+ 
+@@ -142,12 +149,36 @@ int main(int argc, char** argv)
+         if (!hlslDumpDir.empty())
+             std::filesystem::create_directories(hlslDumpDir);
+ 
++        if (const char* onlyHash = std::getenv("XENOS_RECOMP_ONLY_HASH"))
++        {
++            const XXH64_hash_t requestedHash = std::strtoull(onlyHash, nullptr, 0);
++            for (auto shader = shaders.begin(); shader != shaders.end();)
++            {
++                if (shader->first != requestedHash)
++                    shader = shaders.erase(shader);
++                else
++                    ++shader;
++            }
++            fmt::println("Hash filter 0x{:016X}: {} matching shader(s)", requestedHash, shaders.size());
++        }
++
+         std::atomic<uint32_t> progress = 0;
+ 
+-        std::for_each(std::execution::par_unseq, shaders.begin(), shaders.end(), [&](auto& hashShaderPair)
++        // DXC's COM implementation is unstable under this tool's parallel
++        // invocation on Windows. Deterministic AOT is more important here.
++        std::for_each(std::execution::seq, shaders.begin(), shaders.end(), [&](auto& hashShaderPair)
+             {
+                 auto& shader = hashShaderPair.second;
+                 const XXH64_hash_t hash = hashShaderPair.first;
++                if (std::getenv("XENOS_RECOMP_VERBOSE_RANGE") != nullptr)
++                {
++                    const auto container = reinterpret_cast<const ShaderContainer*>(shader.data);
++                    const auto metadata = reinterpret_cast<const Shader*>(shader.data + container->shaderOffset);
++                    fmt::println(stderr, "Container virtual={} physical={} shaderOffset={} codeOffset={} codeSize={}",
++                        uint32_t(container->virtualSize), uint32_t(container->physicalSize),
++                        uint32_t(container->shaderOffset), uint32_t(metadata->physicalOffset), uint32_t(metadata->size));
++                    fmt::println(stderr, "Source path={} offset={}", shader.sourcePath, shader.sourceOffset);
++                }
+ 
+                 auto recordFailure = [hash](const char* reason)
+                 {
+@@ -159,6 +190,12 @@ int main(int argc, char** argv)
+                 recompiler = {};
+                 recompiler.recompile(shader.data, include);
+ 
++                if (!recompiler.valid)
++                {
++                    recordFailure("invalid-microcode-range");
++                    return;
++                }
++
+                 shader.specConstantsMask = recompiler.specConstantsMask;
+ 
+                 if (!hlslDumpDir.empty())
+@@ -193,35 +230,69 @@ int main(int argc, char** argv)
+                 }
+ #endif
+ 
+-                IDxcBlob* spirv = dxcCompiler.compile(recompiler.out, recompiler.isPixelShader, false, true);
+-                if (spirv == nullptr)
++                if (!dxilOnly)
+                 {
+-                    recordFailure("dxc-spirv-compile-failed");
+-                    return;
+-                }
++                    IDxcBlob* spirv = dxcCompiler.compile(recompiler.out, recompiler.isPixelShader, false, true);
++                    if (spirv == nullptr)
++                    {
++                        recordFailure("dxc-spirv-compile-failed");
++                        return;
++                    }
++
++                    if (!smolv::Encode(spirv->GetBufferPointer(), spirv->GetBufferSize(), shader.spirv, smolv::kEncodeFlagStripDebugInfo))
++                    {
++                        spirv->Release();
++                        recordFailure("smolv-encode-failed");
++                        return;
++                    }
+ 
+-                if (!smolv::Encode(spirv->GetBufferPointer(), spirv->GetBufferSize(), shader.spirv, smolv::kEncodeFlagStripDebugInfo))
+-                {
+                     spirv->Release();
+-                    recordFailure("smolv-encode-failed");
+-                    return;
+                 }
+ 
+-                spirv->Release();
+-
+                 size_t currentProgress = ++progress;
+-                if ((currentProgress % 10) == 0 || (currentProgress == shaders.size() - 1))
++                if ((currentProgress % 1000) == 0 || currentProgress == shaders.size())
+                     fmt::println("Recompiling shaders... {}%", currentProgress / float(shaders.size()) * 100.0f);
+             });
+ 
++        const size_t discoveredShaderCount = shaders.size();
++        std::map<std::string, size_t> failureCounts;
++        for (const auto& failure : g_failures)
++            ++failureCounts[failure.reason];
++
++        for (auto shader = shaders.begin(); shader != shaders.end();)
++        {
++#ifdef XENOS_RECOMP_DXIL
++            const bool usable = shader->second.dxil != nullptr && (dxilOnly || !shader->second.spirv.empty());
++#else
++            const bool usable = !shader->second.spirv.empty();
++#endif
++            if (!usable)
++                shader = shaders.erase(shader);
++            else
++                ++shader;
++        }
++
+         if (!g_failures.empty())
+         {
+-            fmt::println(stderr, "Recompile failures ({}):", g_failures.size());
+-            for (const auto& failure : g_failures)
+-                fmt::println(stderr, "  hash=0x{:016X} reason={}", failure.hash, failure.reason);
+-            return 2;
++            fmt::println(stderr, "Recompile failures: {} of {} unique shaders", g_failures.size(), discoveredShaderCount);
++            for (const auto& [reason, count] : failureCounts)
++                fmt::println(stderr, "  {}: {}", reason, count);
++
++            const size_t detailCount = std::min<size_t>(g_failures.size(), 64);
++            fmt::println(stderr, "First {} failed hashes:", detailCount);
++            for (size_t i = 0; i < detailCount; ++i)
++                fmt::println(stderr, "  hash=0x{:016X} reason={}", g_failures[i].hash, g_failures[i].reason);
++            if (g_failures.size() > detailCount)
++                fmt::println(stderr, "  ... {} additional failures omitted", g_failures.size() - detailCount);
+         }
+ 
++        fmt::println("AOT coverage: {} of {} unique shaders ({:.2f}%)",
++            shaders.size(), discoveredShaderCount,
++            discoveredShaderCount == 0 ? 100.0 : shaders.size() * 100.0 / discoveredShaderCount);
++
++        if (shaders.empty())
++            return g_failures.empty() ? 0 : 2;
++
+         fmt::println("Creating shader cache...");
+ 
+         StringBuffer f;
+@@ -286,6 +357,9 @@ int main(int argc, char** argv)
+         f.println("const size_t g_shaderCacheEntryCount = {};", shaders.size());
+ 
+         writeAllBytes(output, f.out.data(), f.out.size());
++
++        if (!g_failures.empty())
++            return 2;
+     }
+     else
+     {
+diff --git a/XenosRecomp/shader_common.h b/XenosRecomp/shader_common.h
+index 63c37c9..64e2d6c 100644
+--- a/XenosRecomp/shader_common.h
++++ b/XenosRecomp/shader_common.h
+@@ -44,17 +44,17 @@ struct PushConstants
+ 
+ #ifdef REBLUE_RECOMP
+ // 256-bit boolean register file (BD bool addresses reach ~158), then per-usage 16-bit-pair swap masks.
+-#define g_Booleans(i)              vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 256 + (i)*4)
+-#define g_SwappedTexcoords         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 288)
+-#define g_HalfPixelOffset          vk::RawBufferLoad<float2>(g_PushConstants.SharedConstants + 292)
+-#define g_AlphaThreshold           vk::RawBufferLoad<float>(g_PushConstants.SharedConstants + 300)
+-#define g_SwappedNormals           vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 304)
+-#define g_SwappedBinormals         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 308)
+-#define g_SwappedTangents          vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 312)
+-#define g_SwappedBlendWeights      vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 316)
+-#define g_SwappedPositions         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 320)
+-#define g_SintTexcoords            vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 324)
+-#define g_ShadowPcfScale           vk::RawBufferLoad<float>(g_PushConstants.SharedConstants + 328)
++#define g_Booleans(i)              vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 512 + (i)*4)
++#define g_SwappedTexcoords         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 544)
++#define g_HalfPixelOffset          vk::RawBufferLoad<float2>(g_PushConstants.SharedConstants + 548)
++#define g_AlphaThreshold           vk::RawBufferLoad<float>(g_PushConstants.SharedConstants + 556)
++#define g_SwappedNormals           vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 560)
++#define g_SwappedBinormals         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 564)
++#define g_SwappedTangents          vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 568)
++#define g_SwappedBlendWeights      vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 572)
++#define g_SwappedPositions         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 576)
++#define g_SintTexcoords            vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 580)
++#define g_ShadowPcfScale           vk::RawBufferLoad<float>(g_PushConstants.SharedConstants + 584)
+ #else
+ #define g_Booleans                 vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 256)
+ #define g_SwappedTexcoords         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 260)
+@@ -70,17 +70,17 @@ struct PushConstants
+ 
+ #ifdef REBLUE_RECOMP
+ #define DEFINE_SHARED_CONSTANTS() \
+-    uint4 g_BooleansArr[2] : packoffset(c16); \
+-    uint g_SwappedTexcoords : packoffset(c18.x); \
+-    float2 g_HalfPixelOffset : packoffset(c18.y); \
+-    float g_AlphaThreshold : packoffset(c18.w); \
+-    uint g_SwappedNormals : packoffset(c19.x); \
+-    uint g_SwappedBinormals : packoffset(c19.y); \
+-    uint g_SwappedTangents : packoffset(c19.z); \
+-    uint g_SwappedBlendWeights : packoffset(c19.w); \
+-    uint g_SwappedPositions : packoffset(c20.x); \
+-    uint g_SintTexcoords : packoffset(c20.y); \
+-    float g_ShadowPcfScale : packoffset(c20.z);
++    uint4 g_BooleansArr[2] : packoffset(c32); \
++    uint g_SwappedTexcoords : packoffset(c34.x); \
++    float2 g_HalfPixelOffset : packoffset(c34.y); \
++    float g_AlphaThreshold : packoffset(c34.w); \
++    uint g_SwappedNormals : packoffset(c35.x); \
++    uint g_SwappedBinormals : packoffset(c35.y); \
++    uint g_SwappedTangents : packoffset(c35.z); \
++    uint g_SwappedBlendWeights : packoffset(c35.w); \
++    uint g_SwappedPositions : packoffset(c36.x); \
++    uint g_SintTexcoords : packoffset(c36.y); \
++    float g_ShadowPcfScale : packoffset(c36.z);
+ 
+ #define g_Booleans(i) (g_BooleansArr[(i) / 4][(i) % 4])
+ #else
+@@ -118,6 +118,12 @@ float4 tfetch2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float
+     return texture.Sample(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord + offset / getTexture2DDimensions(texture));
+ }
+ 
++float4 tfetchVertex2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float2 texCoord, float2 offset)
++{
++    Texture2D<float4> texture = g_Texture2DDescriptorHeap[resourceDescriptorIndex];
++    return texture.SampleLevel(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord + offset / getTexture2DDimensions(texture), 0.0);
++}
++
+ float2 getWeights2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float2 texCoord, float2 offset)
+ {
+     Texture2D<float4> texture = g_Texture2DDescriptorHeap[resourceDescriptorIndex];
+@@ -221,6 +227,11 @@ float4 tfetch3D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float
+     return g_Texture3DDescriptorHeap[resourceDescriptorIndex].Sample(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord);
+ }
+ 
++float4 tfetchVertex3D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float3 texCoord)
++{
++    return g_Texture3DDescriptorHeap[resourceDescriptorIndex].SampleLevel(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord, 0.0);
++}
++
+ struct CubeMapData
+ {
+ 	#ifdef REBLUE_RECOMP
+@@ -239,6 +250,11 @@ float4 tfetchCube(uint resourceDescriptorIndex, uint samplerDescriptorIndex, flo
+     return g_TextureCubeDescriptorHeap[resourceDescriptorIndex].Sample(g_SamplerDescriptorHeap[samplerDescriptorIndex], cubeMapData.cubeMapDirections[texCoord.z]);
+ }
+ 
++float4 tfetchVertexCube(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float3 texCoord, inout CubeMapData cubeMapData)
++{
++    return g_TextureCubeDescriptorHeap[resourceDescriptorIndex].SampleLevel(g_SamplerDescriptorHeap[samplerDescriptorIndex], cubeMapData.cubeMapDirections[texCoord.z], 0.0);
++}
++
+ #ifdef REBLUE_RECOMP
+ // DEC3N normal decode; IA binds as R32_UINT so lane .x carries the raw bits (asuint recovers them).
+ float4 tfetchR11G11B10(float4 value)
+diff --git a/XenosRecomp/shader_recompiler.cpp b/XenosRecomp/shader_recompiler.cpp
+index feaa90b..5e34dc0 100644
+--- a/XenosRecomp/shader_recompiler.cpp
++++ b/XenosRecomp/shader_recompiler.cpp
+@@ -372,7 +372,7 @@ void ShaderRecompiler::recompile(const TextureFetchInstruction& instr, bool bicu
+             out += "1.0 - ";
+     #endif
+ 
+-        out += "tfetch";
++        out += isPixelShader ? "tfetch" : "tfetchVertex";
+         break;
+     }
+     case FetchOpcode::GetTextureWeights:
+@@ -554,37 +554,27 @@ void ShaderRecompiler::recompile(const AluInstruction& instr)
+             {
+                 regFormatted = fmt::format("r{}", reg);
+             }
++            else if (literalFloat4Constants.find(reg) != literalFloat4Constants.end())
++            {
++                // Definition-table constants are materialized as locals below.
++                regFormatted = fmt::format("c{}", reg);
++            }
+             else
+             {
+-                auto findResult = float4Constants.find(reg);
+-                if (findResult != float4Constants.end())
++                // LEGO uses constants that are absent from reflection and also
++                // indexes unnamed ranges dynamically. Always address the full
++                // hardware constant bank; named reflection entries are useful
++                // metadata, but are not a safe declaration boundary.
++                const bool relative = instr.const0Relative || instr.const1Relative;
++                if (relative)
+                 {
+-                    const char* constantName = reinterpret_cast<const char*>(constantTableData + findResult->second->name);
+-                    if (findResult->second->registerCount > 1)
+-                    {
+-                    #ifdef UNLEASHED_RECOMP
+-                        if (hasMtxProjection && strcmp(constantName, "g_MtxProjection") == 0)
+-                        {
+-                            regFormatted = fmt::format("(iterationIndex == 0 ? mtxProjectionReverseZ[{0}] : mtxProjection[{0}])",
+-                                reg - findResult->second->registerIndex);
+-                        }
+-                        else
+-                    #endif
+-                        {
+-                            regFormatted = fmt::format("{}({}{})", constantName,
+-                                reg - findResult->second->registerIndex, instr.const0Relative ? (instr.constAddressRegisterRelative ? " + a0" : " + aL") : "");
+-                        }
+-                    }
+-                    else
+-                    {
+-                        assert(!instr.const0Relative && !instr.const1Relative);
+-                        regFormatted = constantName;
+-                    }
++                    regFormatted = fmt::format("LOAD_C(clamp({} + {}, 0, {}))", reg,
++                        instr.constAddressRegisterRelative ? "a0" : "aL",
++                        isPixelShader ? 223 : 255);
+                 }
+                 else
+                 {
+-                    assert(!instr.const0Relative && !instr.const1Relative);
+-                    regFormatted = fmt::format("c{}", reg);
++                    regFormatted = fmt::format("LOAD_C({})", reg);
+                 }
+             }
+ 
+@@ -1368,11 +1358,11 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+             for (size_t j = 0; j < std::size(TEXTURE_DIMENSIONS); j++)
+             {
+                 println("#define {}_Texture{}DescriptorIndex vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + {})",
+-                    constantName, TEXTURE_DIMENSIONS[j], j * 64 + constantInfo->registerIndex * 4);
++                    constantName, TEXTURE_DIMENSIONS[j], j * 128 + constantInfo->registerIndex * 4);
+             }
+ 
+             println("#define {}_SamplerDescriptorIndex vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + {})",
+-                constantName, std::size(TEXTURE_DIMENSIONS) * 64 + constantInfo->registerIndex * 4);
++                constantName, std::size(TEXTURE_DIMENSIONS) * 128 + constantInfo->registerIndex * 4);
+ 
+             samplers.emplace(constantInfo->registerIndex, constantName);
+             break;
+@@ -1382,18 +1372,18 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+     }
+ 
+ #ifdef REBLUE_RECOMP
+-    // BD bodies reference unnamed sampler slots; emit fallback descriptor-index defines for any slot 0..15 the table didn't name.
+-    for (uint32_t r = 0; r < 16; r++)
++    // LEGO uses the complete five-bit Xenos texture-constant bank.
++    for (uint32_t r = 0; r < 32; r++)
+     {
+         if (samplers.find(r) != samplers.end())
+             continue;
+         for (size_t j = 0; j < std::size(TEXTURE_DIMENSIONS); j++)
+         {
+             println("#define s{}_Texture{}DescriptorIndex vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + {})",
+-                r, TEXTURE_DIMENSIONS[j], j * 64 + r * 4);
++                r, TEXTURE_DIMENSIONS[j], j * 128 + r * 4);
+         }
+         println("#define s{}_SamplerDescriptorIndex vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + {})",
+-            r, std::size(TEXTURE_DIMENSIONS) * 64 + r * 4);
++            r, std::size(TEXTURE_DIMENSIONS) * 128 + r * 4);
+     }
+ #endif
+ 
+@@ -1401,37 +1391,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+ 
+     println("cbuffer {}ShaderConstants : register(b{}, space4)", isPixelShader ? "Pixel" : "Vertex", isPixelShader ? 1 : 0);
+     out += "{\n";
+-
+-    for (uint32_t i = 0; i < constantTableContainer->constantTable.constants; i++)
+-    {
+-        const auto constantInfo = reinterpret_cast<const ConstantInfo*>(
+-            constantTableData + constantTableContainer->constantTable.constantInfo + i * sizeof(ConstantInfo));
+-
+-        if (constantInfo->registerSet == RegisterSet::Float4)
+-        {
+-#ifdef REBLUE_RECOMP
+-            // Only the alias winner gets a packoffset slot; a loser would overlap it in the cbuffer and fail DXC.
+-            auto winner = float4Constants.find(constantInfo->registerIndex);
+-            if (winner == float4Constants.end() || winner->second != constantInfo)
+-                continue;
+-#endif
+-
+-            const char* constantName = reinterpret_cast<const char*>(constantTableData + constantInfo->name);
+-
+-            print("\tfloat4 {}", constantName);
+-
+-            if (constantInfo->registerCount > 1)
+-                print("[{}]", constantInfo->registerCount.get());
+-
+-            println(" : packoffset(c{});", constantInfo->registerIndex.get());
+-
+-            if (constantInfo->registerCount > 1)
+-            {
+-                uint32_t tailCount = (isPixelShader ? 224 : 256) - constantInfo->registerIndex;
+-                println("#define {0}(INDEX) select((INDEX) < {1}, {0}[min(INDEX, {2})], 0.0)", constantName, tailCount, tailCount - 1);
+-            }
+-        }
+-    }
++    println("\tfloat4 g_FloatConstants[{}];", isPixelShader ? 224 : 256);
+ 
+     out += "};\n\n";
+ 
+@@ -1450,33 +1410,39 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+             for (size_t j = 0; j < std::size(TEXTURE_DIMENSIONS); j++)
+             {
+                 println("\tuint {}_Texture{}DescriptorIndex : packoffset(c{}.{});",
+-                    constantName, TEXTURE_DIMENSIONS[j], j * 4 + constantInfo->registerIndex / 4, SWIZZLES[constantInfo->registerIndex % 4]);
++                    constantName, TEXTURE_DIMENSIONS[j], j * 8 + constantInfo->registerIndex / 4, SWIZZLES[constantInfo->registerIndex % 4]);
+             }
+ 
+             println("\tuint {}_SamplerDescriptorIndex : packoffset(c{}.{});",
+-                constantName, 4 * std::size(TEXTURE_DIMENSIONS) + constantInfo->registerIndex / 4, SWIZZLES[constantInfo->registerIndex % 4]);
++                constantName, 8 * std::size(TEXTURE_DIMENSIONS) + constantInfo->registerIndex / 4, SWIZZLES[constantInfo->registerIndex % 4]);
+         }
+     }
+ 
+ #ifdef REBLUE_RECOMP
+     // Mirror the SPIR-V fallback: packoffset slots for any unnamed sampler index 0..15.
+-    for (uint32_t r = 0; r < 16; r++)
++    for (uint32_t r = 0; r < 32; r++)
+     {
+         if (samplers.find(r) != samplers.end())
+             continue;
+         for (size_t j = 0; j < std::size(TEXTURE_DIMENSIONS); j++)
+         {
+             println("\tuint s{}_Texture{}DescriptorIndex : packoffset(c{}.{});",
+-                r, TEXTURE_DIMENSIONS[j], j * 4 + r / 4, SWIZZLES[r % 4]);
++                r, TEXTURE_DIMENSIONS[j], j * 8 + r / 4, SWIZZLES[r % 4]);
+         }
+         println("\tuint s{}_SamplerDescriptorIndex : packoffset(c{}.{});",
+-            r, 4 * std::size(TEXTURE_DIMENSIONS) + r / 4, SWIZZLES[r % 4]);
++            r, 8 * std::size(TEXTURE_DIMENSIONS) + r / 4, SWIZZLES[r % 4]);
+     }
+ #endif
+ 
+     out += "\tDEFINE_SHARED_CONSTANTS();\n";
+     out += "};\n\n";
+ 
++    out += "#endif\n";
++    out += "#ifdef __spirv__\n";
++    println("#define LOAD_C(INDEX) vk::RawBufferLoad<float4>(g_PushConstants.{}ShaderConstants + (INDEX) * 16, 0x10)",
++        isPixelShader ? "Pixel" : "Vertex");
++    out += "#else\n";
++    out += "#define LOAD_C(INDEX) g_FloatConstants[INDEX]\n";
+     out += "#endif\n";
+ 
+     for (uint32_t i = 0; i < constantTableContainer->constantTable.constants; i++)
+@@ -1535,16 +1501,14 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+         out += "#endif\n";
+ 
+         auto pixelShader = reinterpret_cast<const PixelShader*>(shader);
+-        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_COLOR0)
+-            out += ",\n\tout float4 oC0 : SV_Target0";
+-        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_COLOR1)
+-            out += ",\n\tout float4 oC1 : SV_Target1";
+-        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_COLOR2)
+-            out += ",\n\tout float4 oC2 : SV_Target2";
+-        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_COLOR3)
+-            out += ",\n\tout float4 oC3 : SV_Target3";
+-        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_DEPTH)
+-            out += ",\n\tout float oDepth : SV_Depth";
++        // Some LEGO containers under-report MRT outputs in reflection while
++        // the microcode still exports them. A superset signature is harmless
++        // in D3D12 and prevents valid shaders from failing AOT compilation.
++        out += ",\n\tout float4 oC0 : SV_Target0";
++        out += ",\n\tout float4 oC1 : SV_Target1";
++        out += ",\n\tout float4 oC2 : SV_Target2";
++        out += ",\n\tout float4 oC3 : SV_Target3";
++        out += ",\n\tout float oDepth : SV_Depth";
+     }
+     else
+     {
+@@ -1606,6 +1570,21 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+     out += ")\n";
+     out += "{\n";
+ 
++    // Xenos exposes 64 temporary vector registers and 16 integer constants.
++    // Reflection in LEGO frequently omits integer definitions, so initialize
++    // the complete bank and let a definition table overwrite the slots it has.
++    for (uint32_t i = 0; i < 16; ++i)
++        println("\tint4 i{} = 0;", i);
++
++    if (isPixelShader)
++    {
++        out += "\toC0 = 0.0;\n";
++        out += "\toC1 = 0.0;\n";
++        out += "\toC2 = 0.0;\n";
++        out += "\toC3 = 0.0;\n";
++        out += "\toDepth = 0.0;\n";
++    }
++
+ #ifdef UNLEASHED_RECOMP
+     if (hasMtxProjection)
+     {
+@@ -1628,7 +1607,8 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+         while (*definitions != 0)
+         {
+             auto definition = reinterpret_cast<const Float4Definition*>(definitions);
+-            auto value = reinterpret_cast<const be<uint32_t>*>(shaderData + shaderContainer->virtualSize + definition->physicalOffset);
++            const uint32_t physicalBase = uint32_t(shaderContainer->virtualSize) + sizeof(uint32_t);
++            auto value = reinterpret_cast<const be<uint32_t>*>(shaderData + physicalBase + definition->physicalOffset);
+             for (uint16_t i = 0; i < (definition->count + 3) / 4; i++)
+             {
+ #ifdef REBLUE_RECOMP
+@@ -1659,11 +1639,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+                     }
+                     return fmt::format("asfloat(0x{:X}u)", u);
+                 };
++                literalFloat4Constants.emplace(definition->registerIndex + i - (isPixelShader ? 256 : 0));
+                 println("\tfloat4 c{} = float4({}, {}, {}, {});",
+                     definition->registerIndex + i - (isPixelShader ? 256 : 0),
+                     shadowKernelComponent(value[0].get()), shadowKernelComponent(value[1].get()),
+                     shadowKernelComponent(value[2].get()), shadowKernelComponent(value[3].get()));
+ #else
++                literalFloat4Constants.emplace(definition->registerIndex + i - (isPixelShader ? 256 : 0));
+                 println("\tfloat4 c{} = asfloat(uint4(0x{:X}, 0x{:X}, 0x{:X}, 0x{:X}));",
+                     definition->registerIndex + i - (isPixelShader ? 256 : 0), value[0].get(), value[1].get(), value[2].get(), value[3].get());
+ #endif
+@@ -1692,7 +1674,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+ 
+                 value = definition->values[i].get();
+ 
+-                println("\tint4 i{} = int4({}, {}, {}, {});",
++                println("\ti{} = int4({}, {}, {}, {});",
+                     (definition->registerIndex - 8992) / 4 + i, x, y, z, w);
+             }
+             definitions += 2;
+@@ -1702,7 +1684,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+         out += "\n";
+     }
+ 
+-    bool printedRegisters[32]{};
++    bool printedRegisters[64]{};
+ 
+     uint32_t interpolatorCount = (shader->interpolatorInfo >> 5) & 0x1F;
+ 
+@@ -1744,7 +1726,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+         out += "\n";
+     }
+ 
+-    for (size_t i = 0; i < 32; i++)
++    for (size_t i = 0; i < 64; i++)
+     {
+         if (!printedRegisters[i])
+         {
+@@ -1782,7 +1764,8 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+ #endif
+     }
+ 
+-    const be<uint32_t>* code = reinterpret_cast<const be<uint32_t>*>(shaderData + shaderContainer->virtualSize + shader->physicalOffset);
++    const uint32_t physicalBase = uint32_t(shaderContainer->virtualSize) + sizeof(uint32_t);
++    const be<uint32_t>* code = reinterpret_cast<const be<uint32_t>*>(shaderData + physicalBase + shader->physicalOffset);
+ 
+     union
+     {
+@@ -1798,7 +1781,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+ 
+     auto controlFlowCode = code;
+     uint32_t instrAddress = 0;
+-    uint32_t instrSize = shader->size;
++    const uint32_t shaderSize = shader->size;
++    uint32_t instrSize = shaderSize;
++    if ((shaderSize % 12) != 0)
++    {
++        valid = false;
++        return;
++    }
+     bool simpleControlFlow = true;
+ 
+     // Per-instruction control flow summary for the structurizer below.
+@@ -2209,6 +2198,16 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+             }
+             }
+ 
++            const uint32_t instructionCount = shaderSize / 12;
++            if (address > instructionCount || count > instructionCount - address)
++            {
++                if (std::getenv("XENOS_RECOMP_VERBOSE_RANGE") != nullptr)
++                    fmt::println(stderr, "Invalid instruction range: address={} count={} slots={} bytes={}",
++                        address, count, instructionCount, shaderSize);
++                valid = false;
++                return;
++            }
++
+             auto instructionCode = code + address * 3;
+             
+             for (uint32_t i = 0; i < count; i++)
+diff --git a/XenosRecomp/shader_recompiler.h b/XenosRecomp/shader_recompiler.h
+index 791def7..c7336ad 100644
+--- a/XenosRecomp/shader_recompiler.h
++++ b/XenosRecomp/shader_recompiler.h
+@@ -24,11 +24,13 @@ struct StringBuffer
+ struct ShaderRecompiler : StringBuffer
+ {
+     uint32_t indentation = 0;
++    bool valid = true;
+     bool isPixelShader = false;
+     const uint8_t* constantTableData = nullptr;
+     std::unordered_map<uint32_t, VertexElement> vertexElements;
+     std::unordered_map<uint32_t, std::string> interpolators;
+     std::unordered_map<uint32_t, const ConstantInfo*> float4Constants;
++    std::unordered_set<uint32_t> literalFloat4Constants;
+     std::unordered_map<uint32_t, const char*> boolConstants;
+     std::unordered_map<uint32_t, const char*> samplers;
+     std::unordered_map<uint32_t, uint32_t> ifEndLabels;
+
diff --git a/tools/gpu-shader-extract/.gitignore b/tools/gpu-shader-extract/.gitignore
new file mode 100644
index 0000000..f6f63c7
--- /dev/null
+++ b/tools/gpu-shader-extract/.gitignore
@@ -0,0 +1,3 @@
+bin/
+obj/
+
diff --git a/tools/gpu-shader-extract/Program.cs b/tools/gpu-shader-extract/Program.cs
new file mode 100644
index 0000000..b19f13d
--- /dev/null
+++ b/tools/gpu-shader-extract/Program.cs
@@ -0,0 +1,126 @@
+using System.Globalization;
+using DimensionsModManager;
+
+namespace GpuShaderExtract;
+
+internal static class Program
+{
+    private static readonly string[] ShaderExtensions =
+    {
+        ".vso", ".pso", ".vsh", ".psh", ".fxo", ".shader", ".bin"
+    };
+
+    private static int Main(string[] args)
+    {
+        if (args.Length < 2 || (args[0] != "inventory" && args[0] != "extract"))
+        {
+            Console.Error.WriteLine(
+                "Usage: gpu-shader-extract <inventory|extract> <game-dir> [output-dir]");
+            return 2;
+        }
+
+        string gameDir = Path.GetFullPath(args[1]);
+        if (!Directory.Exists(gameDir))
+        {
+            Console.Error.WriteLine($"Game directory not found: {gameDir}");
+            return 2;
+        }
+
+        string? outputDir = args.Length >= 3 ? Path.GetFullPath(args[2]) : null;
+        if (args[0] == "extract" && outputDir is null)
+        {
+            Console.Error.WriteLine("extract requires an output directory");
+            return 2;
+        }
+
+        var extensionCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+        int namedEntries = 0;
+        int extractedEntries = 0;
+        int skippedEntries = 0;
+
+        foreach (string datPath in Directory.EnumerateFiles(gameDir, "*.DAT",
+                                                             SearchOption.TopDirectoryOnly)
+                                                .OrderBy(path => path,
+                                                         StringComparer.OrdinalIgnoreCase))
+        {
+            string hdrPath = DatArchive.HdrPathFor(datPath);
+            if (!File.Exists(hdrPath))
+            {
+                Console.WriteLine($"skip {Path.GetFileName(datPath)}: no matching header");
+                continue;
+            }
+
+            var archive = new DatArchive(datPath);
+            var names = archive.EnumerateNames().OrderBy(pair => pair.Key,
+                                                          StringComparer.OrdinalIgnoreCase)
+                               .ToArray();
+            namedEntries += names.Length;
+            Console.WriteLine($"{Path.GetFileName(datPath)}: {archive.FileCount} entries, " +
+                              $"{names.Length} named");
+
+            foreach ((string internalPath, int entryIndex) in names)
+            {
+                string extension = Path.GetExtension(internalPath);
+                extension = string.IsNullOrEmpty(extension) ? "<none>" : extension.ToLowerInvariant();
+                extensionCounts.TryGetValue(extension, out int count);
+                extensionCounts[extension] = count + 1;
+
+                if (args[0] != "extract" || !IsShaderCandidate(internalPath))
+                {
+                    continue;
+                }
+
+                byte[] data;
+                try
+                {
+                    data = archive.ReadEntryDataDecompressed(entryIndex);
+                }
+                catch (Exception exception) when (exception is IOException or
+                                                  InvalidDataException or
+                                                  ArgumentOutOfRangeException)
+                {
+                    // Some optional install archives are intentionally sparse:
+                    // their headers describe files supplied by another disc or
+                    // install phase, while the local DAT ends before the slot.
+                    // A missing optional entry must not discard thousands of
+                    // valid shaders extracted from the other archives.
+                    Console.Error.WriteLine(
+                        $"skip {Path.GetFileName(datPath)}:{internalPath}: {exception.Message}");
+                    skippedEntries++;
+                    continue;
+                }
+                string archiveName = Path.GetFileNameWithoutExtension(datPath);
+                string safeRelative = internalPath.Replace('\\', Path.DirectorySeparatorChar)
+                                                  .Replace('/', Path.DirectorySeparatorChar)
+                                                  .TrimStart(Path.DirectorySeparatorChar);
+                string destination = Path.Combine(outputDir!, archiveName, safeRelative);
+                Directory.CreateDirectory(Path.GetDirectoryName(destination)!);
+                File.WriteAllBytes(destination, data);
+                extractedEntries++;
+            }
+        }
+
+        Console.WriteLine($"named entries: {namedEntries.ToString(CultureInfo.InvariantCulture)}");
+        foreach ((string extension, int count) in extensionCounts
+                     .OrderByDescending(pair => pair.Value)
+                     .ThenBy(pair => pair.Key, StringComparer.OrdinalIgnoreCase))
+        {
+            Console.WriteLine($"{count,7} {extension}");
+        }
+
+        if (args[0] == "extract")
+        {
+            Console.WriteLine($"extracted shader candidates: {extractedEntries}");
+            Console.WriteLine($"skipped unreadable candidates: {skippedEntries}");
+            return extractedEntries == 0 ? 1 : 0;
+        }
+        return 0;
+    }
+
+    private static bool IsShaderCandidate(string path)
+    {
+        string extension = Path.GetExtension(path);
+        return ShaderExtensions.Contains(extension, StringComparer.OrdinalIgnoreCase) ||
+               path.Contains("shader", StringComparison.OrdinalIgnoreCase);
+    }
+}
diff --git a/tools/gpu-shader-extract/README.md b/tools/gpu-shader-extract/README.md
new file mode 100644
index 0000000..2e577b1
--- /dev/null
+++ b/tools/gpu-shader-extract/README.md
@@ -0,0 +1,15 @@
+# GPU shader extractor
+
+Read-only inventory/extraction helper for the native renderer work. It reuses
+the project's TT Games DAT/HDR and DFLT readers and never opens an archive for
+writing.
+
+```powershell
+gpu-shader-extract inventory <game-directory>
+gpu-shader-extract extract <game-directory> <output-directory>
+```
+
+`inventory` reports the file extensions present in every classic disc archive.
+`extract` writes only likely shader-container files (`.vso`, `.pso`, related
+extensions, and paths containing `shader`) beneath one directory per archive.
+The extracted directory can be passed directly to XenosRecomp.
diff --git a/tools/gpu-shader-extract/gpu-shader-extract.csproj b/tools/gpu-shader-extract/gpu-shader-extract.csproj
new file mode 100644
index 0000000..7b48572
--- /dev/null
+++ b/tools/gpu-shader-extract/gpu-shader-extract.csproj
@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AssemblyName>gpu-shader-extract</AssemblyName>
+    <RootNamespace>GpuShaderExtract</RootNamespace>
+  </PropertyGroup>
+
+  <!-- Reuse the archive reader that the mod manager and installer already
+       exercise. These are links, so the extraction tool cannot silently drift
+       to a second interpretation of TT's DAT/HDR format. -->
+  <ItemGroup>
+    <Compile Include="..\..\DimensionsModManager\DatArchive.cs"
+             Link="Archive\DatArchive.cs" />
+    <Compile Include="..\..\DimensionsModManager\Dflt.cs"
+             Link="Archive\Dflt.cs" />
+  </ItemGroup>
+</Project>

From 60902d9aced22f86dc70c35876901fc86a9da7a8 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 14:09:21 +0300
Subject: [PATCH 04/44] gpu: bootstrap native D3D12 device

---
 .gitignore                                    |     6 +
 .gitmodules                                   |     3 +
 rexlego/CMakeLists.txt                        |    98 +
 rexlego/src/gpu/shaders/linked_shader_cache.h |     7 +
 rexlego/src/gpu/shaders/shader_cache.h        |     7 +
 rexlego/src/gpu_native/d3d.h                  |    64 +
 rexlego/src/gpu_native/device.cpp             |   129 +
 rexlego/src/gpu_native/device.h               |    41 +
 rexlego/src/gpu_native/dxc_link.cpp           |   127 +
 rexlego/src/gpu_native/dxc_link.h             |    31 +
 rexlego/src/gpu_native/hooks_device.cpp       |    58 +
 rexlego/src/gpu_native/linked_shader_cache.h  |    31 +
 rexlego/src/gpu_native/shader_archive.cpp     |   127 +
 rexlego/src/gpu_native/shader_archive.h       |    57 +
 rexlego/src/legodimensions_app.h              |    22 +
 thirdparty/miniz/miniz.cpp                    |     4 +
 thirdparty/miniz/miniz.h                      |  5100 ++++
 thirdparty/plume                              |     1 +
 thirdparty/zstd/LICENSE                       |    30 +
 thirdparty/zstd/zstd.h                        |  3142 +++
 thirdparty/zstd/zstd_errors.h                 |   106 +
 thirdparty/zstd/zstddeclib.c                  | 22228 ++++++++++++++++
 tools/gpu-aot/d3d-hook-map.md                 |    66 +
 tools/gpu-aot/prelink_shader_cache.cpp        |   198 +
 24 files changed, 31683 insertions(+)
 create mode 100644 rexlego/src/gpu/shaders/linked_shader_cache.h
 create mode 100644 rexlego/src/gpu/shaders/shader_cache.h
 create mode 100644 rexlego/src/gpu_native/d3d.h
 create mode 100644 rexlego/src/gpu_native/device.cpp
 create mode 100644 rexlego/src/gpu_native/device.h
 create mode 100644 rexlego/src/gpu_native/dxc_link.cpp
 create mode 100644 rexlego/src/gpu_native/dxc_link.h
 create mode 100644 rexlego/src/gpu_native/hooks_device.cpp
 create mode 100644 rexlego/src/gpu_native/linked_shader_cache.h
 create mode 100644 rexlego/src/gpu_native/shader_archive.cpp
 create mode 100644 rexlego/src/gpu_native/shader_archive.h
 create mode 100644 thirdparty/miniz/miniz.cpp
 create mode 100644 thirdparty/miniz/miniz.h
 create mode 160000 thirdparty/plume
 create mode 100644 thirdparty/zstd/LICENSE
 create mode 100644 thirdparty/zstd/zstd.h
 create mode 100644 thirdparty/zstd/zstd_errors.h
 create mode 100644 thirdparty/zstd/zstddeclib.c
 create mode 100644 tools/gpu-aot/d3d-hook-map.md
 create mode 100644 tools/gpu-aot/prelink_shader_cache.cpp

diff --git a/.gitignore b/.gitignore
index 16d749d..469dc21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,12 @@
 !/rexglue-sdk
 !/DimensionsModManager
 !/DimensionsSaveConverter
+!/thirdparty/
+!/thirdparty/plume
+!/thirdparty/miniz/
+!/thirdparty/miniz/**
+!/thirdparty/zstd/
+!/thirdparty/zstd/**
 !/xexdump/
 !/tools/
 
diff --git a/.gitmodules b/.gitmodules
index 1671731..d983f12 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -8,3 +8,6 @@
 [submodule "DimensionsSaveConverter"]
 	path = DimensionsSaveConverter
 	url = https://github.com/NeverCookFirst/DimensionsSaveConverter.git
+[submodule "thirdparty/plume"]
+	path = thirdparty/plume
+	url = https://github.com/zolaware/plume.git
diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index d8a6e3c..728d76e 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -16,6 +16,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 include(generated/rexglue.cmake)
 
+# Native D3D12 renderer work stays opt-in until its hook map and presentation
+# path are complete. The shipping rexgpu-xenos path is untouched when this is
+# disabled.
+option(LEGODIMENSIONS_NATIVE_GPU "Build the experimental native D3D12 renderer" OFF)
+
 # Sources
 set(LEGODIMENSIONS_SOURCES
     src/main.cpp
@@ -31,6 +36,33 @@ set(LEGODIMENSIONS_SOURCES
     src/name_trace.cpp
 )
 
+if(LEGODIMENSIONS_NATIVE_GPU)
+    if(NOT WIN32)
+        message(FATAL_ERROR "LEGODIMENSIONS_NATIVE_GPU currently supports Windows D3D12 only")
+    endif()
+
+    enable_language(C)
+    set(PLUME_D3D12_AGILITY_SDK_ENABLED OFF CACHE BOOL "" FORCE)
+    add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/plume"
+                     "${CMAKE_CURRENT_BINARY_DIR}/thirdparty/plume"
+                     EXCLUDE_FROM_ALL)
+
+    # Plume enables its Vulkan backend unconditionally upstream. This build is
+    # deliberately D3D12-only, so keep the shared interface and Win32 backend
+    # but do not compile or link the unused Vulkan implementation.
+    get_target_property(LEGO_PLUME_SOURCES plume SOURCES)
+    list(FILTER LEGO_PLUME_SOURCES EXCLUDE REGEX "plume_vulkan\\.(cpp|h)$")
+    set_property(TARGET plume PROPERTY SOURCES "${LEGO_PLUME_SOURCES}")
+
+    list(APPEND LEGODIMENSIONS_SOURCES
+        src/gpu_native/device.cpp
+        src/gpu_native/hooks_device.cpp
+        src/gpu_native/shader_archive.cpp
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c"
+    )
+endif()
+
 # Research probes: a guard-page watchpoint on the area flags and the failed
 # attempt at forcing the 3D portrait. They are off in a release because one of
 # them single-steps every read of a watched page, which is slow enough to look
@@ -52,6 +84,72 @@ endif()
 
 rexglue_setup_target(legodimensions)
 
+if(LEGODIMENSIONS_NATIVE_GPU)
+    target_compile_definitions(legodimensions PRIVATE LEGODIMENSIONS_NATIVE_GPU=1)
+    target_include_directories(legodimensions PRIVATE src)
+    target_include_directories(legodimensions PRIVATE
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd")
+    target_link_libraries(legodimensions PRIVATE plume d3d12 dxgi rex::xxhash)
+
+    # Generated locally by tools/gpu-aot. Keeping it out of Git avoids a
+    # multi-megabyte generated source file while still making an enabled build
+    # fail loudly if the AOT stage has not been run.
+    set(LEGO_NATIVE_SHADER_CACHE
+        "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/lego-dxil-cache.cpp")
+    if(NOT EXISTS "${LEGO_NATIVE_SHADER_CACHE}")
+        message(FATAL_ERROR
+            "Native GPU needs ${LEGO_NATIVE_SHADER_CACHE}; run tools/gpu-aot first")
+    endif()
+    target_sources(legodimensions PRIVATE "${LEGO_NATIVE_SHADER_CACHE}")
+    set_source_files_properties("${LEGO_NATIVE_SHADER_CACHE}"
+        PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+
+    # XenosRecomp emits a DXIL library whenever a shader reads one of its two
+    # specialization bits. Link every used value at build time; gameplay then
+    # creates complete shaders directly and never invokes DXC on a frame.
+    set(LEGO_NATIVE_LINKED_SHADER_CACHE
+        "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/lego-linked-dxil-cache.cpp")
+    set(LEGO_DXC_ROOT
+        "${CMAKE_CURRENT_SOURCE_DIR}/../research/reblue/thirdparty/XenosRecomp/thirdparty/dxc-bin"
+        CACHE PATH "Pinned DXC binary package used by the LEGO GPU AOT prelinker")
+    set(LEGO_DXC_ARCH x64)
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "ARM64|arm64|aarch64")
+        set(LEGO_DXC_ARCH arm64)
+    endif()
+
+    add_executable(lego_gpu_prelink EXCLUDE_FROM_ALL
+        "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/prelink_shader_cache.cpp"
+        src/gpu_native/dxc_link.cpp
+        "${LEGO_NATIVE_SHADER_CACHE}"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c")
+    target_include_directories(lego_gpu_prelink PRIVATE
+        src
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd"
+        "${LEGO_DXC_ROOT}/inc")
+    target_link_libraries(lego_gpu_prelink PRIVATE
+        "${LEGO_DXC_ROOT}/lib/${LEGO_DXC_ARCH}/dxcompiler.lib")
+    add_custom_command(TARGET lego_gpu_prelink POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "${LEGO_DXC_ROOT}/bin/${LEGO_DXC_ARCH}/dxcompiler.dll"
+            "${LEGO_DXC_ROOT}/bin/${LEGO_DXC_ARCH}/dxil.dll"
+            "$<TARGET_FILE_DIR:lego_gpu_prelink>"
+        VERBATIM)
+
+    add_custom_command(
+        OUTPUT "${LEGO_NATIVE_LINKED_SHADER_CACHE}"
+        COMMAND "$<TARGET_FILE:lego_gpu_prelink>"
+                "${LEGO_NATIVE_LINKED_SHADER_CACHE}"
+        DEPENDS lego_gpu_prelink
+        COMMENT "Pre-linking LEGO specialization-constant shader variants"
+        VERBATIM)
+    target_sources(legodimensions PRIVATE "${LEGO_NATIVE_LINKED_SHADER_CACHE}")
+    set_source_files_properties("${LEGO_NATIVE_LINKED_SHADER_CACHE}"
+        PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+endif()
+
 # The achievement unlock sound, converted from the user-supplied MP3 to a quiet
 # 48 kHz WAV. Copied next to the executable because that is where the SDK cvar
 # ui_sound_achievement looks by default; point the cvar elsewhere to swap it.
diff --git a/rexlego/src/gpu/shaders/linked_shader_cache.h b/rexlego/src/gpu/shaders/linked_shader_cache.h
new file mode 100644
index 0000000..0971d86
--- /dev/null
+++ b/rexlego/src/gpu/shaders/linked_shader_cache.h
@@ -0,0 +1,7 @@
+/**
+ * Compatibility include for the source emitted by lego_gpu_prelink.
+ */
+
+#pragma once
+
+#include "gpu_native/linked_shader_cache.h"
diff --git a/rexlego/src/gpu/shaders/shader_cache.h b/rexlego/src/gpu/shaders/shader_cache.h
new file mode 100644
index 0000000..8667a7f
--- /dev/null
+++ b/rexlego/src/gpu/shaders/shader_cache.h
@@ -0,0 +1,7 @@
+/**
+ * Compatibility include for the source emitted by XenosRecomp.
+ */
+
+#pragma once
+
+#include "gpu_native/shader_archive.h"
diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h
new file mode 100644
index 0000000..ce1abd3
--- /dev/null
+++ b/rexlego/src/gpu_native/d3d.h
@@ -0,0 +1,64 @@
+// LEGO Dimensions TU23 guest D3D layouts used by the native renderer.
+#pragma once
+
+#include <cstddef>
+
+#include <rex/types.h>
+
+namespace legodimensions::gpu_native {
+
+constexpr u32 kGuestDeviceSize = 0x6080;
+constexpr u32 kRenderStateCount = 101;
+constexpr u32 kSamplerStateCount = 20;
+constexpr u32 kFetchConstantCount = 26;
+
+struct D3DViewport9 {
+  be_u32 x;
+  be_u32 y;
+  be_u32 width;
+  be_u32 height;
+  be_f32 min_z;
+  be_f32 max_z;
+};
+static_assert(sizeof(D3DViewport9) == 0x18);
+
+struct D3DRect {
+  be_i32 left;
+  be_i32 top;
+  be_i32 right;
+  be_i32 bottom;
+};
+static_assert(sizeof(D3DRect) == 0x10);
+
+struct FetchConstant {
+  be_u32 dword[6];
+};
+static_assert(sizeof(FetchConstant) == 0x18);
+
+// Only fields proven from TU23 code are named. Keep unknown spans opaque until
+// an accessor or state writer establishes their meaning.
+struct D3DDevice {
+  u8 unknown_0000[0x040];
+  be_u32 set_render_state[kRenderStateCount];
+  be_u32 set_sampler_state[kSamplerStateCount];
+  be_u32 get_render_state[kRenderStateCount];
+  be_u32 get_sampler_state[kSamplerStateCount];
+  u8 unknown_0408[0x480 - 0x408];
+  FetchConstant fetch_constants[kFetchConstantCount];
+  u8 unknown_06F0[0x32E0 - 0x6F0];
+  D3DViewport9 viewport;
+  be_u32 viewport_reserved;
+  D3DRect scissor;
+  u8 unknown_330C[kGuestDeviceSize - 0x330C];
+};
+
+static_assert(sizeof(D3DDevice) == kGuestDeviceSize);
+static_assert(offsetof(D3DDevice, set_render_state) == 0x040);
+static_assert(offsetof(D3DDevice, set_sampler_state) == 0x1D4);
+static_assert(offsetof(D3DDevice, get_render_state) == 0x224);
+static_assert(offsetof(D3DDevice, get_sampler_state) == 0x3B8);
+static_assert(offsetof(D3DDevice, fetch_constants) == 0x480);
+static_assert(offsetof(D3DDevice, viewport) == 0x32E0);
+static_assert(offsetof(D3DDevice, scissor) == 0x32FC);
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
new file mode 100644
index 0000000..8e2dc50
--- /dev/null
+++ b/rexlego/src/gpu_native/device.cpp
@@ -0,0 +1,129 @@
+#include "gpu_native/device.h"
+
+#include <mutex>
+#include <string>
+
+#include <plume_render_interface.h>
+#include <rex/logging.h>
+#include <rex/ui/window.h>
+
+namespace plume {
+std::unique_ptr<RenderInterface> CreateD3D12Interface();
+}  // namespace plume
+
+namespace legodimensions::gpu_native {
+namespace {
+
+struct State {
+  std::unique_ptr<plume::RenderInterface> render_interface;
+  std::unique_ptr<plume::RenderDevice> device;
+  std::unique_ptr<plume::RenderCommandQueue> queue;
+  std::unique_ptr<plume::RenderSwapChain> swap_chain;
+  std::unique_ptr<plume::RenderCommandFence> idle_fence;
+  Backend backend = Backend::kD3D12;
+};
+
+std::mutex g_mutex;
+std::unique_ptr<State> g_state;
+
+std::unique_ptr<plume::RenderInterface> CreateInterface(Backend backend) {
+  switch (backend) {
+    case Backend::kD3D12:
+      return plume::CreateD3D12Interface();
+    case Backend::kVulkan:
+      // The frontend deliberately has a Vulkan slot. Its Plume implementation
+      // is compiled back in once the D3D12 command path is stable.
+      return nullptr;
+  }
+  return nullptr;
+}
+
+const char* NameOf(Backend backend) {
+  return backend == Backend::kD3D12 ? "D3D12" : "Vulkan";
+}
+
+}  // namespace
+
+bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
+  std::lock_guard lock(g_mutex);
+  if (g_state) {
+    return true;
+  }
+  if (!window || !window->GetNativeWindowHandle()) {
+    REXLOG_ERROR("Native GPU: no native window handle");
+    return false;
+  }
+
+  auto state = std::make_unique<State>();
+  state->backend = backend;
+  state->render_interface = CreateInterface(backend);
+  if (!state->render_interface) {
+    REXLOG_ERROR("Native GPU: failed to create {} interface", NameOf(backend));
+    return false;
+  }
+
+  state->device = state->render_interface->createDevice();
+  if (!state->device) {
+    REXLOG_ERROR("Native GPU: failed to create {} device", NameOf(backend));
+    return false;
+  }
+  state->queue = state->device->createCommandQueue(plume::RenderCommandListType::DIRECT);
+  state->idle_fence = state->device->createCommandFence();
+  if (!state->queue || !state->idle_fence) {
+    REXLOG_ERROR("Native GPU: failed to create command queue or fence");
+    return false;
+  }
+
+  auto native_window = static_cast<HWND>(window->GetNativeWindowHandle());
+  state->swap_chain = state->queue->createSwapChain(plume::RenderSwapChainDesc(
+      native_window, plume::RenderFormat::B8G8R8A8_UNORM, 3));
+  if (!state->swap_chain || !state->swap_chain->resize() || state->swap_chain->isEmpty()) {
+    REXLOG_ERROR("Native GPU: failed to create or size the swap chain");
+    return false;
+  }
+
+  const auto& description = state->device->getDescription();
+  REXLOG_INFO("Native GPU: {} ready on '{}' ({}x{}, {} images)", NameOf(backend),
+              description.name, state->swap_chain->getWidth(), state->swap_chain->getHeight(),
+              state->swap_chain->getTextureCount());
+  g_state = std::move(state);
+  return true;
+}
+
+void HostDevice::Shutdown() {
+  std::lock_guard lock(g_mutex);
+  if (!g_state) {
+    return;
+  }
+  if (g_state->swap_chain) {
+    g_state->swap_chain->wait();
+  }
+  g_state.reset();
+}
+
+bool HostDevice::IsReady() {
+  std::lock_guard lock(g_mutex);
+  return g_state && g_state->device && g_state->queue && g_state->swap_chain;
+}
+
+plume::RenderDevice* HostDevice::Device() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? g_state->device.get() : nullptr;
+}
+
+plume::RenderCommandQueue* HostDevice::Queue() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? g_state->queue.get() : nullptr;
+}
+
+plume::RenderSwapChain* HostDevice::SwapChain() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? g_state->swap_chain.get() : nullptr;
+}
+
+std::string_view HostDevice::BackendName() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? NameOf(g_state->backend) : "none";
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/device.h b/rexlego/src/gpu_native/device.h
new file mode 100644
index 0000000..b6f6661
--- /dev/null
+++ b/rexlego/src/gpu_native/device.h
@@ -0,0 +1,41 @@
+// Backend-neutral host device owner for the native GPU path.
+#pragma once
+
+#include <memory>
+#include <string_view>
+
+namespace plume {
+struct RenderCommandFence;
+struct RenderCommandQueue;
+struct RenderDevice;
+struct RenderInterface;
+struct RenderSwapChain;
+}  // namespace plume
+
+namespace rex::ui {
+class Window;
+}  // namespace rex::ui
+
+namespace legodimensions::gpu_native {
+
+enum class Backend {
+  kD3D12,
+  kVulkan,
+};
+
+class HostDevice {
+ public:
+  static bool Create(rex::ui::Window* window, Backend backend = Backend::kD3D12);
+  static void Shutdown();
+  static bool IsReady();
+
+  static plume::RenderDevice* Device();
+  static plume::RenderCommandQueue* Queue();
+  static plume::RenderSwapChain* SwapChain();
+  static std::string_view BackendName();
+
+ private:
+  HostDevice() = default;
+};
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/dxc_link.cpp b/rexlego/src/gpu_native/dxc_link.cpp
new file mode 100644
index 0000000..6003e20
--- /dev/null
+++ b/rexlego/src/gpu_native/dxc_link.cpp
@@ -0,0 +1,127 @@
+/**
+ * @file    gpu/shaders/dxc_link.cpp
+ * @brief   Isolated COM TU. No engine headers, only <unknwn.h> and
+ *          <dxcapi.h>.
+ * @copyright Copyright (c) 2026 Tom Clay <tomc@tctechstuff.com>
+ *            All rights reserved.
+ * @license   BSD 3-Clause License
+ *            See LICENSE file in the project root for full license text.
+ */
+#include "gpu_native/dxc_link.h"
+
+#include <unknwn.h>
+
+#include <dxcapi.h>
+
+#include <cstdio>
+
+namespace legodimensions::gpu_native {
+namespace {
+
+template <typename T> class Com {
+public:
+  Com() = default;
+  ~Com() {
+    if (p_)
+      p_->Release();
+  }
+  Com(const Com &) = delete;
+  Com &operator=(const Com &) = delete;
+  T **put() { return &p_; }
+  T *operator->() const { return p_; }
+  T *get() const { return p_; }
+  explicit operator bool() const { return p_ != nullptr; }
+
+private:
+  T *p_ = nullptr;
+};
+
+std::vector<uint8_t> ToVector(IDxcBlob *blob) {
+  const auto *bytes = static_cast<const uint8_t *>(blob->GetBufferPointer());
+  return std::vector<uint8_t>(bytes, bytes + blob->GetBufferSize());
+}
+
+} // namespace
+
+std::vector<uint8_t> CompileSpecConstantLib(uint32_t value) {
+  Com<IDxcCompiler3> compiler;
+  if (FAILED(
+          DxcCreateInstance(CLSID_DxcCompiler, IID_PPV_ARGS(compiler.put())))) {
+    return {};
+  }
+
+  char hlsl[128];
+  const int len =
+      std::snprintf(hlsl, sizeof(hlsl),
+                    "export uint g_SpecConstants() { return %u; }", value);
+  DxcBuffer buffer{};
+  buffer.Ptr = hlsl;
+  buffer.Size = static_cast<SIZE_T>(len);
+  buffer.Encoding = DXC_CP_ACP;
+
+  const wchar_t *args[] = {L"-T", L"lib_6_3"};
+  Com<IDxcResult> result;
+  if (FAILED(compiler->Compile(&buffer, args, 2, nullptr,
+                               IID_PPV_ARGS(result.put()))) ||
+      !result) {
+    return {};
+  }
+
+  Com<IDxcBlob> blob;
+  if (FAILED(result->GetResult(blob.put())) || !blob ||
+      blob->GetBufferSize() == 0) {
+    return {};
+  }
+  return ToVector(blob.get());
+}
+
+std::vector<uint8_t> LinkSpecConstantLib(const uint8_t *libraryDxil,
+                                         uint32_t libraryDxilSize,
+                                         const uint8_t *specLib,
+                                         size_t specLibSize,
+                                         const wchar_t *profile) {
+  if (!libraryDxil || libraryDxilSize == 0 || !specLib || specLibSize == 0)
+    return {};
+
+  Com<IDxcUtils> utils;
+  if (FAILED(DxcCreateInstance(CLSID_DxcUtils, IID_PPV_ARGS(utils.put()))))
+    return {};
+
+  Com<IDxcBlobEncoding> specBlob;
+  Com<IDxcBlobEncoding> shaderBlob;
+  if (FAILED(utils->CreateBlobFromPinned(specLib,
+                                         static_cast<UINT32>(specLibSize),
+                                         DXC_CP_ACP, specBlob.put())) ||
+      FAILED(utils->CreateBlobFromPinned(libraryDxil, libraryDxilSize,
+                                         DXC_CP_ACP, shaderBlob.put()))) {
+    return {};
+  }
+
+  Com<IDxcLinker> linker;
+  if (FAILED(DxcCreateInstance(CLSID_DxcLinker, IID_PPV_ARGS(linker.put()))))
+    return {};
+
+  linker->RegisterLibrary(L"SpecConstants", specBlob.get());
+  linker->RegisterLibrary(L"Shader", shaderBlob.get());
+  const wchar_t *libNames[] = {L"SpecConstants", L"Shader"};
+
+  Com<IDxcOperationResult> result;
+  if (FAILED(linker->Link(L"main", profile, libNames, 2, nullptr, 0,
+                          result.put())) ||
+      !result) {
+    return {};
+  }
+  // Link reports a failed link through the operation status, not the HRESULT.
+  HRESULT status = E_FAIL;
+  if (FAILED(result->GetStatus(&status)) || FAILED(status))
+    return {};
+
+  Com<IDxcBlob> linked;
+  if (FAILED(result->GetResult(linked.put())) || !linked ||
+      linked->GetBufferSize() == 0) {
+    return {};
+  }
+  return ToVector(linked.get());
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/dxc_link.h b/rexlego/src/gpu_native/dxc_link.h
new file mode 100644
index 0000000..b3ed4f2
--- /dev/null
+++ b/rexlego/src/gpu_native/dxc_link.h
@@ -0,0 +1,31 @@
+/**
+ * @file    gpu/shaders/dxc_link.h
+ * @brief   DXC spec constant library compile and link, shared by the runtime
+ *          linker and the build-time reblue_prelink. reblue_prelink compiles
+ *          this without the rex SDK on its include path, hence <cstdint> and
+ *          no logging: failure comes back as an empty vector.
+ * @copyright Copyright (c) 2026 Tom Clay <tomc@tctechstuff.com>
+ *            All rights reserved.
+ * @license   BSD 3-Clause License
+ *            See LICENSE file in the project root for full license text.
+ */
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace legodimensions::gpu_native {
+
+// lib_6_3 DXIL for 'export uint g_SpecConstants() { return value; }'.
+std::vector<uint8_t> CompileSpecConstantLib(uint32_t value);
+
+// Links a spec constant DXIL library against that lib for the given target
+// profile ("vs_6_0" / "ps_6_0"), returning the complete shader bytecode.
+std::vector<uint8_t> LinkSpecConstantLib(const uint8_t *libraryDxil,
+                                         uint32_t libraryDxilSize,
+                                         const uint8_t *specLib,
+                                         size_t specLibSize,
+                                         const wchar_t *profile);
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/hooks_device.cpp b/rexlego/src/gpu_native/hooks_device.cpp
new file mode 100644
index 0000000..f751313
--- /dev/null
+++ b/rexlego/src/gpu_native/hooks_device.cpp
@@ -0,0 +1,58 @@
+#include <cstring>
+
+#include <rex/hook.h>
+#include <rex/system/kernel_state.h>
+#include <rex/types.h>
+
+#include "gpu_native/d3d.h"
+#include "gpu_native/device.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+constexpr u32 kRenderStateTable = 0x847F9B18;
+constexpr u32 kSamplerStateTable = 0x847F9FD8;
+
+void CopyStateDispatch(D3DDevice* device, u8* membase) {
+  const auto* render = reinterpret_cast<const be_u32*>(membase + kRenderStateTable);
+  const auto* sampler = reinterpret_cast<const be_u32*>(membase + kSamplerStateTable);
+  for (u32 i = 0; i < kRenderStateCount; ++i) {
+    device->get_render_state[i] = render[i * 3 + 0];
+    device->set_render_state[i] = render[i * 3 + 1];
+  }
+  for (u32 i = 0; i < kSamplerStateCount; ++i) {
+    device->get_sampler_state[i] = sampler[i * 3 + 0];
+    device->set_sampler_state[i] = sampler[i * 3 + 1];
+  }
+}
+
+u32 Direct3DCreateDeviceHook(u32 /*adapter*/, u32 /*device_type*/, u32 /*focus_window*/,
+                             u32 /*behavior_flags*/, u32 /*present_params*/,
+                             mapped_u32 out_device) {
+  if (!out_device || !HostDevice::IsReady()) {
+    return 0x8007000Eu;
+  }
+  *out_device = 0;
+
+  auto* memory = REX_KERNEL_MEMORY();
+  const u32 guest_address = memory->SystemHeapAlloc(kGuestDeviceSize, 0x80);
+  if (!guest_address) {
+    return 0x8007000Eu;
+  }
+  memory->Zero(guest_address, kGuestDeviceSize);
+
+  auto* device = reinterpret_cast<D3DDevice*>(memory->virtual_membase() + guest_address);
+  CopyStateDispatch(device, memory->virtual_membase());
+  device->viewport.width = 1280;
+  device->viewport.height = 720;
+  device->viewport.max_z = 1.0f;
+  device->scissor.right = 1280;
+  device->scissor.bottom = 720;
+  *out_device = guest_address;
+  return 0;
+}
+
+}  // namespace
+}  // namespace legodimensions::gpu_native
+
+REX_HOOK(sub_83FAF9A0, legodimensions::gpu_native::Direct3DCreateDeviceHook);
diff --git a/rexlego/src/gpu_native/linked_shader_cache.h b/rexlego/src/gpu_native/linked_shader_cache.h
new file mode 100644
index 0000000..a3806d7
--- /dev/null
+++ b/rexlego/src/gpu_native/linked_shader_cache.h
@@ -0,0 +1,31 @@
+/**
+ * @file    gpu_native/linked_shader_cache.h
+ * @brief   Build-time pre-linked spec constant shader variants, emitted by
+ *          lego_gpu_prelink into lego-linked-dxil-cache.cpp. Referenced
+ *          by unqualified name from the generated cpp, so this must stay in
+ *          the global namespace.
+ * @copyright Copyright (c) 2026 Tom Clay <tomc@tctechstuff.com>
+ *            All rights reserved.
+ * @license   BSD 3-Clause License
+ *            See LICENSE file in the project root for full license text.
+ */
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+struct LinkedShaderCacheEntry {
+  const uint64_t hash;          // == ShaderCacheEntry::hash
+  const uint32_t specConstants; // masked g_SpecConstants() value
+  const uint32_t dxilOffset;    // into the decompressed linked cache
+  const uint32_t dxilSize;
+};
+
+// Sorted by (hash, specConstants) for binary search.
+extern LinkedShaderCacheEntry g_linkedShaderCacheEntries[];
+extern const size_t g_linkedShaderCacheEntryCount;
+
+// miniz (deflate) compressed concatenation of the linked DXIL blobs.
+extern const uint8_t g_compressedLinkedDxilCache[];
+extern const size_t g_linkedDxilCacheCompressedSize;
+extern const size_t g_linkedDxilCacheDecompressedSize;
diff --git a/rexlego/src/gpu_native/shader_archive.cpp b/rexlego/src/gpu_native/shader_archive.cpp
new file mode 100644
index 0000000..4026fd7
--- /dev/null
+++ b/rexlego/src/gpu_native/shader_archive.cpp
@@ -0,0 +1,127 @@
+/**
+ * @file        gpu_native/shader_archive.cpp
+ * @brief       LEGO Dimensions AOT DXIL cache lookup
+ *
+ * @copyright   Copyright (c) 2026 NeverCookFirst
+ * @license     BSD 3-Clause License
+ */
+
+#include "gpu_native/shader_archive.h"
+
+#include <algorithm>
+#include <memory>
+#include <mutex>
+#include <utility>
+
+#define MINIZ_HEADER_FILE_ONLY
+#include <miniz.h>
+#include <xxhash.h>
+#include <zstd.h>
+
+#include <rex/logging.h>
+
+#include "gpu_native/linked_shader_cache.h"
+
+namespace legodimensions::gpu_native {
+
+namespace {
+
+uint32_t LoadBigEndianU32(const uint8_t* bytes) {
+  uint32_t value = 0;
+  for (size_t i = 0; i != sizeof(value); ++i) {
+    value = (value << 8) | bytes[i];
+  }
+  return value;
+}
+
+}  // namespace
+
+size_t ShaderContainerByteLength(const void* shader_container) {
+  const auto* bytes = static_cast<const uint8_t*>(shader_container);
+  const uint32_t virtual_size = LoadBigEndianU32(bytes + sizeof(uint32_t));
+  const uint32_t physical_size = LoadBigEndianU32(bytes + 2 * sizeof(uint32_t));
+  return size_t{virtual_size} + sizeof(uint32_t) + size_t{physical_size};
+}
+
+const ShaderCacheEntry* FindShader(uint64_t hash) {
+  const ShaderCacheEntry* const begin = g_shaderCacheEntries;
+  const ShaderCacheEntry* const end = begin + g_shaderCacheEntryCount;
+  const ShaderCacheEntry* const result = std::lower_bound(
+      begin, end, hash,
+      [](const ShaderCacheEntry& entry, uint64_t key) { return entry.hash < key; });
+  return result != end && result->hash == hash ? result : nullptr;
+}
+
+uint64_t HashShaderContainer(const void* shader_container) {
+  if (!shader_container) {
+    return 0;
+  }
+  return XXH3_64bits(shader_container, ShaderContainerByteLength(shader_container));
+}
+
+ShaderBytecode FindDxil(uint64_t hash, uint32_t spec_constants) {
+  const ShaderCacheEntry* const shader = FindShader(hash);
+  if (!shader) {
+    REXLOG_WARN("Native GPU shader cache miss: hash=0x{:016X}", hash);
+    return {};
+  }
+
+  static std::once_flag dxil_once;
+  static std::unique_ptr<uint8_t[]> dxil;
+  std::call_once(dxil_once, [] {
+    auto data = std::make_unique<uint8_t[]>(g_dxilCacheDecompressedSize);
+    const size_t result = ZSTD_decompress(data.get(), g_dxilCacheDecompressedSize,
+                                          g_compressedDxilCache,
+                                          g_dxilCacheCompressedSize);
+    if (ZSTD_isError(result) || result != g_dxilCacheDecompressedSize) {
+      REXLOG_ERROR("Native GPU DXIL archive decompression failed: {} of {} bytes",
+                   result, g_dxilCacheDecompressedSize);
+      return;
+    }
+    dxil = std::move(data);
+  });
+  if (!dxil) {
+    return {};
+  }
+
+  if (shader->specConstantsMask == 0) {
+    return {dxil.get() + shader->dxilOffset, shader->dxilSize};
+  }
+
+  const uint32_t masked = spec_constants & shader->specConstantsMask;
+  const LinkedShaderCacheEntry* const first = g_linkedShaderCacheEntries;
+  const LinkedShaderCacheEntry* const end = first + g_linkedShaderCacheEntryCount;
+  const auto key = std::pair{hash, masked};
+  const LinkedShaderCacheEntry* const linked = std::lower_bound(
+      first, end, key,
+      [](const LinkedShaderCacheEntry& entry, const auto& rhs) {
+        return entry.hash < rhs.first ||
+               (entry.hash == rhs.first && entry.specConstants < rhs.second);
+      });
+  if (linked == end || linked->hash != hash || linked->specConstants != masked) {
+    REXLOG_ERROR("Native GPU linked shader miss: hash=0x{:016X} spec=0x{:X}",
+                 hash, masked);
+    return {};
+  }
+
+  static std::once_flag linked_once;
+  static std::unique_ptr<uint8_t[]> linked_dxil;
+  std::call_once(linked_once, [] {
+    auto data = std::make_unique<uint8_t[]>(g_linkedDxilCacheDecompressedSize);
+    mz_ulong size = static_cast<mz_ulong>(g_linkedDxilCacheDecompressedSize);
+    const int result = mz_uncompress(
+        data.get(), &size, g_compressedLinkedDxilCache,
+        static_cast<mz_ulong>(g_linkedDxilCacheCompressedSize));
+    if (result != MZ_OK || size != g_linkedDxilCacheDecompressedSize) {
+      REXLOG_ERROR("Native GPU linked DXIL archive decompression failed");
+      return;
+    }
+    linked_dxil = std::move(data);
+  });
+  if (!linked_dxil) {
+    return {};
+  }
+  return {linked_dxil.get() + linked->dxilOffset, linked->dxilSize};
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/shader_archive.h b/rexlego/src/gpu_native/shader_archive.h
new file mode 100644
index 0000000..fb38a0b
--- /dev/null
+++ b/rexlego/src/gpu_native/shader_archive.h
@@ -0,0 +1,57 @@
+/**
+ * @file        gpu_native/shader_archive.h
+ * @brief       LEGO Dimensions AOT DXIL cache schema and lookup helpers
+ *
+ * @copyright   Copyright (c) 2026 NeverCookFirst
+ * @license     BSD 3-Clause License
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+// XenosRecomp emits these names in the global namespace.
+struct ShaderCacheEntry {
+  const uint64_t hash;
+  const uint32_t dxilOffset;
+  const uint32_t dxilSize;
+  const uint32_t spirvOffset;
+  const uint32_t spirvSize;
+  const uint32_t specConstantsMask;
+};
+
+extern ShaderCacheEntry g_shaderCacheEntries[];
+extern const size_t g_shaderCacheEntryCount;
+
+extern const uint8_t g_compressedDxilCache[];
+extern const size_t g_dxilCacheCompressedSize;
+extern const size_t g_dxilCacheDecompressedSize;
+
+extern const uint8_t g_compressedSpirvCache[];
+extern const size_t g_spirvCacheCompressedSize;
+extern const size_t g_spirvCacheDecompressedSize;
+
+namespace legodimensions::gpu_native {
+
+struct ShaderBytecode {
+  const uint8_t* data = nullptr;
+  size_t size = 0;
+
+  explicit operator bool() const { return data != nullptr && size != 0; }
+};
+
+// Xbox shader containers place a four-byte separator between the virtual and
+// physical programs. LEGO's cache key covers that separator as well.
+size_t ShaderContainerByteLength(const void* shader_container);
+
+const ShaderCacheEntry* FindShader(uint64_t hash);
+
+// Hashes the exact Xbox container span used by the AOT generator.
+uint64_t HashShaderContainer(const void* shader_container);
+
+// Returns complete DXIL ready for D3D12. Specialization-library entries are
+// resolved exclusively through the build-time linked archive.
+ShaderBytecode FindDxil(uint64_t hash, uint32_t spec_constants);
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/legodimensions_app.h b/rexlego/src/legodimensions_app.h
index c6faa7e..e486533 100644
--- a/rexlego/src/legodimensions_app.h
+++ b/rexlego/src/legodimensions_app.h
@@ -31,6 +31,9 @@
 #include "toypad_app.h"
 #include "ui_theme.h"
 #include "updates.h"
+#ifdef LEGODIMENSIONS_NATIVE_GPU
+#include "gpu_native/device.h"
+#endif
 
 class LegodimensionsApp : public rex::ReXApp {
  public:
@@ -63,6 +66,22 @@ class LegodimensionsApp : public rex::ReXApp {
     legodimensions::toypad_app::StartIfEnabled();
   }
 
+#ifdef LEGODIMENSIONS_NATIVE_GPU
+  // Detached renderer mode: the SDK still owns the window/input/audio, while
+  // the title's D3D calls are handled by gpu_native instead of rexgpu-xenos.
+  void OnPreSetup(rex::RuntimeConfig& config) override {
+    config.graphics.reset();
+    config.gpu_plugin.clear();
+  }
+
+  void OnPreLaunchModule() override {
+    if (!legodimensions::gpu_native::HostDevice::Create(window())) {
+      REXLOG_ERROR("Native GPU initialization failed; stopping before guest launch");
+      app_context().QuitFromUIThread();
+    }
+  }
+#endif
+
   // The window closing is where the app must be let go: ReXApp::OnClosing then
   // hard-exits with std::_Exit, so OnShutdown never runs. This asks nicely; the
   // job object is the backstop if the game dies without getting here.
@@ -79,6 +98,9 @@ class LegodimensionsApp : public rex::ReXApp {
 
   // Dropping the pipe is what clears the presence; Discord does the rest.
   void OnShutdown() override {
+#ifdef LEGODIMENSIONS_NATIVE_GPU
+    legodimensions::gpu_native::HostDevice::Shutdown();
+#endif
     legodimensions::discord::Stop();
     legodimensions::toypad_app::StopIfStarted();
     // Drops the freeze thread before the guest memory it writes into goes away.
diff --git a/thirdparty/miniz/miniz.cpp b/thirdparty/miniz/miniz.cpp
new file mode 100644
index 0000000..aa016f6
--- /dev/null
+++ b/thirdparty/miniz/miniz.cpp
@@ -0,0 +1,4 @@
+#include <cstdlib>
+#include <cstring>
+#include <cstdio>
+#include "miniz.h"
diff --git a/thirdparty/miniz/miniz.h b/thirdparty/miniz/miniz.h
new file mode 100644
index 0000000..f276a4d
--- /dev/null
+++ b/thirdparty/miniz/miniz.h
@@ -0,0 +1,5100 @@
+/* miniz.c v1.15 - public domain deflate/inflate, zlib-subset, ZIP reading/writing/appending, PNG writing
+   See "unlicense" statement at the end of this file.
+   Rich Geldreich <richgel99@gmail.com>, last updated Oct. 13, 2013
+   Implements RFC 1950: http://www.ietf.org/rfc/rfc1950.txt and RFC 1951: http://www.ietf.org/rfc/rfc1951.txt
+
+   Most API's defined in miniz.c are optional. For example, to disable the archive related functions just define
+   MINIZ_NO_ARCHIVE_APIS, or to get rid of all stdio usage define MINIZ_NO_STDIO (see the list below for more macros).
+
+   * Change History
+     10/13/13 v1.15 r4 - Interim bugfix release while I work on the next major release with Zip64 support (almost there!):
+       - Critical fix for the MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY bug (thanks kahmyong.moon@hp.com) which could cause locate files to not find files. This bug
+        would only have occurred in earlier versions if you explicitly used this flag, OR if you used mz_zip_extract_archive_file_to_heap() or mz_zip_add_mem_to_archive_file_in_place()
+        (which used this flag). If you can't switch to v1.15 but want to fix this bug, just remove the uses of this flag from both helper funcs (and of course don't use the flag).
+       - Bugfix in mz_zip_reader_extract_to_mem_no_alloc() from kymoon when pUser_read_buf is not NULL and compressed size is > uncompressed size
+       - Fixing mz_zip_reader_extract_*() funcs so they don't try to extract compressed data from directory entries, to account for weird zipfiles which contain zero-size compressed data on dir entries.
+         Hopefully this fix won't cause any issues on weird zip archives, because it assumes the low 16-bits of zip external attributes are DOS attributes (which I believe they always are in practice).
+       - Fixing mz_zip_reader_is_file_a_directory() so it doesn't check the internal attributes, just the filename and external attributes
+       - mz_zip_reader_init_file() - missing MZ_FCLOSE() call if the seek failed
+       - Added cmake support for Linux builds which builds all the examples, tested with clang v3.3 and gcc v4.6.
+       - Clang fix for tdefl_write_image_to_png_file_in_memory() from toffaletti
+       - Merged MZ_FORCEINLINE fix from hdeanclark
+       - Fix <time.h> include before config #ifdef, thanks emil.brink
+       - Added tdefl_write_image_to_png_file_in_memory_ex(): supports Y flipping (super useful for OpenGL apps), and explicit control over the compression level (so you can
+        set it to 1 for real-time compression).
+       - Merged in some compiler fixes from paulharris's github repro.
+       - Retested this build under Windows (VS 2010, including static analysis), tcc  0.9.26, gcc v4.6 and clang v3.3.
+       - Added example6.c, which dumps an image of the mandelbrot set to a PNG file.
+       - Modified example2 to help test the MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY flag more.
+       - In r3: Bugfix to mz_zip_writer_add_file() found during merge: Fix possible src file fclose() leak if alignment bytes+local header file write faiiled
+       - In r4: Minor bugfix to mz_zip_writer_add_from_zip_reader(): Was pushing the wrong central dir header offset, appears harmless in this release, but it became a problem in the zip64 branch
+     5/20/12 v1.14 - MinGW32/64 GCC 4.6.1 compiler fixes: added MZ_FORCEINLINE, #include <time.h> (thanks fermtect).
+     5/19/12 v1.13 - From jason@cornsyrup.org and kelwert@mtu.edu - Fix mz_crc32() so it doesn't compute the wrong CRC-32's when mz_ulong is 64-bit.
+       - Temporarily/locally slammed in "typedef unsigned long mz_ulong" and re-ran a randomized regression test on ~500k files.
+       - Eliminated a bunch of warnings when compiling with GCC 32-bit/64.
+       - Ran all examples, miniz.c, and tinfl.c through MSVC 2008's /analyze (static analysis) option and fixed all warnings (except for the silly
+        "Use of the comma-operator in a tested expression.." analysis warning, which I purposely use to work around a MSVC compiler warning).
+       - Created 32-bit and 64-bit Codeblocks projects/workspace. Built and tested Linux executables. The codeblocks workspace is compatible with Linux+Win32/x64.
+       - Added miniz_tester solution/project, which is a useful little app derived from LZHAM's tester app that I use as part of the regression test.
+       - Ran miniz.c and tinfl.c through another series of regression testing on ~500,000 files and archives.
+       - Modified example5.c so it purposely disables a bunch of high-level functionality (MINIZ_NO_STDIO, etc.). (Thanks to corysama for the MINIZ_NO_STDIO bug report.)
+       - Fix ftell() usage in examples so they exit with an error on files which are too large (a limitation of the examples, not miniz itself).
+     4/12/12 v1.12 - More comments, added low-level example5.c, fixed a couple minor level_and_flags issues in the archive API's.
+      level_and_flags can now be set to MZ_DEFAULT_COMPRESSION. Thanks to Bruce Dawson <bruced@valvesoftware.com> for the feedback/bug report.
+     5/28/11 v1.11 - Added statement from unlicense.org
+     5/27/11 v1.10 - Substantial compressor optimizations:
+      - Level 1 is now ~4x faster than before. The L1 compressor's throughput now varies between 70-110MB/sec. on a
+      - Core i7 (actual throughput varies depending on the type of data, and x64 vs. x86).
+      - Improved baseline L2-L9 compression perf. Also, greatly improved compression perf. issues on some file types.
+      - Refactored the compression code for better readability and maintainability.
+      - Added level 10 compression level (L10 has slightly better ratio than level 9, but could have a potentially large
+       drop in throughput on some files).
+     5/15/11 v1.09 - Initial stable release.
+
+   * Low-level Deflate/Inflate implementation notes:
+
+     Compression: Use the "tdefl" API's. The compressor supports raw, static, and dynamic blocks, lazy or
+     greedy parsing, match length filtering, RLE-only, and Huffman-only streams. It performs and compresses
+     approximately as well as zlib.
+
+     Decompression: Use the "tinfl" API's. The entire decompressor is implemented as a single function
+     coroutine: see tinfl_decompress(). It supports decompression into a 32KB (or larger power of 2) wrapping buffer, or into a memory
+     block large enough to hold the entire file.
+
+     The low-level tdefl/tinfl API's do not make any use of dynamic memory allocation.
+
+   * zlib-style API notes:
+
+     miniz.c implements a fairly large subset of zlib. There's enough functionality present for it to be a drop-in
+     zlib replacement in many apps:
+        The z_stream struct, optional memory allocation callbacks
+        deflateInit/deflateInit2/deflate/deflateReset/deflateEnd/deflateBound
+        inflateInit/inflateInit2/inflate/inflateEnd
+        compress, compress2, compressBound, uncompress
+        CRC-32, Adler-32 - Using modern, minimal code size, CPU cache friendly routines.
+        Supports raw deflate streams or standard zlib streams with adler-32 checking.
+
+     Limitations:
+      The callback API's are not implemented yet. No support for gzip headers or zlib static dictionaries.
+      I've tried to closely emulate zlib's various flavors of stream flushing and return status codes, but
+      there are no guarantees that miniz.c pulls this off perfectly.
+
+   * PNG writing: See the tdefl_write_image_to_png_file_in_memory() function, originally written by
+     Alex Evans. Supports 1-4 bytes/pixel images.
+
+   * ZIP archive API notes:
+
+     The ZIP archive API's where designed with simplicity and efficiency in mind, with just enough abstraction to
+     get the job done with minimal fuss. There are simple API's to retrieve file information, read files from
+     existing archives, create new archives, append new files to existing archives, or clone archive data from
+     one archive to another. It supports archives located in memory or the heap, on disk (using stdio.h),
+     or you can specify custom file read/write callbacks.
+
+     - Archive reading: Just call this function to read a single file from a disk archive:
+
+      void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name,
+        size_t *pSize, mz_uint zip_flags);
+
+     For more complex cases, use the "mz_zip_reader" functions. Upon opening an archive, the entire central
+     directory is located and read as-is into memory, and subsequent file access only occurs when reading individual files.
+
+     - Archives file scanning: The simple way is to use this function to scan a loaded archive for a specific file:
+
+     int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags);
+
+     The locate operation can optionally check file comments too, which (as one example) can be used to identify
+     multiple versions of the same file in an archive. This function uses a simple linear search through the central
+     directory, so it's not very fast.
+
+     Alternately, you can iterate through all the files in an archive (using mz_zip_reader_get_num_files()) and
+     retrieve detailed info on each file by calling mz_zip_reader_file_stat().
+
+     - Archive creation: Use the "mz_zip_writer" functions. The ZIP writer immediately writes compressed file data
+     to disk and builds an exact image of the central directory in memory. The central directory image is written
+     all at once at the end of the archive file when the archive is finalized.
+
+     The archive writer can optionally align each file's local header and file data to any power of 2 alignment,
+     which can be useful when the archive will be read from optical media. Also, the writer supports placing
+     arbitrary data blobs at the very beginning of ZIP archives. Archives written using either feature are still
+     readable by any ZIP tool.
+
+     - Archive appending: The simple way to add a single file to an archive is to call this function:
+
+      mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name,
+        const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags);
+
+     The archive will be created if it doesn't already exist, otherwise it'll be appended to.
+     Note the appending is done in-place and is not an atomic operation, so if something goes wrong
+     during the operation it's possible the archive could be left without a central directory (although the local
+     file headers and file data will be fine, so the archive will be recoverable).
+
+     For more complex archive modification scenarios:
+     1. The safest way is to use a mz_zip_reader to read the existing archive, cloning only those bits you want to
+     preserve into a new archive using using the mz_zip_writer_add_from_zip_reader() function (which compiles the
+     compressed file data as-is). When you're done, delete the old archive and rename the newly written archive, and
+     you're done. This is safe but requires a bunch of temporary disk space or heap memory.
+
+     2. Or, you can convert an mz_zip_reader in-place to an mz_zip_writer using mz_zip_writer_init_from_reader(),
+     append new files as needed, then finalize the archive which will write an updated central directory to the
+     original archive. (This is basically what mz_zip_add_mem_to_archive_file_in_place() does.) There's a
+     possibility that the archive's central directory could be lost with this method if anything goes wrong, though.
+
+     - ZIP archive support limitations:
+     No zip64 or spanning support. Extraction functions can only handle unencrypted, stored or deflated files.
+     Requires streams capable of seeking.
+
+   * This is a header file library, like stb_image.c. To get only a header file, either cut and paste the
+     below header, or create miniz.h, #define MINIZ_HEADER_FILE_ONLY, and then include miniz.c from it.
+
+   * Important: For best perf. be sure to customize the below macros for your target platform:
+     #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1
+     #define MINIZ_LITTLE_ENDIAN 1
+     #define MINIZ_HAS_64BIT_REGISTERS 1
+
+   * On platforms using glibc, Be sure to "#define _LARGEFILE64_SOURCE 1" before including miniz.c to ensure miniz
+     uses the 64-bit variants: fopen64(), stat64(), etc. Otherwise you won't be able to process large files
+     (i.e. 32-bit stat() fails for me on files > 0x7FFFFFFF bytes).
+*/
+
+#ifndef MINIZ_HEADER_INCLUDED
+#define MINIZ_HEADER_INCLUDED
+
+/*#include <stdlib.h>*/
+
+// Defines to completely disable specific portions of miniz.c:
+// If all macros here are defined the only functionality remaining will be CRC-32, adler-32, tinfl, and tdefl.
+
+// Define MINIZ_NO_STDIO to disable all usage and any functions which rely on stdio for file I/O.
+//#define MINIZ_NO_STDIO
+
+// If MINIZ_NO_TIME is specified then the ZIP archive functions will not be able to get the current time, or
+// get/set file times, and the C run-time funcs that get/set times won't be called.
+// The current downside is the times written to your archives will be from 1979.
+//#define MINIZ_NO_TIME
+
+// Define MINIZ_NO_DEFLATE_APIS to disable all compression API's.
+//#define MINIZ_NO_DEFLATE_APIS
+
+// Define MINIZ_NO_INFLATE_APIS to disable all decompression API's.
+//#define MINIZ_NO_INFLATE_APIS
+
+// Define MINIZ_NO_ARCHIVE_APIS to disable all ZIP archive API's.
+//#define MINIZ_NO_ARCHIVE_APIS
+
+// Define MINIZ_NO_ARCHIVE_APIS to disable all writing related ZIP archive API's.
+// reblue: writer enabled for the save-migration backup (save_store.cpp).
+//#define MINIZ_NO_ARCHIVE_WRITING_APIS
+
+// Define MINIZ_NO_ZLIB_APIS to remove all ZLIB-style compression/decompression API's.
+//#define MINIZ_NO_ZLIB_APIS
+
+// Define MINIZ_NO_ZLIB_COMPATIBLE_NAME to disable zlib names, to prevent conflicts against stock zlib.
+#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES
+
+// Define MINIZ_NO_MALLOC to disable all calls to malloc, free, and realloc.
+// Note if MINIZ_NO_MALLOC is defined then the user must always provide custom user alloc/free/realloc
+// callbacks to the zlib and archive API's, and a few stand-alone helper API's which don't provide custom user
+// functions (such as tdefl_compress_mem_to_heap() and tinfl_decompress_mem_to_heap()) won't work.
+//#define MINIZ_NO_MALLOC
+//#define MINIZ_SDL_MALLOC
+
+#ifdef MINIZ_NO_INFLATE_APIS
+#define MINIZ_NO_ARCHIVE_APIS
+#endif
+
+#ifdef MINIZ_NO_DEFLATE_APIS
+#define MINIZ_NO_ARCHIVE_WRITING_APIS
+#endif
+
+// Define MINIZ_STATIC_FUNCTIONS to make all functions static. You probably
+// want this if you're using miniz in a library
+//#define MINIZ_STATIC_FUNCTIONS
+
+#if defined(__TINYC__) && (defined(__linux) || defined(__linux__))
+  // TODO: Work around "error: include file 'sys\utime.h' when compiling with tcc on Linux
+  #define MINIZ_NO_TIME
+#endif
+
+#if !defined(MINIZ_NO_TIME) && !defined(MINIZ_NO_ARCHIVE_APIS)
+  #include <time.h>
+#endif
+
+#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__i386) || defined(__i486__) || defined(__i486) || defined(i386) || defined(__ia64__) || defined(__x86_64__)
+// MINIZ_X86_OR_X64_CPU is only used to help set the below macros.
+#define MINIZ_X86_OR_X64_CPU 1
+#endif
+
+#ifndef MINIZ_LITTLE_ENDIAN /* if not defined by SDL */
+#if (__BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) || MINIZ_X86_OR_X64_CPU
+// Set MINIZ_LITTLE_ENDIAN to 1 if the processor is little endian.
+#define MINIZ_LITTLE_ENDIAN 1
+#endif
+#endif // MINIZ_LITTLE_ENDIAN /* if not defined by SDL */
+
+#ifndef MINIZ_USE_UNALIGNED_LOADS_AND_STORES
+#if MINIZ_X86_OR_X64_CPU
+// Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES to 1 on CPU's that permit efficient integer loads and stores from unaligned addresses.
+#define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1
+#endif
+#endif // MINIZ_USE_UNALIGNED_LOADS_AND_STORES
+
+#if defined(_M_X64) || defined(_WIN64) || defined(__MINGW64__) || defined(_LP64) || defined(__LP64__) || defined(__ia64__) || defined(__x86_64__)
+// Set MINIZ_HAS_64BIT_REGISTERS to 1 if operations on 64-bit integers are reasonably fast (and don't involve compiler generated calls to helper functions).
+#define MINIZ_HAS_64BIT_REGISTERS 1
+#endif
+
+#ifdef MINIZ_STATIC_FUNCTIONS
+#define MINIZ_STATIC static
+#else
+#define MINIZ_STATIC
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// ------------------- zlib-style API Definitions.
+
+// For more compatibility with zlib, miniz.c uses unsigned long for some parameters/struct members. Beware: mz_ulong can be either 32 or 64-bits!
+typedef unsigned long mz_ulong;
+
+// mz_free() internally uses the MZ_FREE() macro (which by default calls free() unless you've modified the MZ_MALLOC macro) to release a block allocated from the heap.
+MINIZ_STATIC void mz_free(void *p);
+
+#define MZ_ADLER32_INIT (1)
+// mz_adler32() returns the initial adler-32 value to use when called with ptr==NULL.
+MINIZ_STATIC mz_ulong mz_adler32(mz_ulong adler, const unsigned char *ptr, size_t buf_len);
+
+#define MZ_CRC32_INIT (0)
+// mz_crc32() returns the initial CRC-32 value to use when called with ptr==NULL.
+MINIZ_STATIC mz_ulong mz_crc32(mz_ulong crc, const unsigned char *ptr, size_t buf_len);
+
+// Compression strategies.
+enum { MZ_DEFAULT_STRATEGY = 0, MZ_FILTERED = 1, MZ_HUFFMAN_ONLY = 2, MZ_RLE = 3, MZ_FIXED = 4 };
+
+// Method
+#define MZ_DEFLATED 8
+
+#ifndef MINIZ_NO_ZLIB_APIS
+
+// Heap allocation callbacks.
+// Note that mz_alloc_func parameter types purpsosely differ from zlib's: items/size is size_t, not unsigned long.
+typedef void *(*mz_alloc_func)(void *opaque, size_t items, size_t size);
+typedef void (*mz_free_func)(void *opaque, void *address);
+typedef void *(*mz_realloc_func)(void *opaque, void *address, size_t items, size_t size);
+
+#define MZ_VERSION          "9.1.15"
+#define MZ_VERNUM           0x91F0
+#define MZ_VER_MAJOR        9
+#define MZ_VER_MINOR        1
+#define MZ_VER_REVISION     15
+#define MZ_VER_SUBREVISION  0
+
+// Flush values. For typical usage you only need MZ_NO_FLUSH and MZ_FINISH. The other values are for advanced use (refer to the zlib docs).
+enum { MZ_NO_FLUSH = 0, MZ_PARTIAL_FLUSH = 1, MZ_SYNC_FLUSH = 2, MZ_FULL_FLUSH = 3, MZ_FINISH = 4, MZ_BLOCK = 5 };
+
+// Return status codes. MZ_PARAM_ERROR is non-standard.
+enum { MZ_OK = 0, MZ_STREAM_END = 1, MZ_NEED_DICT = 2, MZ_ERRNO = -1, MZ_STREAM_ERROR = -2, MZ_DATA_ERROR = -3, MZ_MEM_ERROR = -4, MZ_BUF_ERROR = -5, MZ_VERSION_ERROR = -6, MZ_PARAM_ERROR = -10000 };
+
+// Compression levels: 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow), MZ_DEFAULT_COMPRESSION=MZ_DEFAULT_LEVEL.
+enum { MZ_NO_COMPRESSION = 0, MZ_BEST_SPEED = 1, MZ_BEST_COMPRESSION = 9, MZ_UBER_COMPRESSION = 10, MZ_DEFAULT_LEVEL = 6, MZ_DEFAULT_COMPRESSION = -1 };
+
+// Window bits
+#define MZ_DEFAULT_WINDOW_BITS 15
+
+struct mz_internal_state;
+
+// Compression/decompression stream struct.
+typedef struct mz_stream_s
+{
+  const unsigned char *next_in;     // pointer to next byte to read
+  unsigned int avail_in;            // number of bytes available at next_in
+  mz_ulong total_in;                // total number of bytes consumed so far
+
+  unsigned char *next_out;          // pointer to next byte to write
+  unsigned int avail_out;           // number of bytes that can be written to next_out
+  mz_ulong total_out;               // total number of bytes produced so far
+
+  char *msg;                        // error msg (unused)
+  struct mz_internal_state *state;  // internal state, allocated by zalloc/zfree
+
+  mz_alloc_func zalloc;             // optional heap allocation function (defaults to malloc)
+  mz_free_func zfree;               // optional heap free function (defaults to free)
+  void *opaque;                     // heap alloc function user pointer
+
+  int data_type;                    // data_type (unused)
+  mz_ulong adler;                   // adler32 of the source or uncompressed data
+  mz_ulong reserved;                // not used
+} mz_stream;
+
+typedef mz_stream *mz_streamp;
+
+// Returns the version string of miniz.c.
+MINIZ_STATIC const char *mz_version(void);
+
+#ifndef MINIZ_NO_DEFLATE_APIS
+
+// mz_deflateInit() initializes a compressor with default options:
+// Parameters:
+//  pStream must point to an initialized mz_stream struct.
+//  level must be between [MZ_NO_COMPRESSION, MZ_BEST_COMPRESSION].
+//  level 1 enables a specially optimized compression function that's been optimized purely for performance, not ratio.
+//  (This special func. is currently only enabled when MINIZ_USE_UNALIGNED_LOADS_AND_STORES and MINIZ_LITTLE_ENDIAN are defined.)
+// Return values:
+//  MZ_OK on success.
+//  MZ_STREAM_ERROR if the stream is bogus.
+//  MZ_PARAM_ERROR if the input parameters are bogus.
+//  MZ_MEM_ERROR on out of memory.
+MINIZ_STATIC int mz_deflateInit(mz_streamp pStream, int level);
+
+// mz_deflateInit2() is like mz_deflate(), except with more control:
+// Additional parameters:
+//   method must be MZ_DEFLATED
+//   window_bits must be MZ_DEFAULT_WINDOW_BITS (to wrap the deflate stream with zlib header/adler-32 footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate/no header or footer)
+//   mem_level must be between [1, 9] (it's checked but ignored by miniz.c)
+MINIZ_STATIC int mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy);
+
+// Quickly resets a compressor without having to reallocate anything. Same as calling mz_deflateEnd() followed by mz_deflateInit()/mz_deflateInit2().
+MINIZ_STATIC int mz_deflateReset(mz_streamp pStream);
+
+// mz_deflate() compresses the input to output, consuming as much of the input and producing as much output as possible.
+// Parameters:
+//   pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members.
+//   flush may be MZ_NO_FLUSH, MZ_PARTIAL_FLUSH/MZ_SYNC_FLUSH, MZ_FULL_FLUSH, or MZ_FINISH.
+// Return values:
+//   MZ_OK on success (when flushing, or if more input is needed but not available, and/or there's more output to be written but the output buffer is full).
+//   MZ_STREAM_END if all input has been consumed and all output bytes have been written. Don't call mz_deflate() on the stream anymore.
+//   MZ_STREAM_ERROR if the stream is bogus.
+//   MZ_PARAM_ERROR if one of the parameters is invalid.
+//   MZ_BUF_ERROR if no forward progress is possible because the input and/or output buffers are empty. (Fill up the input buffer or free up some output space and try again.)
+MINIZ_STATIC int mz_deflate(mz_streamp pStream, int flush);
+
+// mz_deflateEnd() deinitializes a compressor:
+// Return values:
+//  MZ_OK on success.
+//  MZ_STREAM_ERROR if the stream is bogus.
+MINIZ_STATIC int mz_deflateEnd(mz_streamp pStream);
+
+// mz_deflateBound() returns a (very) conservative upper bound on the amount of data that could be generated by deflate(), assuming flush is set to only MZ_NO_FLUSH or MZ_FINISH.
+MINIZ_STATIC mz_ulong mz_deflateBound(mz_streamp pStream, mz_ulong source_len);
+
+// Single-call compression functions mz_compress() and mz_compress2():
+// Returns MZ_OK on success, or one of the error codes from mz_deflate() on failure.
+MINIZ_STATIC int mz_compress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len);
+MINIZ_STATIC int mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len, int level);
+
+// mz_compressBound() returns a (very) conservative upper bound on the amount of data that could be generated by calling mz_compress().
+MINIZ_STATIC mz_ulong mz_compressBound(mz_ulong source_len);
+
+#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/
+
+#ifndef MINIZ_NO_INFLATE_APIS
+
+// Initializes a decompressor.
+MINIZ_STATIC int mz_inflateInit(mz_streamp pStream);
+
+// mz_inflateInit2() is like mz_inflateInit() with an additional option that controls the window size and whether or not the stream has been wrapped with a zlib header/footer:
+// window_bits must be MZ_DEFAULT_WINDOW_BITS (to parse zlib header/footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate).
+MINIZ_STATIC int mz_inflateInit2(mz_streamp pStream, int window_bits);
+
+// Decompresses the input stream to the output, consuming only as much of the input as needed, and writing as much to the output as possible.
+// Parameters:
+//   pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members.
+//   flush may be MZ_NO_FLUSH, MZ_SYNC_FLUSH, or MZ_FINISH.
+//   On the first call, if flush is MZ_FINISH it's assumed the input and output buffers are both sized large enough to decompress the entire stream in a single call (this is slightly faster).
+//   MZ_FINISH implies that there are no more source bytes available beside what's already in the input buffer, and that the output buffer is large enough to hold the rest of the decompressed data.
+// Return values:
+//   MZ_OK on success. Either more input is needed but not available, and/or there's more output to be written but the output buffer is full.
+//   MZ_STREAM_END if all needed input has been consumed and all output bytes have been written. For zlib streams, the adler-32 of the decompressed data has also been verified.
+//   MZ_STREAM_ERROR if the stream is bogus.
+//   MZ_DATA_ERROR if the deflate stream is invalid.
+//   MZ_PARAM_ERROR if one of the parameters is invalid.
+//   MZ_BUF_ERROR if no forward progress is possible because the input buffer is empty but the inflater needs more input to continue, or if the output buffer is not large enough. Call mz_inflate() again
+//   with more input data, or with more room in the output buffer (except when using single call decompression, described above).
+MINIZ_STATIC int mz_inflate(mz_streamp pStream, int flush);
+
+// Deinitializes a decompressor.
+MINIZ_STATIC int mz_inflateEnd(mz_streamp pStream);
+
+// Single-call decompression.
+// Returns MZ_OK on success, or one of the error codes from mz_inflate() on failure.
+MINIZ_STATIC int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len);
+#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/
+
+// Returns a string description of the specified error code, or NULL if the error code is invalid.
+MINIZ_STATIC const char *mz_error(int err);
+
+// Redefine zlib-compatible names to miniz equivalents, so miniz.c can be used as a drop-in replacement for the subset of zlib that miniz.c supports.
+// Define MINIZ_NO_ZLIB_COMPATIBLE_NAMES to disable zlib-compatibility if you use zlib in the same project.
+#ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES
+  typedef unsigned char Byte;
+  typedef unsigned int uInt;
+  typedef mz_ulong uLong;
+  typedef Byte Bytef;
+  typedef uInt uIntf;
+  typedef char charf;
+  typedef int intf;
+  typedef void *voidpf;
+  typedef uLong uLongf;
+  typedef void *voidp;
+  typedef void *const voidpc;
+  #define Z_NULL                0
+  #define Z_NO_FLUSH            MZ_NO_FLUSH
+  #define Z_PARTIAL_FLUSH       MZ_PARTIAL_FLUSH
+  #define Z_SYNC_FLUSH          MZ_SYNC_FLUSH
+  #define Z_FULL_FLUSH          MZ_FULL_FLUSH
+  #define Z_FINISH              MZ_FINISH
+  #define Z_BLOCK               MZ_BLOCK
+  #define Z_OK                  MZ_OK
+  #define Z_STREAM_END          MZ_STREAM_END
+  #define Z_NEED_DICT           MZ_NEED_DICT
+  #define Z_ERRNO               MZ_ERRNO
+  #define Z_STREAM_ERROR        MZ_STREAM_ERROR
+  #define Z_DATA_ERROR          MZ_DATA_ERROR
+  #define Z_MEM_ERROR           MZ_MEM_ERROR
+  #define Z_BUF_ERROR           MZ_BUF_ERROR
+  #define Z_VERSION_ERROR       MZ_VERSION_ERROR
+  #define Z_PARAM_ERROR         MZ_PARAM_ERROR
+  #define Z_NO_COMPRESSION      MZ_NO_COMPRESSION
+  #define Z_BEST_SPEED          MZ_BEST_SPEED
+  #define Z_BEST_COMPRESSION    MZ_BEST_COMPRESSION
+  #define Z_DEFAULT_COMPRESSION MZ_DEFAULT_COMPRESSION
+  #define Z_DEFAULT_STRATEGY    MZ_DEFAULT_STRATEGY
+  #define Z_FILTERED            MZ_FILTERED
+  #define Z_HUFFMAN_ONLY        MZ_HUFFMAN_ONLY
+  #define Z_RLE                 MZ_RLE
+  #define Z_FIXED               MZ_FIXED
+  #define Z_DEFLATED            MZ_DEFLATED
+  #define Z_DEFAULT_WINDOW_BITS MZ_DEFAULT_WINDOW_BITS
+  #define alloc_func            mz_alloc_func
+  #define free_func             mz_free_func
+  #define internal_state        mz_internal_state
+  #define z_stream              mz_stream
+#ifndef MINIZ_NO_DEFLATE_APIS
+  #define deflateInit           mz_deflateInit
+  #define deflateInit2          mz_deflateInit2
+  #define deflateReset          mz_deflateReset
+  #define deflate               mz_deflate
+  #define deflateEnd            mz_deflateEnd
+  #define deflateBound          mz_deflateBound
+  #define compress              mz_compress
+  #define compress2             mz_compress2
+  #define compressBound         mz_compressBound
+#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/
+
+#ifndef MINIZ_NO_INFLATE_APIS
+  #define inflateInit           mz_inflateInit
+  #define inflateInit2          mz_inflateInit2
+  #define inflate               mz_inflate
+  #define inflateEnd            mz_inflateEnd
+  #define uncompress            mz_uncompress
+#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/
+  #define crc32                 mz_crc32
+  #define adler32               mz_adler32
+  #define MAX_WBITS             15
+  #define MAX_MEM_LEVEL         9
+  #define zError                mz_error
+  #define ZLIB_VERSION          MZ_VERSION
+  #define ZLIB_VERNUM           MZ_VERNUM
+  #define ZLIB_VER_MAJOR        MZ_VER_MAJOR
+  #define ZLIB_VER_MINOR        MZ_VER_MINOR
+  #define ZLIB_VER_REVISION     MZ_VER_REVISION
+  #define ZLIB_VER_SUBREVISION  MZ_VER_SUBREVISION
+  #define zlibVersion           mz_version
+  #define zlib_version          mz_version()
+#endif // #ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES
+
+#endif // MINIZ_NO_ZLIB_APIS
+
+// ------------------- Types and macros
+
+typedef unsigned char mz_uint8;
+typedef signed short mz_int16;
+typedef unsigned short mz_uint16;
+typedef unsigned int mz_uint32;
+typedef unsigned int mz_uint;
+typedef long long mz_int64;
+typedef unsigned long long mz_uint64;
+typedef int mz_bool;
+
+#define MZ_FALSE (0)
+#define MZ_TRUE (1)
+
+// An attempt to work around MSVC's spammy "warning C4127: conditional expression is constant" message.
+#ifdef _MSC_VER
+   #define MZ_MACRO_END while (0, 0)
+#else
+   #define MZ_MACRO_END while (0)
+#endif
+
+// ------------------- ZIP archive reading/writing
+
+#ifndef MINIZ_NO_ARCHIVE_APIS
+
+enum
+{
+  MZ_ZIP_MAX_IO_BUF_SIZE = 64*1024,
+  MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE = 260,
+  MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE = 256
+};
+
+typedef struct
+{
+  mz_uint32 m_file_index;
+  mz_uint32 m_central_dir_ofs;
+  mz_uint16 m_version_made_by;
+  mz_uint16 m_version_needed;
+  mz_uint16 m_bit_flag;
+  mz_uint16 m_method;
+#ifndef MINIZ_NO_TIME
+  time_t m_time;
+#endif
+  mz_uint32 m_crc32;
+  mz_uint64 m_comp_size;
+  mz_uint64 m_uncomp_size;
+  mz_uint16 m_internal_attr;
+  mz_uint32 m_external_attr;
+  mz_uint64 m_local_header_ofs;
+  mz_uint32 m_comment_size;
+  char m_filename[MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE];
+  char m_comment[MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE];
+} mz_zip_archive_file_stat;
+
+typedef size_t (*mz_file_read_func)(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n);
+typedef size_t (*mz_file_write_func)(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n);
+
+struct mz_zip_internal_state_tag;
+typedef struct mz_zip_internal_state_tag mz_zip_internal_state;
+
+typedef enum
+{
+  MZ_ZIP_MODE_INVALID = 0,
+  MZ_ZIP_MODE_READING = 1,
+  MZ_ZIP_MODE_WRITING = 2,
+  MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED = 3
+} mz_zip_mode;
+
+typedef struct mz_zip_archive_tag
+{
+  mz_uint64 m_archive_size;
+  mz_uint64 m_central_directory_file_ofs;
+  mz_uint m_total_files;
+  mz_zip_mode m_zip_mode;
+
+  mz_uint m_file_offset_alignment;
+
+  mz_alloc_func m_pAlloc;
+  mz_free_func m_pFree;
+  mz_realloc_func m_pRealloc;
+  void *m_pAlloc_opaque;
+
+  mz_file_read_func m_pRead;
+  mz_file_write_func m_pWrite;
+  void *m_pIO_opaque;
+
+  mz_zip_internal_state *m_pState;
+
+} mz_zip_archive;
+
+typedef enum
+{
+  MZ_ZIP_FLAG_CASE_SENSITIVE                = 0x0100,
+  MZ_ZIP_FLAG_IGNORE_PATH                   = 0x0200,
+  MZ_ZIP_FLAG_COMPRESSED_DATA               = 0x0400,
+  MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY = 0x0800
+} mz_zip_flags;
+
+// ZIP archive reading
+
+// Inits a ZIP archive reader.
+// These functions read and validate the archive's central directory.
+MINIZ_STATIC mz_bool mz_zip_reader_init(mz_zip_archive *pZip, mz_uint64 size, mz_uint32 flags);
+MINIZ_STATIC mz_bool mz_zip_reader_init_mem(mz_zip_archive *pZip, const void *pMem, size_t size, mz_uint32 flags);
+
+#ifndef MINIZ_NO_STDIO
+MINIZ_STATIC mz_bool mz_zip_reader_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint32 flags);
+#endif
+
+// Returns the total number of files in the archive.
+MINIZ_STATIC mz_uint mz_zip_reader_get_num_files(mz_zip_archive *pZip);
+
+// Returns detailed information about an archive file entry.
+MINIZ_STATIC mz_bool mz_zip_reader_file_stat(mz_zip_archive *pZip, mz_uint file_index, mz_zip_archive_file_stat *pStat);
+
+// Determines if an archive file entry is a directory entry.
+MINIZ_STATIC mz_bool mz_zip_reader_is_file_a_directory(mz_zip_archive *pZip, mz_uint file_index);
+MINIZ_STATIC mz_bool mz_zip_reader_is_file_encrypted(mz_zip_archive *pZip, mz_uint file_index);
+
+// Retrieves the filename of an archive file entry.
+// Returns the number of bytes written to pFilename, or if filename_buf_size is 0 this function returns the number of bytes needed to fully store the filename.
+MINIZ_STATIC mz_uint mz_zip_reader_get_filename(mz_zip_archive *pZip, mz_uint file_index, char *pFilename, mz_uint filename_buf_size);
+
+// Attempts to locates a file in the archive's central directory.
+// Valid flags: MZ_ZIP_FLAG_CASE_SENSITIVE, MZ_ZIP_FLAG_IGNORE_PATH
+// Returns -1 if the file cannot be found.
+MINIZ_STATIC int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags);
+
+// Extracts a archive file to a memory buffer using no memory allocation.
+MINIZ_STATIC mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size);
+MINIZ_STATIC mz_bool mz_zip_reader_extract_file_to_mem_no_alloc(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size);
+
+// Extracts a archive file to a memory buffer.
+MINIZ_STATIC mz_bool mz_zip_reader_extract_to_mem(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags);
+MINIZ_STATIC mz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags);
+
+// Extracts a archive file to a dynamically allocated heap buffer.
+MINIZ_STATIC void *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, size_t *pSize, mz_uint flags);
+MINIZ_STATIC void *mz_zip_reader_extract_file_to_heap(mz_zip_archive *pZip, const char *pFilename, size_t *pSize, mz_uint flags);
+
+// Extracts a archive file using a callback function to output the file's data.
+MINIZ_STATIC mz_bool mz_zip_reader_extract_to_callback(mz_zip_archive *pZip, mz_uint file_index, mz_file_write_func pCallback, void *pOpaque, mz_uint flags);
+MINIZ_STATIC mz_bool mz_zip_reader_extract_file_to_callback(mz_zip_archive *pZip, const char *pFilename, mz_file_write_func pCallback, void *pOpaque, mz_uint flags);
+
+#ifndef MINIZ_NO_STDIO
+// Extracts a archive file to a disk file and sets its last accessed and modified times.
+// This function only extracts files, not archive directory records.
+MINIZ_STATIC mz_bool mz_zip_reader_extract_to_file(mz_zip_archive *pZip, mz_uint file_index, const char *pDst_filename, mz_uint flags);
+MINIZ_STATIC mz_bool mz_zip_reader_extract_file_to_file(mz_zip_archive *pZip, const char *pArchive_filename, const char *pDst_filename, mz_uint flags);
+#endif
+
+// Ends archive reading, freeing all allocations, and closing the input archive file if mz_zip_reader_init_file() was used.
+MINIZ_STATIC mz_bool mz_zip_reader_end(mz_zip_archive *pZip);
+
+// ZIP archive writing
+
+#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS
+
+// Inits a ZIP archive writer.
+MINIZ_STATIC mz_bool mz_zip_writer_init(mz_zip_archive *pZip, mz_uint64 existing_size);
+MINIZ_STATIC mz_bool mz_zip_writer_init_heap(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size);
+
+#ifndef MINIZ_NO_STDIO
+MINIZ_STATIC mz_bool mz_zip_writer_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning);
+#endif
+
+// Converts a ZIP archive reader object into a writer object, to allow efficient in-place file appends to occur on an existing archive.
+// For archives opened using mz_zip_reader_init_file, pFilename must be the archive's filename so it can be reopened for writing. If the file can't be reopened, mz_zip_reader_end() will be called.
+// For archives opened using mz_zip_reader_init_mem, the memory block must be growable using the realloc callback (which defaults to realloc unless you've overridden it).
+// Finally, for archives opened using mz_zip_reader_init, the mz_zip_archive's user provided m_pWrite function cannot be NULL.
+// Note: In-place archive modification is not recommended unless you know what you're doing, because if execution stops or something goes wrong before
+// the archive is finalized the file's central directory will be hosed.
+MINIZ_STATIC mz_bool mz_zip_writer_init_from_reader(mz_zip_archive *pZip, const char *pFilename);
+
+// Adds the contents of a memory buffer to an archive. These functions record the current local time into the archive.
+// To add a directory entry, call this method with an archive name ending in a forwardslash with empty buffer.
+// level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION.
+MINIZ_STATIC mz_bool mz_zip_writer_add_mem(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, mz_uint level_and_flags);
+MINIZ_STATIC mz_bool mz_zip_writer_add_mem_ex(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, mz_uint64 uncomp_size, mz_uint32 uncomp_crc32);
+
+#ifndef MINIZ_NO_STDIO
+// Adds the contents of a disk file to an archive. This function also records the disk file's modified time into the archive.
+// level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION.
+MINIZ_STATIC mz_bool mz_zip_writer_add_file(mz_zip_archive *pZip, const char *pArchive_name, const char *pSrc_filename, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags);
+#endif
+
+// Adds a file to an archive by fully cloning the data from another archive.
+// This function fully clones the source file's compressed data (no recompression), along with its full filename, extra data, and comment fields.
+MINIZ_STATIC mz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive *pSource_zip, mz_uint file_index);
+
+// Finalizes the archive by writing the central directory records followed by the end of central directory record.
+// After an archive is finalized, the only valid call on the mz_zip_archive struct is mz_zip_writer_end().
+// An archive must be manually finalized by calling this function for it to be valid.
+MINIZ_STATIC mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip);
+MINIZ_STATIC mz_bool mz_zip_writer_finalize_heap_archive(mz_zip_archive *pZip, void **pBuf, size_t *pSize);
+
+// Ends archive writing, freeing all allocations, and closing the output file if mz_zip_writer_init_file() was used.
+// Note for the archive to be valid, it must have been finalized before ending.
+MINIZ_STATIC mz_bool mz_zip_writer_end(mz_zip_archive *pZip);
+
+// Misc. high-level helper functions:
+
+// mz_zip_add_mem_to_archive_file_in_place() efficiently (but not atomically) appends a memory blob to a ZIP archive.
+// level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION.
+MINIZ_STATIC mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags);
+
+// Reads a single file from an archive into a heap block.
+// Returns NULL on failure.
+MINIZ_STATIC void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, size_t *pSize, mz_uint zip_flags);
+
+#endif // #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS
+
+#endif // #ifndef MINIZ_NO_ARCHIVE_APIS
+
+// ------------------- Low-level Decompression API Definitions
+#ifndef MINIZ_NO_INFLATE_APIS
+
+// Decompression flags used by tinfl_decompress().
+// TINFL_FLAG_PARSE_ZLIB_HEADER: If set, the input has a valid zlib header and ends with an adler32 checksum (it's a valid zlib stream). Otherwise, the input is a raw deflate stream.
+// TINFL_FLAG_HAS_MORE_INPUT: If set, there are more input bytes available beyond the end of the supplied input buffer. If clear, the input buffer contains all remaining input.
+// TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF: If set, the output buffer is large enough to hold the entire decompressed stream. If clear, the output buffer is at least the size of the dictionary (typically 32KB).
+// TINFL_FLAG_COMPUTE_ADLER32: Force adler-32 checksum computation of the decompressed bytes.
+enum
+{
+  TINFL_FLAG_PARSE_ZLIB_HEADER = 1,
+  TINFL_FLAG_HAS_MORE_INPUT = 2,
+  TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF = 4,
+  TINFL_FLAG_COMPUTE_ADLER32 = 8
+};
+
+// High level decompression functions:
+// tinfl_decompress_mem_to_heap() decompresses a block in memory to a heap block allocated via malloc().
+// On entry:
+//  pSrc_buf, src_buf_len: Pointer and size of the Deflate or zlib source data to decompress.
+// On return:
+//  Function returns a pointer to the decompressed data, or NULL on failure.
+//  *pOut_len will be set to the decompressed data's size, which could be larger than src_buf_len on uncompressible data.
+//  The caller must call mz_free() on the returned block when it's no longer needed.
+MINIZ_STATIC void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags);
+
+// tinfl_decompress_mem_to_mem() decompresses a block in memory to another block in memory.
+// Returns TINFL_DECOMPRESS_MEM_TO_MEM_FAILED on failure, or the number of bytes written on success.
+#define TINFL_DECOMPRESS_MEM_TO_MEM_FAILED ((size_t)(-1))
+MINIZ_STATIC size_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags);
+
+// tinfl_decompress_mem_to_callback() decompresses a block in memory to an internal 32KB buffer, and a user provided callback function will be called to flush the buffer.
+// Returns 1 on success or 0 on failure.
+typedef int (*tinfl_put_buf_func_ptr)(const void* pBuf, int len, void *pUser);
+MINIZ_STATIC int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags);
+
+struct tinfl_decompressor_tag; typedef struct tinfl_decompressor_tag tinfl_decompressor;
+
+// Max size of LZ dictionary.
+#define TINFL_LZ_DICT_SIZE 32768
+
+// Return status.
+typedef enum
+{
+  TINFL_STATUS_BAD_PARAM = -3,
+  TINFL_STATUS_ADLER32_MISMATCH = -2,
+  TINFL_STATUS_FAILED = -1,
+  TINFL_STATUS_DONE = 0,
+  TINFL_STATUS_NEEDS_MORE_INPUT = 1,
+  TINFL_STATUS_HAS_MORE_OUTPUT = 2
+} tinfl_status;
+
+// Initializes the decompressor to its initial state.
+#define tinfl_init(r) do { (r)->m_state = 0; } MZ_MACRO_END
+#define tinfl_get_adler32(r) (r)->m_check_adler32
+
+// Main low-level decompressor coroutine function. This is the only function actually needed for decompression. All the other functions are just high-level helpers for improved usability.
+// This is a universal API, i.e. it can be used as a building block to build any desired higher level decompression API. In the limit case, it can be called once per every byte input or output.
+MINIZ_STATIC tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags);
+
+// Internal/private bits follow.
+enum
+{
+  TINFL_MAX_HUFF_TABLES = 3, TINFL_MAX_HUFF_SYMBOLS_0 = 288, TINFL_MAX_HUFF_SYMBOLS_1 = 32, TINFL_MAX_HUFF_SYMBOLS_2 = 19,
+  TINFL_FAST_LOOKUP_BITS = 10, TINFL_FAST_LOOKUP_SIZE = 1 << TINFL_FAST_LOOKUP_BITS
+};
+
+typedef struct
+{
+  mz_uint8 m_code_size[TINFL_MAX_HUFF_SYMBOLS_0];
+  mz_int16 m_look_up[TINFL_FAST_LOOKUP_SIZE], m_tree[TINFL_MAX_HUFF_SYMBOLS_0 * 2];
+} tinfl_huff_table;
+
+#if MINIZ_HAS_64BIT_REGISTERS
+  #define TINFL_USE_64BIT_BITBUF 1
+#endif
+
+#if TINFL_USE_64BIT_BITBUF
+  typedef mz_uint64 tinfl_bit_buf_t;
+  #define TINFL_BITBUF_SIZE (64)
+#else
+  typedef mz_uint32 tinfl_bit_buf_t;
+  #define TINFL_BITBUF_SIZE (32)
+#endif
+
+struct tinfl_decompressor_tag
+{
+  mz_uint32 m_state, m_num_bits, m_zhdr0, m_zhdr1, m_z_adler32, m_final, m_type, m_check_adler32, m_dist, m_counter, m_num_extra, m_table_sizes[TINFL_MAX_HUFF_TABLES];
+  tinfl_bit_buf_t m_bit_buf;
+  size_t m_dist_from_out_buf_start;
+  tinfl_huff_table m_tables[TINFL_MAX_HUFF_TABLES];
+  mz_uint8 m_raw_header[4], m_len_codes[TINFL_MAX_HUFF_SYMBOLS_0 + TINFL_MAX_HUFF_SYMBOLS_1 + 137];
+};
+#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/
+
+// ------------------- Low-level Compression API Definitions
+#ifndef MINIZ_NO_DEFLATE_APIS
+
+// Set TDEFL_LESS_MEMORY to 1 to use less memory (compression will be slightly slower, and raw/dynamic blocks will be output more frequently).
+#define TDEFL_LESS_MEMORY 0
+
+// tdefl_init() compression flags logically OR'd together (low 12 bits contain the max. number of probes per dictionary search):
+// TDEFL_DEFAULT_MAX_PROBES: The compressor defaults to 128 dictionary probes per dictionary search. 0=Huffman only, 1=Huffman+LZ (fastest/crap compression), 4095=Huffman+LZ (slowest/best compression).
+enum
+{
+  TDEFL_HUFFMAN_ONLY = 0, TDEFL_DEFAULT_MAX_PROBES = 128, TDEFL_MAX_PROBES_MASK = 0xFFF
+};
+
+// TDEFL_WRITE_ZLIB_HEADER: If set, the compressor outputs a zlib header before the deflate data, and the Adler-32 of the source data at the end. Otherwise, you'll get raw deflate data.
+// TDEFL_COMPUTE_ADLER32: Always compute the adler-32 of the input data (even when not writing zlib headers).
+// TDEFL_GREEDY_PARSING_FLAG: Set to use faster greedy parsing, instead of more efficient lazy parsing.
+// TDEFL_NONDETERMINISTIC_PARSING_FLAG: Enable to decrease the compressor's initialization time to the minimum, but the output may vary from run to run given the same input (depending on the contents of memory).
+// TDEFL_RLE_MATCHES: Only look for RLE matches (matches with a distance of 1)
+// TDEFL_FILTER_MATCHES: Discards matches <= 5 chars if enabled.
+// TDEFL_FORCE_ALL_STATIC_BLOCKS: Disable usage of optimized Huffman tables.
+// TDEFL_FORCE_ALL_RAW_BLOCKS: Only use raw (uncompressed) deflate blocks.
+// The low 12 bits are reserved to control the max # of hash probes per dictionary lookup (see TDEFL_MAX_PROBES_MASK).
+enum
+{
+  TDEFL_WRITE_ZLIB_HEADER             = 0x01000,
+  TDEFL_COMPUTE_ADLER32               = 0x02000,
+  TDEFL_GREEDY_PARSING_FLAG           = 0x04000,
+  TDEFL_NONDETERMINISTIC_PARSING_FLAG = 0x08000,
+  TDEFL_RLE_MATCHES                   = 0x10000,
+  TDEFL_FILTER_MATCHES                = 0x20000,
+  TDEFL_FORCE_ALL_STATIC_BLOCKS       = 0x40000,
+  TDEFL_FORCE_ALL_RAW_BLOCKS          = 0x80000
+};
+
+#ifndef MINIZ_SDL_NOUNUSED
+// High level compression functions:
+// tdefl_compress_mem_to_heap() compresses a block in memory to a heap block allocated via malloc().
+// On entry:
+//  pSrc_buf, src_buf_len: Pointer and size of source block to compress.
+//  flags: The max match finder probes (default is 128) logically OR'd against the above flags. Higher probes are slower but improve compression.
+// On return:
+//  Function returns a pointer to the compressed data, or NULL on failure.
+//  *pOut_len will be set to the compressed data's size, which could be larger than src_buf_len on uncompressible data.
+//  The caller must free() the returned block when it's no longer needed.
+MINIZ_STATIC void *tdefl_compress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags);
+
+// tdefl_compress_mem_to_mem() compresses a block in memory to another block in memory.
+// Returns 0 on failure.
+MINIZ_STATIC size_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags);
+#endif /* MINIZ_SDL_NOUNUSED */
+
+// Compresses an image to a compressed PNG file in memory.
+// On entry:
+//  pImage, w, h, and num_chans describe the image to compress. num_chans may be 1, 2, 3, or 4.
+//  The image pitch in bytes per scanline will be w*num_chans. The leftmost pixel on the top scanline is stored first in memory.
+//  level may range from [0,10], use MZ_NO_COMPRESSION, MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc. or a decent default is MZ_DEFAULT_LEVEL
+//  If flip is true, the image will be flipped on the Y axis (useful for OpenGL apps).
+// On return:
+//  Function returns a pointer to the compressed data, or NULL on failure.
+//  *pLen_out will be set to the size of the PNG image file.
+//  The caller must mz_free() the returned heap block (which will typically be larger than *pLen_out) when it's no longer needed.
+MINIZ_STATIC void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int h, int num_chans, int bpl, size_t *pLen_out, mz_uint level, mz_bool flip, mz_uint8 *plte, int plte_size, mz_uint8 *trns, int trns_size);
+#ifndef MINIZ_SDL_NOUNUSED
+MINIZ_STATIC void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, int bpl, size_t *pLen_out);
+#endif
+
+// Output stream interface. The compressor uses this interface to write compressed data. It'll typically be called TDEFL_OUT_BUF_SIZE at a time.
+typedef mz_bool (*tdefl_put_buf_func_ptr)(const void* pBuf, int len, void *pUser);
+
+#ifndef MINIZ_SDL_NOUNUSED
+// tdefl_compress_mem_to_output() compresses a block to an output stream. The above helpers use this function internally.
+MINIZ_STATIC mz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags);
+#endif
+
+//enum { TDEFL_MAX_HUFF_TABLES = 3, TDEFL_MAX_HUFF_SYMBOLS_0 = 288, TDEFL_MAX_HUFF_SYMBOLS_1 = 32, TDEFL_MAX_HUFF_SYMBOLS_2 = 19, TDEFL_LZ_DICT_SIZE = 32768, TDEFL_LZ_DICT_SIZE_MASK = TDEFL_LZ_DICT_SIZE - 1, TDEFL_MIN_MATCH_LEN = 3, TDEFL_MAX_MATCH_LEN = 258 };
+#define TDEFL_MAX_HUFF_TABLES	3
+#define TDEFL_MAX_HUFF_SYMBOLS_0 288
+#define TDEFL_MAX_HUFF_SYMBOLS_1 32
+#define TDEFL_MAX_HUFF_SYMBOLS_2 19
+#define TDEFL_LZ_DICT_SIZE 32768
+#define TDEFL_LZ_DICT_SIZE_MASK	( TDEFL_LZ_DICT_SIZE - 1 )
+#define TDEFL_MIN_MATCH_LEN	3
+#define TDEFL_MAX_MATCH_LEN 258
+
+// TDEFL_OUT_BUF_SIZE MUST be large enough to hold a single entire compressed output block (using static/fixed Huffman codes).
+#if TDEFL_LESS_MEMORY
+enum { TDEFL_LZ_CODE_BUF_SIZE = 24 * 1024, TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13 ) / 10, TDEFL_MAX_HUFF_SYMBOLS = 288, TDEFL_LZ_HASH_BITS = 12, TDEFL_LEVEL1_HASH_SIZE_MASK = 4095, TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3, TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS };
+#else
+enum { TDEFL_LZ_CODE_BUF_SIZE = 64 * 1024, TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13 ) / 10, TDEFL_MAX_HUFF_SYMBOLS = 288, TDEFL_LZ_HASH_BITS = 15, TDEFL_LEVEL1_HASH_SIZE_MASK = 4095, TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3, TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS };
+#endif
+
+// The low-level tdefl functions below may be used directly if the above helper functions aren't flexible enough. The low-level functions don't make any heap allocations, unlike the above helper functions.
+typedef enum
+{
+  TDEFL_STATUS_BAD_PARAM = -2,
+  TDEFL_STATUS_PUT_BUF_FAILED = -1,
+  TDEFL_STATUS_OKAY = 0,
+  TDEFL_STATUS_DONE = 1,
+} tdefl_status;
+
+// Must map to MZ_NO_FLUSH, MZ_SYNC_FLUSH, etc. enums
+typedef enum
+{
+  TDEFL_NO_FLUSH = 0,
+  TDEFL_SYNC_FLUSH = 2,
+  TDEFL_FULL_FLUSH = 3,
+  TDEFL_FINISH = 4
+} tdefl_flush;
+
+// tdefl's compression state structure.
+typedef struct
+{
+  tdefl_put_buf_func_ptr m_pPut_buf_func;
+  void *m_pPut_buf_user;
+  mz_uint m_flags, m_max_probes[2];
+  int m_greedy_parsing;
+  mz_uint m_adler32, m_lookahead_pos, m_lookahead_size, m_dict_size;
+  mz_uint8 *m_pLZ_code_buf, *m_pLZ_flags, *m_pOutput_buf, *m_pOutput_buf_end;
+  mz_uint m_num_flags_left, m_total_lz_bytes, m_lz_code_buf_dict_pos, m_bits_in, m_bit_buffer;
+  mz_uint m_saved_match_dist, m_saved_match_len, m_saved_lit, m_output_flush_ofs, m_output_flush_remaining, m_finished, m_block_index, m_wants_to_finish;
+  tdefl_status m_prev_return_status;
+  const void *m_pIn_buf;
+  void *m_pOut_buf;
+  size_t *m_pIn_buf_size, *m_pOut_buf_size;
+  tdefl_flush m_flush;
+  const mz_uint8 *m_pSrc;
+  size_t m_src_buf_left, m_out_buf_ofs;
+  mz_uint8 m_dict[TDEFL_LZ_DICT_SIZE + TDEFL_MAX_MATCH_LEN - 1];
+  mz_uint16 m_huff_count[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];
+  mz_uint16 m_huff_codes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];
+  mz_uint8 m_huff_code_sizes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];
+  mz_uint8 m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE];
+  mz_uint16 m_next[TDEFL_LZ_DICT_SIZE];
+  mz_uint16 m_hash[TDEFL_LZ_HASH_SIZE];
+  mz_uint8 m_output_buf[TDEFL_OUT_BUF_SIZE];
+} tdefl_compressor;
+
+// Initializes the compressor.
+// There is no corresponding deinit() function because the tdefl API's do not dynamically allocate memory.
+// pBut_buf_func: If NULL, output data will be supplied to the specified callback. In this case, the user should call the tdefl_compress_buffer() API for compression.
+// If pBut_buf_func is NULL the user should always call the tdefl_compress() API.
+// flags: See the above enums (TDEFL_HUFFMAN_ONLY, TDEFL_WRITE_ZLIB_HEADER, etc.)
+MINIZ_STATIC tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags);
+
+// Compresses a block of data, consuming as much of the specified input buffer as possible, and writing as much compressed data to the specified output buffer as possible.
+MINIZ_STATIC tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush);
+
+// tdefl_compress_buffer() is only usable when the tdefl_init() is called with a non-NULL tdefl_put_buf_func_ptr.
+// tdefl_compress_buffer() always consumes the entire input buffer.
+MINIZ_STATIC tdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush);
+
+#ifndef MINIZ_SDL_NOUNUSED
+MINIZ_STATIC tdefl_status tdefl_get_prev_return_status(tdefl_compressor *d);
+MINIZ_STATIC mz_uint32 tdefl_get_adler32(tdefl_compressor *d);
+#endif
+
+// Can't use tdefl_create_comp_flags_from_zip_params if MINIZ_NO_ZLIB_APIS isn't defined, because it uses some of its macros.
+#ifndef MINIZ_NO_ZLIB_APIS
+// Create tdefl_compress() flags given zlib-style compression parameters.
+// level may range from [0,10] (where 10 is absolute max compression, but may be much slower on some files)
+// window_bits may be -15 (raw deflate) or 15 (zlib)
+// strategy may be either MZ_DEFAULT_STRATEGY, MZ_FILTERED, MZ_HUFFMAN_ONLY, MZ_RLE, or MZ_FIXED
+MINIZ_STATIC mz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy);
+#endif // #ifndef MINIZ_NO_ZLIB_APIS
+
+#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // MINIZ_HEADER_INCLUDED
+
+// ------------------- End of Header: Implementation follows. (If you only want the header, define MINIZ_HEADER_FILE_ONLY.)
+
+#ifndef MINIZ_HEADER_FILE_ONLY
+
+#ifdef _MSC_VER
+#pragma warning (push)
+#pragma warning (disable:4505) // unreferenced local function has been removed
+#endif
+
+typedef unsigned char mz_validate_uint16[sizeof(mz_uint16)==2 ? 1 : -1];
+typedef unsigned char mz_validate_uint32[sizeof(mz_uint32)==4 ? 1 : -1];
+typedef unsigned char mz_validate_uint64[sizeof(mz_uint64)==8 ? 1 : -1];
+
+/*#include <string.h>*/
+
+#ifndef MZ_ASSERT
+#include <assert.h>
+#define MZ_ASSERT(x) assert(x)
+#endif
+
+#ifdef MINIZ_NO_MALLOC
+  #define MZ_MALLOC(x) NULL
+  #define MZ_FREE(x) (void)x, ((void)0)
+  #define MZ_REALLOC(p, x) NULL
+#elif defined(MINIZ_SDL_MALLOC)
+  #define MZ_MALLOC(x) SDL_malloc(x)
+  #define MZ_FREE(x) SDL_free(x)
+  #define MZ_REALLOC(p, x) SDL_realloc(p, x)
+#else
+  #define MZ_MALLOC(x) malloc(x)
+  #define MZ_FREE(x) free(x)
+  #define MZ_REALLOC(p, x) realloc(p, x)
+#endif
+
+#define MZ_MAX(a,b) (((a)>(b))?(a):(b))
+#define MZ_MIN(a,b) (((a)<(b))?(a):(b))
+#define MZ_CLEAR_OBJ(obj) memset(&(obj), 0, sizeof(obj))
+#define MZ_CLEAR_ARR(obj) memset((obj), 0, sizeof(obj))
+#define MZ_CLEAR_PTR(obj) memset((obj), 0, sizeof(*obj))
+
+#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN
+  #define MZ_READ_LE16(p) *((const mz_uint16 *)(p))
+  #define MZ_READ_LE32(p) *((const mz_uint32 *)(p))
+#else
+  #define MZ_READ_LE16(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U))
+  #define MZ_READ_LE32(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U) | ((mz_uint32)(((const mz_uint8 *)(p))[2]) << 16U) | ((mz_uint32)(((const mz_uint8 *)(p))[3]) << 24U))
+#endif
+
+#ifdef _MSC_VER
+  #define MZ_FORCEINLINE __forceinline
+#elif defined(__GNUC__)
+  #define MZ_FORCEINLINE inline __attribute__((__always_inline__))
+#else
+  #define MZ_FORCEINLINE inline
+#endif
+
+#ifdef __cplusplus
+  extern "C" {
+#endif
+
+// ------------------- zlib-style API's
+
+mz_ulong mz_adler32(mz_ulong adler, const unsigned char *ptr, size_t buf_len)
+{
+  mz_uint32 i, s1 = (mz_uint32)(adler & 0xffff), s2 = (mz_uint32)(adler >> 16); size_t block_len = buf_len % 5552;
+  if (!ptr) return MZ_ADLER32_INIT;
+  while (buf_len) {
+    for (i = 0; i + 7 < block_len; i += 8, ptr += 8) {
+      s1 += ptr[0], s2 += s1; s1 += ptr[1], s2 += s1; s1 += ptr[2], s2 += s1; s1 += ptr[3], s2 += s1;
+      s1 += ptr[4], s2 += s1; s1 += ptr[5], s2 += s1; s1 += ptr[6], s2 += s1; s1 += ptr[7], s2 += s1;
+    }
+    for ( ; i < block_len; ++i) s1 += *ptr++, s2 += s1;
+    s1 %= 65521U, s2 %= 65521U; buf_len -= block_len; block_len = 5552;
+  }
+  return (s2 << 16) + s1;
+}
+
+// Karl Malbrain's compact CRC-32. See "A compact CCITT crc16 and crc32 C implementation that balances processor cache usage against speed": http://www.geocities.com/malbrain/
+mz_ulong mz_crc32(mz_ulong crc, const mz_uint8 *ptr, size_t buf_len)
+{
+  static const mz_uint32 s_crc32[16] = { 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
+    0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c };
+  mz_uint32 crcu32 = (mz_uint32)crc;
+  if (!ptr) return MZ_CRC32_INIT;
+  crcu32 = ~crcu32; while (buf_len--) { mz_uint8 b = *ptr++; crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b & 0xF)]; crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b >> 4)]; }
+  return ~crcu32;
+}
+
+MINIZ_STATIC void mz_free(void *p)
+{
+  MZ_FREE(p);
+}
+
+#ifndef MINIZ_NO_ZLIB_APIS
+
+static void *def_alloc_func(void *opaque, size_t items, size_t size) { (void)opaque, (void)items, (void)size; return MZ_MALLOC(items * size); }
+static void def_free_func(void *opaque, void *address) { (void)opaque, (void)address; MZ_FREE(address); }
+static void *def_realloc_func(void *opaque, void *address, size_t items, size_t size) { (void)opaque, (void)address, (void)items, (void)size; return MZ_REALLOC(address, items * size); }
+
+const char *mz_version(void)
+{
+  return MZ_VERSION;
+}
+
+#ifndef MINIZ_NO_DEFLATE_APIS
+
+int mz_deflateInit(mz_streamp pStream, int level)
+{
+  return mz_deflateInit2(pStream, level, MZ_DEFLATED, MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY);
+}
+
+int mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy)
+{
+  tdefl_compressor *pComp;
+  mz_uint comp_flags = TDEFL_COMPUTE_ADLER32 | tdefl_create_comp_flags_from_zip_params(level, window_bits, strategy);
+
+  if (!pStream) return MZ_STREAM_ERROR;
+  if ((method != MZ_DEFLATED) || ((mem_level < 1) || (mem_level > 9)) || ((window_bits != MZ_DEFAULT_WINDOW_BITS) && (-window_bits != MZ_DEFAULT_WINDOW_BITS))) return MZ_PARAM_ERROR;
+
+  pStream->data_type = 0;
+  pStream->adler = MZ_ADLER32_INIT;
+  pStream->msg = NULL;
+  pStream->reserved = 0;
+  pStream->total_in = 0;
+  pStream->total_out = 0;
+  if (!pStream->zalloc) pStream->zalloc = def_alloc_func;
+  if (!pStream->zfree) pStream->zfree = def_free_func;
+
+  pComp = (tdefl_compressor *)pStream->zalloc(pStream->opaque, 1, sizeof(tdefl_compressor));
+  if (!pComp)
+    return MZ_MEM_ERROR;
+
+  pStream->state = (struct mz_internal_state *)pComp;
+
+  if (tdefl_init(pComp, NULL, NULL, comp_flags) != TDEFL_STATUS_OKAY)
+  {
+    mz_deflateEnd(pStream);
+    return MZ_PARAM_ERROR;
+  }
+
+  return MZ_OK;
+}
+
+int mz_deflateReset(mz_streamp pStream)
+{
+  if ((!pStream) || (!pStream->state) || (!pStream->zalloc) || (!pStream->zfree)) return MZ_STREAM_ERROR;
+  pStream->total_in = pStream->total_out = 0;
+  tdefl_init((tdefl_compressor*)pStream->state, NULL, NULL, ((tdefl_compressor*)pStream->state)->m_flags);
+  return MZ_OK;
+}
+
+int mz_deflate(mz_streamp pStream, int flush)
+{
+  size_t in_bytes, out_bytes;
+  mz_ulong orig_total_in, orig_total_out;
+  int mz_status = MZ_OK;
+
+  if ((!pStream) || (!pStream->state) || (flush < 0) || (flush > MZ_FINISH) || (!pStream->next_out)) return MZ_STREAM_ERROR;
+  if (!pStream->avail_out) return MZ_BUF_ERROR;
+
+  if (flush == MZ_PARTIAL_FLUSH) flush = MZ_SYNC_FLUSH;
+
+  if (((tdefl_compressor*)pStream->state)->m_prev_return_status == TDEFL_STATUS_DONE)
+    return (flush == MZ_FINISH) ? MZ_STREAM_END : MZ_BUF_ERROR;
+
+  orig_total_in = pStream->total_in; orig_total_out = pStream->total_out;
+  for ( ; ; )
+  {
+    tdefl_status defl_status;
+    in_bytes = pStream->avail_in; out_bytes = pStream->avail_out;
+
+    defl_status = tdefl_compress((tdefl_compressor*)pStream->state, pStream->next_in, &in_bytes, pStream->next_out, &out_bytes, (tdefl_flush)flush);
+    pStream->next_in += (mz_uint)in_bytes; pStream->avail_in -= (mz_uint)in_bytes;
+    pStream->total_in += (mz_uint)in_bytes; pStream->adler = tdefl_get_adler32((tdefl_compressor*)pStream->state);
+
+    pStream->next_out += (mz_uint)out_bytes; pStream->avail_out -= (mz_uint)out_bytes;
+    pStream->total_out += (mz_uint)out_bytes;
+
+    if (defl_status < 0)
+    {
+      mz_status = MZ_STREAM_ERROR;
+      break;
+    }
+    else if (defl_status == TDEFL_STATUS_DONE)
+    {
+      mz_status = MZ_STREAM_END;
+      break;
+    }
+    else if (!pStream->avail_out)
+      break;
+    else if ((!pStream->avail_in) && (flush != MZ_FINISH))
+    {
+      if ((flush) || (pStream->total_in != orig_total_in) || (pStream->total_out != orig_total_out))
+        break;
+      return MZ_BUF_ERROR; // Can't make forward progress without some input.
+    }
+  }
+  return mz_status;
+}
+
+int mz_deflateEnd(mz_streamp pStream)
+{
+  if (!pStream) return MZ_STREAM_ERROR;
+  if (pStream->state)
+  {
+    pStream->zfree(pStream->opaque, pStream->state);
+    pStream->state = NULL;
+  }
+  return MZ_OK;
+}
+
+mz_ulong mz_deflateBound(mz_streamp pStream, mz_ulong source_len)
+{
+  (void)pStream;
+  // This is really over conservative. (And lame, but it's actually pretty tricky to compute a true upper bound given the way tdefl's blocking works.)
+  return MZ_MAX(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5);
+}
+
+int mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len, int level)
+{
+  int status;
+  mz_stream stream;
+  memset(&stream, 0, sizeof(stream));
+
+  // In case mz_ulong is 64-bits (argh I hate longs).
+  if ((source_len | *pDest_len) > 0xFFFFFFFFU) return MZ_PARAM_ERROR;
+
+  stream.next_in = pSource;
+  stream.avail_in = (mz_uint32)source_len;
+  stream.next_out = pDest;
+  stream.avail_out = (mz_uint32)*pDest_len;
+
+  status = mz_deflateInit(&stream, level);
+  if (status != MZ_OK) return status;
+
+  status = mz_deflate(&stream, MZ_FINISH);
+  if (status != MZ_STREAM_END)
+  {
+    mz_deflateEnd(&stream);
+    return (status == MZ_OK) ? MZ_BUF_ERROR : status;
+  }
+
+  *pDest_len = stream.total_out;
+  return mz_deflateEnd(&stream);
+}
+
+int mz_compress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len)
+{
+  return mz_compress2(pDest, pDest_len, pSource, source_len, MZ_DEFAULT_COMPRESSION);
+}
+
+mz_ulong mz_compressBound(mz_ulong source_len)
+{
+  return mz_deflateBound(NULL, source_len);
+}
+
+#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/
+
+#ifndef MINIZ_NO_INFLATE_APIS
+
+typedef struct
+{
+  tinfl_decompressor m_decomp;
+  mz_uint m_dict_ofs, m_dict_avail, m_first_call, m_has_flushed; int m_window_bits;
+  mz_uint8 m_dict[TINFL_LZ_DICT_SIZE];
+  tinfl_status m_last_status;
+} inflate_state;
+
+int mz_inflateInit2(mz_streamp pStream, int window_bits)
+{
+  inflate_state *pDecomp;
+  if (!pStream) return MZ_STREAM_ERROR;
+  if ((window_bits != MZ_DEFAULT_WINDOW_BITS) && (-window_bits != MZ_DEFAULT_WINDOW_BITS)) return MZ_PARAM_ERROR;
+
+  pStream->data_type = 0;
+  pStream->adler = 0;
+  pStream->msg = NULL;
+  pStream->total_in = 0;
+  pStream->total_out = 0;
+  pStream->reserved = 0;
+  if (!pStream->zalloc) pStream->zalloc = def_alloc_func;
+  if (!pStream->zfree) pStream->zfree = def_free_func;
+
+  pDecomp = (inflate_state*)pStream->zalloc(pStream->opaque, 1, sizeof(inflate_state));
+  if (!pDecomp) return MZ_MEM_ERROR;
+
+  pStream->state = (struct mz_internal_state *)pDecomp;
+
+  tinfl_init(&pDecomp->m_decomp);
+  pDecomp->m_dict_ofs = 0;
+  pDecomp->m_dict_avail = 0;
+  pDecomp->m_last_status = TINFL_STATUS_NEEDS_MORE_INPUT;
+  pDecomp->m_first_call = 1;
+  pDecomp->m_has_flushed = 0;
+  pDecomp->m_window_bits = window_bits;
+
+  return MZ_OK;
+}
+
+int mz_inflateInit(mz_streamp pStream)
+{
+   return mz_inflateInit2(pStream, MZ_DEFAULT_WINDOW_BITS);
+}
+
+int mz_inflate(mz_streamp pStream, int flush)
+{
+  inflate_state* pState;
+  mz_uint n, first_call, decomp_flags = TINFL_FLAG_COMPUTE_ADLER32;
+  size_t in_bytes, out_bytes, orig_avail_in;
+  tinfl_status status;
+
+  if ((!pStream) || (!pStream->state)) return MZ_STREAM_ERROR;
+  if (flush == MZ_PARTIAL_FLUSH) flush = MZ_SYNC_FLUSH;
+  if ((flush) && (flush != MZ_SYNC_FLUSH) && (flush != MZ_FINISH)) return MZ_STREAM_ERROR;
+
+  pState = (inflate_state*)pStream->state;
+  if (pState->m_window_bits > 0) decomp_flags |= TINFL_FLAG_PARSE_ZLIB_HEADER;
+  orig_avail_in = pStream->avail_in;
+
+  first_call = pState->m_first_call; pState->m_first_call = 0;
+  if (pState->m_last_status < 0) return MZ_DATA_ERROR;
+
+  if (pState->m_has_flushed && (flush != MZ_FINISH)) return MZ_STREAM_ERROR;
+  pState->m_has_flushed |= (flush == MZ_FINISH);
+
+  if ((flush == MZ_FINISH) && (first_call))
+  {
+    // MZ_FINISH on the first call implies that the input and output buffers are large enough to hold the entire compressed/decompressed file.
+    decomp_flags |= TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF;
+    in_bytes = pStream->avail_in; out_bytes = pStream->avail_out;
+    status = tinfl_decompress(&pState->m_decomp, pStream->next_in, &in_bytes, pStream->next_out, pStream->next_out, &out_bytes, decomp_flags);
+    pState->m_last_status = status;
+    pStream->next_in += (mz_uint)in_bytes; pStream->avail_in -= (mz_uint)in_bytes; pStream->total_in += (mz_uint)in_bytes;
+    pStream->adler = tinfl_get_adler32(&pState->m_decomp);
+    pStream->next_out += (mz_uint)out_bytes; pStream->avail_out -= (mz_uint)out_bytes; pStream->total_out += (mz_uint)out_bytes;
+
+    if (status < 0)
+      return MZ_DATA_ERROR;
+    else if (status != TINFL_STATUS_DONE)
+    {
+      pState->m_last_status = TINFL_STATUS_FAILED;
+      return MZ_BUF_ERROR;
+    }
+    return MZ_STREAM_END;
+  }
+  // flush != MZ_FINISH then we must assume there's more input.
+  if (flush != MZ_FINISH) decomp_flags |= TINFL_FLAG_HAS_MORE_INPUT;
+
+  if (pState->m_dict_avail)
+  {
+    n = MZ_MIN(pState->m_dict_avail, pStream->avail_out);
+    memcpy(pStream->next_out, pState->m_dict + pState->m_dict_ofs, n);
+    pStream->next_out += n; pStream->avail_out -= n; pStream->total_out += n;
+    pState->m_dict_avail -= n; pState->m_dict_ofs = (pState->m_dict_ofs + n) & (TINFL_LZ_DICT_SIZE - 1);
+    return ((pState->m_last_status == TINFL_STATUS_DONE) && (!pState->m_dict_avail)) ? MZ_STREAM_END : MZ_OK;
+  }
+
+  for ( ; ; )
+  {
+    in_bytes = pStream->avail_in;
+    out_bytes = TINFL_LZ_DICT_SIZE - pState->m_dict_ofs;
+
+    status = tinfl_decompress(&pState->m_decomp, pStream->next_in, &in_bytes, pState->m_dict, pState->m_dict + pState->m_dict_ofs, &out_bytes, decomp_flags);
+    pState->m_last_status = status;
+
+    pStream->next_in += (mz_uint)in_bytes; pStream->avail_in -= (mz_uint)in_bytes;
+    pStream->total_in += (mz_uint)in_bytes; pStream->adler = tinfl_get_adler32(&pState->m_decomp);
+
+    pState->m_dict_avail = (mz_uint)out_bytes;
+
+    n = MZ_MIN(pState->m_dict_avail, pStream->avail_out);
+    memcpy(pStream->next_out, pState->m_dict + pState->m_dict_ofs, n);
+    pStream->next_out += n; pStream->avail_out -= n; pStream->total_out += n;
+    pState->m_dict_avail -= n; pState->m_dict_ofs = (pState->m_dict_ofs + n) & (TINFL_LZ_DICT_SIZE - 1);
+
+    if (status < 0)
+       return MZ_DATA_ERROR; // Stream is corrupted (there could be some uncompressed data left in the output dictionary - oh well).
+    else if ((status == TINFL_STATUS_NEEDS_MORE_INPUT) && (!orig_avail_in))
+      return MZ_BUF_ERROR; // Signal caller that we can't make forward progress without supplying more input or by setting flush to MZ_FINISH.
+    else if (flush == MZ_FINISH)
+    {
+       // The output buffer MUST be large to hold the remaining uncompressed data when flush==MZ_FINISH.
+       if (status == TINFL_STATUS_DONE)
+          return pState->m_dict_avail ? MZ_BUF_ERROR : MZ_STREAM_END;
+       // status here must be TINFL_STATUS_HAS_MORE_OUTPUT, which means there's at least 1 more byte on the way. If there's no more room left in the output buffer then something is wrong.
+       else if (!pStream->avail_out)
+          return MZ_BUF_ERROR;
+    }
+    else if ((status == TINFL_STATUS_DONE) || (!pStream->avail_in) || (!pStream->avail_out) || (pState->m_dict_avail))
+      break;
+  }
+
+  return ((status == TINFL_STATUS_DONE) && (!pState->m_dict_avail)) ? MZ_STREAM_END : MZ_OK;
+}
+
+int mz_inflateEnd(mz_streamp pStream)
+{
+  if (!pStream)
+    return MZ_STREAM_ERROR;
+  if (pStream->state)
+  {
+    pStream->zfree(pStream->opaque, pStream->state);
+    pStream->state = NULL;
+  }
+  return MZ_OK;
+}
+
+int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len)
+{
+  mz_stream stream;
+  int status;
+  memset(&stream, 0, sizeof(stream));
+
+  // In case mz_ulong is 64-bits (argh I hate longs).
+  if ((source_len | *pDest_len) > 0xFFFFFFFFU) return MZ_PARAM_ERROR;
+
+  stream.next_in = pSource;
+  stream.avail_in = (mz_uint32)source_len;
+  stream.next_out = pDest;
+  stream.avail_out = (mz_uint32)*pDest_len;
+
+  status = mz_inflateInit(&stream);
+  if (status != MZ_OK)
+    return status;
+
+  status = mz_inflate(&stream, MZ_FINISH);
+  if (status != MZ_STREAM_END)
+  {
+    mz_inflateEnd(&stream);
+    return ((status == MZ_BUF_ERROR) && (!stream.avail_in)) ? MZ_DATA_ERROR : status;
+  }
+  *pDest_len = stream.total_out;
+
+  return mz_inflateEnd(&stream);
+}
+
+#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/
+
+const char *mz_error(int err)
+{
+  static struct { int m_err; const char *m_pDesc; } s_error_descs[] =
+  {
+    { MZ_OK, "" }, { MZ_STREAM_END, "stream end" }, { MZ_NEED_DICT, "need dictionary" }, { MZ_ERRNO, "file error" }, { MZ_STREAM_ERROR, "stream error" },
+    { MZ_DATA_ERROR, "data error" }, { MZ_MEM_ERROR, "out of memory" }, { MZ_BUF_ERROR, "buf error" }, { MZ_VERSION_ERROR, "version error" }, { MZ_PARAM_ERROR, "parameter error" }
+  };
+  mz_uint i; for (i = 0; i < sizeof(s_error_descs) / sizeof(s_error_descs[0]); ++i) if (s_error_descs[i].m_err == err) return s_error_descs[i].m_pDesc;
+  return NULL;
+}
+
+#endif //MINIZ_NO_ZLIB_APIS
+
+// ------------------- Low-level Decompression (completely independent from all compression API's)
+#ifndef MINIZ_NO_INFLATE_APIS
+
+#define TINFL_MEMCPY(d, s, l) memcpy(d, s, l)
+#define TINFL_MEMSET(p, c, l) memset(p, c, l)
+
+#define TINFL_CR_BEGIN switch(r->m_state) { case 0:
+#define TINFL_CR_RETURN(state_index, result) do { status = result; r->m_state = state_index; goto common_exit; case state_index:; } MZ_MACRO_END
+#define TINFL_CR_RETURN_FOREVER(state_index, result) do { for ( ; ; ) { TINFL_CR_RETURN(state_index, result); } } MZ_MACRO_END
+#define TINFL_CR_FINISH }
+
+// TODO: If the caller has indicated that there's no more input, and we attempt to read beyond the input buf, then something is wrong with the input because the inflator never
+// reads ahead more than it needs to. Currently TINFL_GET_BYTE() pads the end of the stream with 0's in this scenario.
+#define TINFL_GET_BYTE(state_index, c) do { \
+  if (pIn_buf_cur >= pIn_buf_end) { \
+    for ( ; ; ) { \
+      if (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT) { \
+        TINFL_CR_RETURN(state_index, TINFL_STATUS_NEEDS_MORE_INPUT); \
+        if (pIn_buf_cur < pIn_buf_end) { \
+          c = *pIn_buf_cur++; \
+          break; \
+        } \
+      } else { \
+        c = 0; \
+        break; \
+      } \
+    } \
+  } else c = *pIn_buf_cur++; } MZ_MACRO_END
+
+#define TINFL_NEED_BITS(state_index, n) do { mz_uint c; TINFL_GET_BYTE(state_index, c); bit_buf |= (((tinfl_bit_buf_t)c) << num_bits); num_bits += 8; } while (num_bits < (mz_uint)(n))
+#define TINFL_SKIP_BITS(state_index, n) do { if (num_bits < (mz_uint)(n)) { TINFL_NEED_BITS(state_index, n); } bit_buf >>= (n); num_bits -= (n); } MZ_MACRO_END
+#define TINFL_GET_BITS(state_index, b, n) do { if (num_bits < (mz_uint)(n)) { TINFL_NEED_BITS(state_index, n); } b = bit_buf & ((1 << (n)) - 1); bit_buf >>= (n); num_bits -= (n); } MZ_MACRO_END
+
+// TINFL_HUFF_BITBUF_FILL() is only used rarely, when the number of bytes remaining in the input buffer falls below 2.
+// It reads just enough bytes from the input stream that are needed to decode the next Huffman code (and absolutely no more). It works by trying to fully decode a
+// Huffman code by using whatever bits are currently present in the bit buffer. If this fails, it reads another byte, and tries again until it succeeds or until the
+// bit buffer contains >=15 bits (deflate's max. Huffman code size).
+#define TINFL_HUFF_BITBUF_FILL(state_index, pHuff) \
+  do { \
+    temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]; \
+    if (temp >= 0) { \
+      code_len = temp >> 9; \
+      if ((code_len) && (num_bits >= code_len)) \
+          break; \
+    } else if (num_bits > TINFL_FAST_LOOKUP_BITS) { \
+       code_len = TINFL_FAST_LOOKUP_BITS; \
+       do { \
+          temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)]; \
+       } while ((temp < 0) && (num_bits >= (code_len + 1))); \
+       if (temp >= 0) break; \
+    } \
+    TINFL_GET_BYTE(state_index, c); \
+    bit_buf |= (((tinfl_bit_buf_t)c) << num_bits); \
+    num_bits += 8; \
+  } while (num_bits < 15);
+
+// TINFL_HUFF_DECODE() decodes the next Huffman coded symbol. It's more complex than you would initially expect because the zlib API expects the decompressor to never read
+// beyond the final byte of the deflate stream. (In other words, when this macro wants to read another byte from the input, it REALLY needs another byte in order to fully
+// decode the next Huffman code.) Handling this properly is particularly important on raw deflate (non-zlib) streams, which aren't followed by a byte aligned adler-32.
+// The slow path is only executed at the very end of the input buffer.
+#define TINFL_HUFF_DECODE(state_index, sym, pHuff) do { \
+  int temp; mz_uint code_len, c; \
+  if (num_bits < 15) { \
+    if ((pIn_buf_end - pIn_buf_cur) < 2) { \
+       TINFL_HUFF_BITBUF_FILL(state_index, pHuff); \
+    } else { \
+       bit_buf |= (((tinfl_bit_buf_t)pIn_buf_cur[0]) << num_bits) | (((tinfl_bit_buf_t)pIn_buf_cur[1]) << (num_bits + 8)); pIn_buf_cur += 2; num_bits += 16; \
+    } \
+  } \
+  if ((temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) \
+    code_len = temp >> 9, temp &= 511; \
+  else { \
+    code_len = TINFL_FAST_LOOKUP_BITS; do { temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)]; } while (temp < 0); \
+  } sym = temp; bit_buf >>= code_len; num_bits -= code_len; } MZ_MACRO_END
+
+tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags)
+{
+  static const int s_length_base[31] = { 3,4,5,6,7,8,9,10,11,13, 15,17,19,23,27,31,35,43,51,59, 67,83,99,115,131,163,195,227,258,0,0 };
+  static const int s_length_extra[31]= { 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 };
+  static const int s_dist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, 257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0};
+  static const int s_dist_extra[32] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13};
+  static const mz_uint8 s_length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 };
+  static const int s_min_table_sizes[3] = { 257, 1, 4 };
+
+  tinfl_status status = TINFL_STATUS_FAILED; mz_uint32 num_bits, dist, counter, num_extra; tinfl_bit_buf_t bit_buf;
+  const mz_uint8 *pIn_buf_cur = pIn_buf_next, *const pIn_buf_end = pIn_buf_next + *pIn_buf_size;
+  mz_uint8 *pOut_buf_cur = pOut_buf_next, *const pOut_buf_end = pOut_buf_next + *pOut_buf_size;
+  size_t out_buf_size_mask = (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF) ? (size_t)-1 : ((pOut_buf_next - pOut_buf_start) + *pOut_buf_size) - 1, dist_from_out_buf_start;
+
+  // Ensure the output buffer's size is a power of 2, unless the output buffer is large enough to hold the entire output file (in which case it doesn't matter).
+  if (((out_buf_size_mask + 1) & out_buf_size_mask) || (pOut_buf_next < pOut_buf_start)) { *pIn_buf_size = *pOut_buf_size = 0; return TINFL_STATUS_BAD_PARAM; }
+
+  num_bits = r->m_num_bits; bit_buf = r->m_bit_buf; dist = r->m_dist; counter = r->m_counter; num_extra = r->m_num_extra; dist_from_out_buf_start = r->m_dist_from_out_buf_start;
+  TINFL_CR_BEGIN
+
+  bit_buf = num_bits = dist = counter = num_extra = r->m_zhdr0 = r->m_zhdr1 = 0; r->m_z_adler32 = r->m_check_adler32 = 1;
+  if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER)
+  {
+    TINFL_GET_BYTE(1, r->m_zhdr0); TINFL_GET_BYTE(2, r->m_zhdr1);
+    counter = (((r->m_zhdr0 * 256 + r->m_zhdr1) % 31 != 0) || (r->m_zhdr1 & 32) || ((r->m_zhdr0 & 15) != 8));
+    if (!(decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)) counter |= (((1U << (8U + (r->m_zhdr0 >> 4))) > 32768U) || ((out_buf_size_mask + 1) < (size_t)(1U << (8U + (r->m_zhdr0 >> 4)))));
+    if (counter) { TINFL_CR_RETURN_FOREVER(36, TINFL_STATUS_FAILED); }
+  }
+
+  do
+  {
+    TINFL_GET_BITS(3, r->m_final, 3); r->m_type = r->m_final >> 1;
+    if (r->m_type == 0)
+    {
+      TINFL_SKIP_BITS(5, num_bits & 7);
+      for (counter = 0; counter < 4; ++counter) { if (num_bits) TINFL_GET_BITS(6, r->m_raw_header[counter], 8); else TINFL_GET_BYTE(7, r->m_raw_header[counter]); }
+      if ((counter = (r->m_raw_header[0] | (r->m_raw_header[1] << 8))) != (mz_uint)(0xFFFF ^ (r->m_raw_header[2] | (r->m_raw_header[3] << 8)))) { TINFL_CR_RETURN_FOREVER(39, TINFL_STATUS_FAILED); }
+      while ((counter) && (num_bits))
+      {
+        TINFL_GET_BITS(51, dist, 8);
+        while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(52, TINFL_STATUS_HAS_MORE_OUTPUT); }
+        *pOut_buf_cur++ = (mz_uint8)dist;
+        counter--;
+      }
+      while (counter)
+      {
+        size_t n; while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(9, TINFL_STATUS_HAS_MORE_OUTPUT); }
+        while (pIn_buf_cur >= pIn_buf_end)
+        {
+          if (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT)
+          {
+            TINFL_CR_RETURN(38, TINFL_STATUS_NEEDS_MORE_INPUT);
+          }
+          else
+          {
+            TINFL_CR_RETURN_FOREVER(40, TINFL_STATUS_FAILED);
+          }
+        }
+        n = MZ_MIN(MZ_MIN((size_t)(pOut_buf_end - pOut_buf_cur), (size_t)(pIn_buf_end - pIn_buf_cur)), counter);
+        TINFL_MEMCPY(pOut_buf_cur, pIn_buf_cur, n); pIn_buf_cur += n; pOut_buf_cur += n; counter -= (mz_uint)n;
+      }
+    }
+    else if (r->m_type == 3)
+    {
+      TINFL_CR_RETURN_FOREVER(10, TINFL_STATUS_FAILED);
+    }
+    else
+    {
+      if (r->m_type == 1)
+      {
+        mz_uint8 *p = r->m_tables[0].m_code_size; mz_uint i;
+        r->m_table_sizes[0] = 288; r->m_table_sizes[1] = 32; TINFL_MEMSET(r->m_tables[1].m_code_size, 5, 32);
+        for ( i = 0; i <= 143; ++i) *p++ = 8;
+        for ( ; i <= 255; ++i) *p++ = 9;
+        for ( ; i <= 279; ++i) *p++ = 7;
+        for ( ; i <= 287; ++i) *p++ = 8;
+      }
+      else
+      {
+        for (counter = 0; counter < 3; counter++) { TINFL_GET_BITS(11, r->m_table_sizes[counter], "\05\05\04"[counter]); r->m_table_sizes[counter] += s_min_table_sizes[counter]; }
+        MZ_CLEAR_ARR(r->m_tables[2].m_code_size); for (counter = 0; counter < r->m_table_sizes[2]; counter++) { mz_uint s; TINFL_GET_BITS(14, s, 3); r->m_tables[2].m_code_size[s_length_dezigzag[counter]] = (mz_uint8)s; }
+        r->m_table_sizes[2] = 19;
+      }
+      for ( ; (int)r->m_type >= 0; r->m_type--)
+      {
+        int tree_next, tree_cur; tinfl_huff_table *pTable;
+        mz_uint i, j, used_syms, total, sym_index, next_code[17], total_syms[16]; pTable = &r->m_tables[r->m_type]; MZ_CLEAR_ARR(total_syms); MZ_CLEAR_ARR(pTable->m_look_up); MZ_CLEAR_ARR(pTable->m_tree);
+        for (i = 0; i < r->m_table_sizes[r->m_type]; ++i) total_syms[pTable->m_code_size[i]]++;
+        used_syms = 0, total = 0; next_code[0] = next_code[1] = 0;
+        for (i = 1; i <= 15; ++i) { used_syms += total_syms[i]; next_code[i + 1] = (total = ((total + total_syms[i]) << 1)); }
+        if ((65536 != total) && (used_syms > 1))
+        {
+          TINFL_CR_RETURN_FOREVER(35, TINFL_STATUS_FAILED);
+        }
+        for (tree_next = -1, sym_index = 0; sym_index < r->m_table_sizes[r->m_type]; ++sym_index)
+        {
+          mz_uint rev_code = 0, l, cur_code, code_size = pTable->m_code_size[sym_index]; if (!code_size) continue;
+          cur_code = next_code[code_size]++; for (l = code_size; l > 0; l--, cur_code >>= 1) rev_code = (rev_code << 1) | (cur_code & 1);
+          if (code_size <= TINFL_FAST_LOOKUP_BITS) { mz_int16 k = (mz_int16)((code_size << 9) | sym_index); while (rev_code < TINFL_FAST_LOOKUP_SIZE) { pTable->m_look_up[rev_code] = k; rev_code += (1 << code_size); } continue; }
+          if (0 == (tree_cur = pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)])) { pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)] = (mz_int16)tree_next; tree_cur = tree_next; tree_next -= 2; }
+          rev_code >>= (TINFL_FAST_LOOKUP_BITS - 1);
+          for (j = code_size; j > (TINFL_FAST_LOOKUP_BITS + 1); j--)
+          {
+            tree_cur -= ((rev_code >>= 1) & 1);
+            if (!pTable->m_tree[-tree_cur - 1]) { pTable->m_tree[-tree_cur - 1] = (mz_int16)tree_next; tree_cur = tree_next; tree_next -= 2; } else tree_cur = pTable->m_tree[-tree_cur - 1];
+          }
+          tree_cur -= ((rev_code >>= 1) & 1); pTable->m_tree[-tree_cur - 1] = (mz_int16)sym_index;
+        }
+        if (r->m_type == 2)
+        {
+          for (counter = 0; counter < (r->m_table_sizes[0] + r->m_table_sizes[1]); )
+          {
+            mz_uint s; TINFL_HUFF_DECODE(16, dist, &r->m_tables[2]); if (dist < 16) { r->m_len_codes[counter++] = (mz_uint8)dist; continue; }
+            if ((dist == 16) && (!counter))
+            {
+              TINFL_CR_RETURN_FOREVER(17, TINFL_STATUS_FAILED);
+            }
+            num_extra = "\02\03\07"[dist - 16]; TINFL_GET_BITS(18, s, num_extra); s += "\03\03\013"[dist - 16];
+            TINFL_MEMSET(r->m_len_codes + counter, (dist == 16) ? r->m_len_codes[counter - 1] : 0, s); counter += s;
+          }
+          if ((r->m_table_sizes[0] + r->m_table_sizes[1]) != counter)
+          {
+            TINFL_CR_RETURN_FOREVER(21, TINFL_STATUS_FAILED);
+          }
+          TINFL_MEMCPY(r->m_tables[0].m_code_size, r->m_len_codes, r->m_table_sizes[0]); TINFL_MEMCPY(r->m_tables[1].m_code_size, r->m_len_codes + r->m_table_sizes[0], r->m_table_sizes[1]);
+        }
+      }
+      for ( ; ; )
+      {
+        mz_uint8 *pSrc;
+        for ( ; ; )
+        {
+          if (((pIn_buf_end - pIn_buf_cur) < 4) || ((pOut_buf_end - pOut_buf_cur) < 2))
+          {
+            TINFL_HUFF_DECODE(23, counter, &r->m_tables[0]);
+            if (counter >= 256)
+              break;
+            while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(24, TINFL_STATUS_HAS_MORE_OUTPUT); }
+            *pOut_buf_cur++ = (mz_uint8)counter;
+          }
+          else
+          {
+            int sym2; mz_uint code_len;
+#if TINFL_USE_64BIT_BITBUF
+            if (num_bits < 30) { bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE32(pIn_buf_cur)) << num_bits); pIn_buf_cur += 4; num_bits += 32; }
+#else
+            if (num_bits < 15) { bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits); pIn_buf_cur += 2; num_bits += 16; }
+#endif
+            if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0)
+              code_len = sym2 >> 9;
+            else
+            {
+              code_len = TINFL_FAST_LOOKUP_BITS; do { sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)]; } while (sym2 < 0);
+            }
+            counter = sym2; bit_buf >>= code_len; num_bits -= code_len;
+            if (counter & 256)
+              break;
+
+#if !TINFL_USE_64BIT_BITBUF
+            if (num_bits < 15) { bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits); pIn_buf_cur += 2; num_bits += 16; }
+#endif
+            if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0)
+              code_len = sym2 >> 9;
+            else
+            {
+              code_len = TINFL_FAST_LOOKUP_BITS; do { sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)]; } while (sym2 < 0);
+            }
+            bit_buf >>= code_len; num_bits -= code_len;
+
+            pOut_buf_cur[0] = (mz_uint8)counter;
+            if (sym2 & 256)
+            {
+              pOut_buf_cur++;
+              counter = sym2;
+              break;
+            }
+            pOut_buf_cur[1] = (mz_uint8)sym2;
+            pOut_buf_cur += 2;
+          }
+        }
+        if ((counter &= 511) == 256) break;
+
+        num_extra = s_length_extra[counter - 257]; counter = s_length_base[counter - 257];
+        if (num_extra) { mz_uint extra_bits; TINFL_GET_BITS(25, extra_bits, num_extra); counter += extra_bits; }
+
+        TINFL_HUFF_DECODE(26, dist, &r->m_tables[1]);
+        num_extra = s_dist_extra[dist]; dist = s_dist_base[dist];
+        if (num_extra) { mz_uint extra_bits; TINFL_GET_BITS(27, extra_bits, num_extra); dist += extra_bits; }
+
+        dist_from_out_buf_start = pOut_buf_cur - pOut_buf_start;
+        if ((dist > dist_from_out_buf_start) && (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF))
+        {
+          TINFL_CR_RETURN_FOREVER(37, TINFL_STATUS_FAILED);
+        }
+
+        pSrc = pOut_buf_start + ((dist_from_out_buf_start - dist) & out_buf_size_mask);
+
+        if ((MZ_MAX(pOut_buf_cur, pSrc) + counter) > pOut_buf_end)
+        {
+          while (counter--)
+          {
+            while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(53, TINFL_STATUS_HAS_MORE_OUTPUT); }
+            *pOut_buf_cur++ = pOut_buf_start[(dist_from_out_buf_start++ - dist) & out_buf_size_mask];
+          }
+          continue;
+        }
+#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES
+        else if ((counter >= 9) && (counter <= dist))
+        {
+          const mz_uint8 *pSrc_end = pSrc + (counter & ~7);
+          do
+          {
+            ((mz_uint32 *)pOut_buf_cur)[0] = ((const mz_uint32 *)pSrc)[0];
+            ((mz_uint32 *)pOut_buf_cur)[1] = ((const mz_uint32 *)pSrc)[1];
+            pOut_buf_cur += 8;
+          } while ((pSrc += 8) < pSrc_end);
+          if ((counter &= 7) < 3)
+          {
+            if (counter)
+            {
+              pOut_buf_cur[0] = pSrc[0];
+              if (counter > 1)
+                pOut_buf_cur[1] = pSrc[1];
+              pOut_buf_cur += counter;
+            }
+            continue;
+          }
+        }
+#endif
+        do
+        {
+          pOut_buf_cur[0] = pSrc[0];
+          pOut_buf_cur[1] = pSrc[1];
+          pOut_buf_cur[2] = pSrc[2];
+          pOut_buf_cur += 3; pSrc += 3;
+        } while ((int)(counter -= 3) > 2);
+        if ((int)counter > 0)
+        {
+          pOut_buf_cur[0] = pSrc[0];
+          if ((int)counter > 1)
+            pOut_buf_cur[1] = pSrc[1];
+          pOut_buf_cur += counter;
+        }
+      }
+    }
+  } while (!(r->m_final & 1));
+  if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER)
+  {
+    TINFL_SKIP_BITS(32, num_bits & 7); for (counter = 0; counter < 4; ++counter) { mz_uint s; if (num_bits) TINFL_GET_BITS(41, s, 8); else TINFL_GET_BYTE(42, s); r->m_z_adler32 = (r->m_z_adler32 << 8) | s; }
+  }
+  TINFL_CR_RETURN_FOREVER(34, TINFL_STATUS_DONE);
+  TINFL_CR_FINISH
+
+common_exit:
+  r->m_num_bits = num_bits; r->m_bit_buf = bit_buf; r->m_dist = dist; r->m_counter = counter; r->m_num_extra = num_extra; r->m_dist_from_out_buf_start = dist_from_out_buf_start;
+  *pIn_buf_size = pIn_buf_cur - pIn_buf_next; *pOut_buf_size = pOut_buf_cur - pOut_buf_next;
+  if ((decomp_flags & (TINFL_FLAG_PARSE_ZLIB_HEADER | TINFL_FLAG_COMPUTE_ADLER32)) && (status >= 0))
+  {
+    const mz_uint8 *ptr = pOut_buf_next; size_t buf_len = *pOut_buf_size;
+    mz_uint32 i, s1 = r->m_check_adler32 & 0xffff, s2 = r->m_check_adler32 >> 16; size_t block_len = buf_len % 5552;
+    while (buf_len)
+    {
+      for (i = 0; i + 7 < block_len; i += 8, ptr += 8)
+      {
+        s1 += ptr[0], s2 += s1; s1 += ptr[1], s2 += s1; s1 += ptr[2], s2 += s1; s1 += ptr[3], s2 += s1;
+        s1 += ptr[4], s2 += s1; s1 += ptr[5], s2 += s1; s1 += ptr[6], s2 += s1; s1 += ptr[7], s2 += s1;
+      }
+      for ( ; i < block_len; ++i) s1 += *ptr++, s2 += s1;
+      s1 %= 65521U, s2 %= 65521U; buf_len -= block_len; block_len = 5552;
+    }
+    r->m_check_adler32 = (s2 << 16) + s1; if ((status == TINFL_STATUS_DONE) && (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) && (r->m_check_adler32 != r->m_z_adler32)) status = TINFL_STATUS_ADLER32_MISMATCH;
+  }
+  return status;
+}
+
+// Higher level helper functions.
+void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags)
+{
+  tinfl_decompressor decomp; void *pBuf = NULL, *pNew_buf; size_t src_buf_ofs = 0, out_buf_capacity = 0;
+  *pOut_len = 0;
+  tinfl_init(&decomp);
+  for ( ; ; )
+  {
+    size_t src_buf_size = src_buf_len - src_buf_ofs, dst_buf_size = out_buf_capacity - *pOut_len, new_out_buf_capacity;
+    tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8*)pSrc_buf + src_buf_ofs, &src_buf_size, (mz_uint8*)pBuf, pBuf ? (mz_uint8*)pBuf + *pOut_len : NULL, &dst_buf_size,
+      (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);
+    if ((status < 0) || (status == TINFL_STATUS_NEEDS_MORE_INPUT))
+    {
+      MZ_FREE(pBuf); *pOut_len = 0; return NULL;
+    }
+    src_buf_ofs += src_buf_size;
+    *pOut_len += dst_buf_size;
+    if (status == TINFL_STATUS_DONE) break;
+    new_out_buf_capacity = out_buf_capacity * 2; if (new_out_buf_capacity < 128) new_out_buf_capacity = 128;
+    pNew_buf = MZ_REALLOC(pBuf, new_out_buf_capacity);
+    if (!pNew_buf)
+    {
+      MZ_FREE(pBuf); *pOut_len = 0; return NULL;
+    }
+    pBuf = pNew_buf; out_buf_capacity = new_out_buf_capacity;
+  }
+  return pBuf;
+}
+
+size_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags)
+{
+  tinfl_decompressor decomp; tinfl_status status; tinfl_init(&decomp);
+  status = tinfl_decompress(&decomp, (const mz_uint8*)pSrc_buf, &src_buf_len, (mz_uint8*)pOut_buf, (mz_uint8*)pOut_buf, &out_buf_len, (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);
+  return (status != TINFL_STATUS_DONE) ? TINFL_DECOMPRESS_MEM_TO_MEM_FAILED : out_buf_len;
+}
+
+int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags)
+{
+  int result = 0;
+  tinfl_decompressor decomp;
+  mz_uint8 *pDict = (mz_uint8*)MZ_MALLOC(TINFL_LZ_DICT_SIZE); size_t in_buf_ofs = 0, dict_ofs = 0;
+  if (!pDict)
+    return TINFL_STATUS_FAILED;
+  memset(pDict,0,TINFL_LZ_DICT_SIZE);
+  tinfl_init(&decomp);
+  for ( ; ; )
+  {
+    size_t in_buf_size = *pIn_buf_size - in_buf_ofs, dst_buf_size = TINFL_LZ_DICT_SIZE - dict_ofs;
+    tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8*)pIn_buf + in_buf_ofs, &in_buf_size, pDict, pDict + dict_ofs, &dst_buf_size,
+      (flags & ~(TINFL_FLAG_HAS_MORE_INPUT | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)));
+    in_buf_ofs += in_buf_size;
+    if ((dst_buf_size) && (!(*pPut_buf_func)(pDict + dict_ofs, (int)dst_buf_size, pPut_buf_user)))
+      break;
+    if (status != TINFL_STATUS_HAS_MORE_OUTPUT)
+    {
+      result = (status == TINFL_STATUS_DONE);
+      break;
+    }
+    dict_ofs = (dict_ofs + dst_buf_size) & (TINFL_LZ_DICT_SIZE - 1);
+  }
+  MZ_FREE(pDict);
+  *pIn_buf_size = in_buf_ofs;
+  return result;
+}
+#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/
+
+// ------------------- Low-level Compression (independent from all decompression API's)
+#ifndef MINIZ_NO_DEFLATE_APIS
+
+// Purposely making these tables static for faster init and thread safety.
+static const mz_uint16 s_tdefl_len_sym[256] = {
+  257,258,259,260,261,262,263,264,265,265,266,266,267,267,268,268,269,269,269,269,270,270,270,270,271,271,271,271,272,272,272,272,
+  273,273,273,273,273,273,273,273,274,274,274,274,274,274,274,274,275,275,275,275,275,275,275,275,276,276,276,276,276,276,276,276,
+  277,277,277,277,277,277,277,277,277,277,277,277,277,277,277,277,278,278,278,278,278,278,278,278,278,278,278,278,278,278,278,278,
+  279,279,279,279,279,279,279,279,279,279,279,279,279,279,279,279,280,280,280,280,280,280,280,280,280,280,280,280,280,280,280,280,
+  281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,
+  282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,
+  283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,
+  284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,285 };
+
+static const mz_uint8 s_tdefl_len_extra[256] = {
+  0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
+  4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+  5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
+  5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0 };
+
+static const mz_uint8 s_tdefl_small_dist_sym[512] = {
+  0,1,2,3,4,4,5,5,6,6,6,6,7,7,7,7,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,
+  11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,
+  13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,
+  14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,
+  14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,
+  15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,
+  16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
+  16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
+  16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
+  17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
+  17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
+  17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17 };
+
+static const mz_uint8 s_tdefl_small_dist_extra[512] = {
+  0,0,0,0,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,
+  5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7 };
+
+static const mz_uint8 s_tdefl_large_dist_sym[128] = {
+  0,0,18,19,20,20,21,21,22,22,22,22,23,23,23,23,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,
+  26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,
+  28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 };
+
+static const mz_uint8 s_tdefl_large_dist_extra[128] = {
+  0,0,8,8,9,9,9,9,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,
+  12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,
+  13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13 };
+
+// Radix sorts tdefl_sym_freq[] array by 16-bit key m_key. Returns ptr to sorted values.
+typedef struct { mz_uint16 m_key, m_sym_index; } tdefl_sym_freq;
+static tdefl_sym_freq* tdefl_radix_sort_syms(mz_uint num_syms, tdefl_sym_freq* pSyms0, tdefl_sym_freq* pSyms1)
+{
+  mz_uint32 total_passes = 2, pass_shift, pass, i, hist[256 * 2]; tdefl_sym_freq* pCur_syms = pSyms0, *pNew_syms = pSyms1; MZ_CLEAR_ARR(hist);
+  for (i = 0; i < num_syms; i++) { mz_uint freq = pSyms0[i].m_key; hist[freq & 0xFF]++; hist[256 + ((freq >> 8) & 0xFF)]++; }
+  while ((total_passes > 1) && (num_syms == hist[(total_passes - 1) * 256])) total_passes--;
+  for (pass_shift = 0, pass = 0; pass < total_passes; pass++, pass_shift += 8)
+  {
+    const mz_uint32* pHist = &hist[pass << 8];
+    mz_uint offsets[256], cur_ofs = 0;
+    for (i = 0; i < 256; i++) { offsets[i] = cur_ofs; cur_ofs += pHist[i]; }
+    for (i = 0; i < num_syms; i++) pNew_syms[offsets[(pCur_syms[i].m_key >> pass_shift) & 0xFF]++] = pCur_syms[i];
+    { tdefl_sym_freq* t = pCur_syms; pCur_syms = pNew_syms; pNew_syms = t; }
+  }
+  return pCur_syms;
+}
+
+// tdefl_calculate_minimum_redundancy() originally written by: Alistair Moffat, alistair@cs.mu.oz.au, Jyrki Katajainen, jyrki@diku.dk, November 1996.
+static void tdefl_calculate_minimum_redundancy(tdefl_sym_freq *A, int n)
+{
+  int root, leaf, next, avbl, used, dpth;
+  if (n==0) return; else if (n==1) { A[0].m_key = 1; return; }
+  A[0].m_key += A[1].m_key; root = 0; leaf = 2;
+  for (next=1; next < n-1; next++)
+  {
+    if (leaf>=n || A[root].m_key<A[leaf].m_key) { A[next].m_key = A[root].m_key; A[root++].m_key = (mz_uint16)next; } else A[next].m_key = A[leaf++].m_key;
+    if (leaf>=n || (root<next && A[root].m_key<A[leaf].m_key)) { A[next].m_key = (mz_uint16)(A[next].m_key + A[root].m_key); A[root++].m_key = (mz_uint16)next; } else A[next].m_key = (mz_uint16)(A[next].m_key + A[leaf++].m_key);
+  }
+  A[n-2].m_key = 0; for (next=n-3; next>=0; next--) A[next].m_key = A[A[next].m_key].m_key+1;
+  avbl = 1; used = dpth = 0; root = n-2; next = n-1;
+  while (avbl>0)
+  {
+    while (root>=0 && (int)A[root].m_key==dpth) { used++; root--; }
+    while (avbl>used) { A[next--].m_key = (mz_uint16)(dpth); avbl--; }
+    avbl = 2*used; dpth++; used = 0;
+  }
+}
+
+// Limits canonical Huffman code table's max code size.
+enum { TDEFL_MAX_SUPPORTED_HUFF_CODESIZE = 32 };
+static void tdefl_huffman_enforce_max_code_size(int *pNum_codes, int code_list_len, int max_code_size)
+{
+  int i; mz_uint32 total = 0; if (code_list_len <= 1) return;
+  for (i = max_code_size + 1; i <= TDEFL_MAX_SUPPORTED_HUFF_CODESIZE; i++) pNum_codes[max_code_size] += pNum_codes[i];
+  for (i = max_code_size; i > 0; i--) total += (((mz_uint32)pNum_codes[i]) << (max_code_size - i));
+  while (total != (1UL << max_code_size))
+  {
+    pNum_codes[max_code_size]--;
+    for (i = max_code_size - 1; i > 0; i--) if (pNum_codes[i]) { pNum_codes[i]--; pNum_codes[i + 1] += 2; break; }
+    total--;
+  }
+}
+
+static void tdefl_optimize_huffman_table(tdefl_compressor *d, int table_num, int table_len, int code_size_limit, int static_table)
+{
+  int i, j, l, num_codes[1 + TDEFL_MAX_SUPPORTED_HUFF_CODESIZE]; mz_uint next_code[TDEFL_MAX_SUPPORTED_HUFF_CODESIZE + 1]; MZ_CLEAR_ARR(num_codes);
+  if (static_table)
+  {
+    for (i = 0; i < table_len; i++) num_codes[d->m_huff_code_sizes[table_num][i]]++;
+  }
+  else
+  {
+    tdefl_sym_freq syms0[TDEFL_MAX_HUFF_SYMBOLS], syms1[TDEFL_MAX_HUFF_SYMBOLS], *pSyms;
+    int num_used_syms = 0;
+    const mz_uint16 *pSym_count = &d->m_huff_count[table_num][0];
+    for (i = 0; i < table_len; i++) if (pSym_count[i]) { syms0[num_used_syms].m_key = (mz_uint16)pSym_count[i]; syms0[num_used_syms++].m_sym_index = (mz_uint16)i; }
+
+    pSyms = tdefl_radix_sort_syms(num_used_syms, syms0, syms1); tdefl_calculate_minimum_redundancy(pSyms, num_used_syms);
+
+    for (i = 0; i < num_used_syms; i++) num_codes[pSyms[i].m_key]++;
+
+    tdefl_huffman_enforce_max_code_size(num_codes, num_used_syms, code_size_limit);
+
+    MZ_CLEAR_ARR(d->m_huff_code_sizes[table_num]); MZ_CLEAR_ARR(d->m_huff_codes[table_num]);
+    for (i = 1, j = num_used_syms; i <= code_size_limit; i++)
+      for (l = num_codes[i]; l > 0; l--) d->m_huff_code_sizes[table_num][pSyms[--j].m_sym_index] = (mz_uint8)(i);
+  }
+
+  next_code[1] = 0; for (j = 0, i = 2; i <= code_size_limit; i++) next_code[i] = j = ((j + num_codes[i - 1]) << 1);
+
+  for (i = 0; i < table_len; i++)
+  {
+    mz_uint rev_code = 0, code, code_size; if ((code_size = d->m_huff_code_sizes[table_num][i]) == 0) continue;
+    code = next_code[code_size]++; for (l = code_size; l > 0; l--, code >>= 1) rev_code = (rev_code << 1) | (code & 1);
+    d->m_huff_codes[table_num][i] = (mz_uint16)rev_code;
+  }
+}
+
+#define TDEFL_PUT_BITS(b, l) do { \
+  mz_uint bits = b; mz_uint len = l; MZ_ASSERT(bits <= ((1U << len) - 1U)); \
+  d->m_bit_buffer |= (bits << d->m_bits_in); d->m_bits_in += len; \
+  while (d->m_bits_in >= 8) { \
+    if (d->m_pOutput_buf < d->m_pOutput_buf_end) \
+      *d->m_pOutput_buf++ = (mz_uint8)(d->m_bit_buffer); \
+    d->m_bit_buffer >>= 8; \
+    d->m_bits_in -= 8; \
+  } \
+} MZ_MACRO_END
+
+#define TDEFL_RLE_PREV_CODE_SIZE() do { \
+  if (rle_repeat_count) { \
+    if (rle_repeat_count < 3) { \
+      d->m_huff_count[2][prev_code_size] = (mz_uint16)(d->m_huff_count[2][prev_code_size] + rle_repeat_count); \
+      while (rle_repeat_count--) packed_code_sizes[num_packed_code_sizes++] = prev_code_size; \
+    } else { \
+      d->m_huff_count[2][16] = (mz_uint16)(d->m_huff_count[2][16] + 1); packed_code_sizes[num_packed_code_sizes++] = 16; packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_repeat_count - 3); \
+  } rle_repeat_count = 0; } \
+} MZ_MACRO_END
+
+#define TDEFL_RLE_ZERO_CODE_SIZE() do { \
+  if (rle_z_count) { \
+    if (rle_z_count < 3) { \
+      d->m_huff_count[2][0] = (mz_uint16)(d->m_huff_count[2][0] + rle_z_count); while (rle_z_count--) packed_code_sizes[num_packed_code_sizes++] = 0; \
+    } else if (rle_z_count <= 10) { \
+      d->m_huff_count[2][17] = (mz_uint16)(d->m_huff_count[2][17] + 1); packed_code_sizes[num_packed_code_sizes++] = 17; packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_z_count - 3); \
+    } else { \
+      d->m_huff_count[2][18] = (mz_uint16)(d->m_huff_count[2][18] + 1); packed_code_sizes[num_packed_code_sizes++] = 18; packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_z_count - 11); \
+  } rle_z_count = 0; } \
+} MZ_MACRO_END
+
+static mz_uint8 s_tdefl_packed_code_size_syms_swizzle[] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
+
+static void tdefl_start_dynamic_block(tdefl_compressor *d)
+{
+  int num_lit_codes, num_dist_codes, num_bit_lengths; mz_uint i, total_code_sizes_to_pack, num_packed_code_sizes, rle_z_count, rle_repeat_count, packed_code_sizes_index;
+  mz_uint8 code_sizes_to_pack[TDEFL_MAX_HUFF_SYMBOLS_0 + TDEFL_MAX_HUFF_SYMBOLS_1], packed_code_sizes[TDEFL_MAX_HUFF_SYMBOLS_0 + TDEFL_MAX_HUFF_SYMBOLS_1], prev_code_size = 0xFF;
+
+  d->m_huff_count[0][256] = 1;
+
+  tdefl_optimize_huffman_table(d, 0, TDEFL_MAX_HUFF_SYMBOLS_0, 15, MZ_FALSE);
+  tdefl_optimize_huffman_table(d, 1, TDEFL_MAX_HUFF_SYMBOLS_1, 15, MZ_FALSE);
+
+  for (num_lit_codes = 286; num_lit_codes > 257; num_lit_codes--) if (d->m_huff_code_sizes[0][num_lit_codes - 1]) break;
+  for (num_dist_codes = 30; num_dist_codes > 1; num_dist_codes--) if (d->m_huff_code_sizes[1][num_dist_codes - 1]) break;
+
+  memcpy(code_sizes_to_pack, &d->m_huff_code_sizes[0][0], num_lit_codes);
+  memcpy(code_sizes_to_pack + num_lit_codes, &d->m_huff_code_sizes[1][0], num_dist_codes);
+  total_code_sizes_to_pack = num_lit_codes + num_dist_codes; num_packed_code_sizes = 0; rle_z_count = 0; rle_repeat_count = 0;
+
+  memset(&d->m_huff_count[2][0], 0, sizeof(d->m_huff_count[2][0]) * TDEFL_MAX_HUFF_SYMBOLS_2);
+  for (i = 0; i < total_code_sizes_to_pack; i++)
+  {
+    mz_uint8 code_size = code_sizes_to_pack[i];
+    if (!code_size)
+    {
+      TDEFL_RLE_PREV_CODE_SIZE();
+      if (++rle_z_count == 138) { TDEFL_RLE_ZERO_CODE_SIZE(); }
+    }
+    else
+    {
+      TDEFL_RLE_ZERO_CODE_SIZE();
+      if (code_size != prev_code_size)
+      {
+        TDEFL_RLE_PREV_CODE_SIZE();
+        d->m_huff_count[2][code_size] = (mz_uint16)(d->m_huff_count[2][code_size] + 1); packed_code_sizes[num_packed_code_sizes++] = code_size;
+      }
+      else if (++rle_repeat_count == 6)
+      {
+        TDEFL_RLE_PREV_CODE_SIZE();
+      }
+    }
+    prev_code_size = code_size;
+  }
+  if (rle_repeat_count) { TDEFL_RLE_PREV_CODE_SIZE(); } else { TDEFL_RLE_ZERO_CODE_SIZE(); }
+
+  tdefl_optimize_huffman_table(d, 2, TDEFL_MAX_HUFF_SYMBOLS_2, 7, MZ_FALSE);
+
+  TDEFL_PUT_BITS(2, 2);
+
+  TDEFL_PUT_BITS(num_lit_codes - 257, 5);
+  TDEFL_PUT_BITS(num_dist_codes - 1, 5);
+
+  for (num_bit_lengths = 18; num_bit_lengths >= 0; num_bit_lengths--) if (d->m_huff_code_sizes[2][s_tdefl_packed_code_size_syms_swizzle[num_bit_lengths]]) break;
+  num_bit_lengths = MZ_MAX(4, (num_bit_lengths + 1)); TDEFL_PUT_BITS(num_bit_lengths - 4, 4);
+  for (i = 0; (int)i < num_bit_lengths; i++) TDEFL_PUT_BITS(d->m_huff_code_sizes[2][s_tdefl_packed_code_size_syms_swizzle[i]], 3);
+
+  for (packed_code_sizes_index = 0; packed_code_sizes_index < num_packed_code_sizes; )
+  {
+    mz_uint code = packed_code_sizes[packed_code_sizes_index++]; MZ_ASSERT(code < TDEFL_MAX_HUFF_SYMBOLS_2);
+    TDEFL_PUT_BITS(d->m_huff_codes[2][code], d->m_huff_code_sizes[2][code]);
+    if (code >= 16) TDEFL_PUT_BITS(packed_code_sizes[packed_code_sizes_index++], "\02\03\07"[code - 16]);
+  }
+}
+
+static void tdefl_start_static_block(tdefl_compressor *d)
+{
+  mz_uint i;
+  mz_uint8 *p = &d->m_huff_code_sizes[0][0];
+
+  for (i = 0; i <= 143; ++i) *p++ = 8;
+  for ( ; i <= 255; ++i) *p++ = 9;
+  for ( ; i <= 279; ++i) *p++ = 7;
+  for ( ; i <= 287; ++i) *p++ = 8;
+
+  memset(d->m_huff_code_sizes[1], 5, 32);
+
+  tdefl_optimize_huffman_table(d, 0, 288, 15, MZ_TRUE);
+  tdefl_optimize_huffman_table(d, 1, 32, 15, MZ_TRUE);
+
+  TDEFL_PUT_BITS(1, 2);
+}
+
+static const mz_uint mz_bitmasks[17] = { 0x0000, 0x0001, 0x0003, 0x0007, 0x000F, 0x001F, 0x003F, 0x007F, 0x00FF, 0x01FF, 0x03FF, 0x07FF, 0x0FFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF };
+
+#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN && MINIZ_HAS_64BIT_REGISTERS
+static mz_bool tdefl_compress_lz_codes(tdefl_compressor *d)
+{
+  mz_uint flags;
+  mz_uint8 *pLZ_codes;
+  mz_uint8 *pOutput_buf = d->m_pOutput_buf;
+  mz_uint8 *pLZ_code_buf_end = d->m_pLZ_code_buf;
+  mz_uint64 bit_buffer = d->m_bit_buffer;
+  mz_uint bits_in = d->m_bits_in;
+
+#define TDEFL_PUT_BITS_FAST(b, l) { bit_buffer |= (((mz_uint64)(b)) << bits_in); bits_in += (l); }
+
+  flags = 1;
+  for (pLZ_codes = d->m_lz_code_buf; pLZ_codes < pLZ_code_buf_end; flags >>= 1)
+  {
+    if (flags == 1)
+      flags = *pLZ_codes++ | 0x100;
+
+    if (flags & 1)
+    {
+      mz_uint s0, s1, n0, n1, sym, num_extra_bits;
+      mz_uint match_len = pLZ_codes[0], match_dist = *(const mz_uint16 *)(pLZ_codes + 1); pLZ_codes += 3;
+
+      MZ_ASSERT(d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);
+      TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][s_tdefl_len_sym[match_len]], d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);
+      TDEFL_PUT_BITS_FAST(match_len & mz_bitmasks[s_tdefl_len_extra[match_len]], s_tdefl_len_extra[match_len]);
+
+      // This sequence coaxes MSVC into using cmov's vs. jmp's.
+      s0 = s_tdefl_small_dist_sym[match_dist & 511];
+      n0 = s_tdefl_small_dist_extra[match_dist & 511];
+      s1 = s_tdefl_large_dist_sym[match_dist >> 8];
+      n1 = s_tdefl_large_dist_extra[match_dist >> 8];
+      sym = (match_dist < 512) ? s0 : s1;
+      num_extra_bits = (match_dist < 512) ? n0 : n1;
+
+      MZ_ASSERT(d->m_huff_code_sizes[1][sym]);
+      TDEFL_PUT_BITS_FAST(d->m_huff_codes[1][sym], d->m_huff_code_sizes[1][sym]);
+      TDEFL_PUT_BITS_FAST(match_dist & mz_bitmasks[num_extra_bits], num_extra_bits);
+    }
+    else
+    {
+      mz_uint lit = *pLZ_codes++;
+      MZ_ASSERT(d->m_huff_code_sizes[0][lit]);
+      TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);
+
+      if (((flags & 2) == 0) && (pLZ_codes < pLZ_code_buf_end))
+      {
+        flags >>= 1;
+        lit = *pLZ_codes++;
+        MZ_ASSERT(d->m_huff_code_sizes[0][lit]);
+        TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);
+
+        if (((flags & 2) == 0) && (pLZ_codes < pLZ_code_buf_end))
+        {
+          flags >>= 1;
+          lit = *pLZ_codes++;
+          MZ_ASSERT(d->m_huff_code_sizes[0][lit]);
+          TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);
+        }
+      }
+    }
+
+    if (pOutput_buf >= d->m_pOutput_buf_end)
+      return MZ_FALSE;
+
+    *(mz_uint64*)pOutput_buf = bit_buffer;
+    pOutput_buf += (bits_in >> 3);
+    bit_buffer >>= (bits_in & ~7);
+    bits_in &= 7;
+  }
+
+#undef TDEFL_PUT_BITS_FAST
+
+  d->m_pOutput_buf = pOutput_buf;
+  d->m_bits_in = 0;
+  d->m_bit_buffer = 0;
+
+  while (bits_in)
+  {
+    mz_uint32 n = MZ_MIN(bits_in, 16);
+    TDEFL_PUT_BITS((mz_uint)bit_buffer & mz_bitmasks[n], n);
+    bit_buffer >>= n;
+    bits_in -= n;
+  }
+
+  TDEFL_PUT_BITS(d->m_huff_codes[0][256], d->m_huff_code_sizes[0][256]);
+
+  return (d->m_pOutput_buf < d->m_pOutput_buf_end);
+}
+#else
+static mz_bool tdefl_compress_lz_codes(tdefl_compressor *d)
+{
+  mz_uint flags;
+  mz_uint8 *pLZ_codes;
+
+  flags = 1;
+  for (pLZ_codes = d->m_lz_code_buf; pLZ_codes < d->m_pLZ_code_buf; flags >>= 1)
+  {
+    if (flags == 1)
+      flags = *pLZ_codes++ | 0x100;
+    if (flags & 1)
+    {
+      mz_uint sym, num_extra_bits;
+      mz_uint match_len = pLZ_codes[0], match_dist = (pLZ_codes[1] | (pLZ_codes[2] << 8)); pLZ_codes += 3;
+
+      MZ_ASSERT(d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);
+      TDEFL_PUT_BITS(d->m_huff_codes[0][s_tdefl_len_sym[match_len]], d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);
+      TDEFL_PUT_BITS(match_len & mz_bitmasks[s_tdefl_len_extra[match_len]], s_tdefl_len_extra[match_len]);
+
+      if (match_dist < 512)
+      {
+        sym = s_tdefl_small_dist_sym[match_dist]; num_extra_bits = s_tdefl_small_dist_extra[match_dist];
+      }
+      else
+      {
+        sym = s_tdefl_large_dist_sym[match_dist >> 8]; num_extra_bits = s_tdefl_large_dist_extra[match_dist >> 8];
+      }
+      MZ_ASSERT(d->m_huff_code_sizes[1][sym]);
+      TDEFL_PUT_BITS(d->m_huff_codes[1][sym], d->m_huff_code_sizes[1][sym]);
+      TDEFL_PUT_BITS(match_dist & mz_bitmasks[num_extra_bits], num_extra_bits);
+    }
+    else
+    {
+      mz_uint lit = *pLZ_codes++;
+      MZ_ASSERT(d->m_huff_code_sizes[0][lit]);
+      TDEFL_PUT_BITS(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);
+    }
+  }
+
+  TDEFL_PUT_BITS(d->m_huff_codes[0][256], d->m_huff_code_sizes[0][256]);
+
+  return (d->m_pOutput_buf < d->m_pOutput_buf_end);
+}
+#endif // MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN && MINIZ_HAS_64BIT_REGISTERS
+
+static mz_bool tdefl_compress_block(tdefl_compressor *d, mz_bool static_block)
+{
+  if (static_block)
+    tdefl_start_static_block(d);
+  else
+    tdefl_start_dynamic_block(d);
+  return tdefl_compress_lz_codes(d);
+}
+
+static int tdefl_flush_block(tdefl_compressor *d, int flush)
+{
+  mz_uint saved_bit_buf, saved_bits_in;
+  mz_uint8 *pSaved_output_buf;
+  mz_bool comp_block_succeeded = MZ_FALSE;
+  int n, use_raw_block = ((d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS) != 0) && (d->m_lookahead_pos - d->m_lz_code_buf_dict_pos) <= d->m_dict_size;
+  mz_uint8 *pOutput_buf_start = ((d->m_pPut_buf_func == NULL) && ((*d->m_pOut_buf_size - d->m_out_buf_ofs) >= TDEFL_OUT_BUF_SIZE)) ? ((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs) : d->m_output_buf;
+
+  d->m_pOutput_buf = pOutput_buf_start;
+  d->m_pOutput_buf_end = d->m_pOutput_buf + TDEFL_OUT_BUF_SIZE - 16;
+
+  MZ_ASSERT(!d->m_output_flush_remaining);
+  d->m_output_flush_ofs = 0;
+  d->m_output_flush_remaining = 0;
+
+  *d->m_pLZ_flags = (mz_uint8)(*d->m_pLZ_flags >> d->m_num_flags_left);
+  d->m_pLZ_code_buf -= (d->m_num_flags_left == 8);
+
+  if ((d->m_flags & TDEFL_WRITE_ZLIB_HEADER) && (!d->m_block_index))
+  {
+    TDEFL_PUT_BITS(0x78, 8); TDEFL_PUT_BITS(0x01, 8);
+  }
+
+  TDEFL_PUT_BITS(flush == TDEFL_FINISH, 1);
+
+  pSaved_output_buf = d->m_pOutput_buf; saved_bit_buf = d->m_bit_buffer; saved_bits_in = d->m_bits_in;
+
+  if (!use_raw_block)
+    comp_block_succeeded = tdefl_compress_block(d, (d->m_flags & TDEFL_FORCE_ALL_STATIC_BLOCKS) || (d->m_total_lz_bytes < 48));
+
+  // If the block gets expanded, forget the current contents of the output buffer and send a raw block instead.
+  if ( ((use_raw_block) || ((d->m_total_lz_bytes) && ((d->m_pOutput_buf - pSaved_output_buf + 1U) >= d->m_total_lz_bytes))) &&
+       ((d->m_lookahead_pos - d->m_lz_code_buf_dict_pos) <= d->m_dict_size) )
+  {
+    mz_uint i; d->m_pOutput_buf = pSaved_output_buf; d->m_bit_buffer = saved_bit_buf, d->m_bits_in = saved_bits_in;
+    TDEFL_PUT_BITS(0, 2);
+    if (d->m_bits_in) { TDEFL_PUT_BITS(0, 8 - d->m_bits_in); }
+    for (i = 2; i; --i, d->m_total_lz_bytes ^= 0xFFFF)
+    {
+      TDEFL_PUT_BITS(d->m_total_lz_bytes & 0xFFFF, 16);
+    }
+    for (i = 0; i < d->m_total_lz_bytes; ++i)
+    {
+      TDEFL_PUT_BITS(d->m_dict[(d->m_lz_code_buf_dict_pos + i) & TDEFL_LZ_DICT_SIZE_MASK], 8);
+    }
+  }
+  // Check for the extremely unlikely (if not impossible) case of the compressed block not fitting into the output buffer when using dynamic codes.
+  else if (!comp_block_succeeded)
+  {
+    d->m_pOutput_buf = pSaved_output_buf; d->m_bit_buffer = saved_bit_buf, d->m_bits_in = saved_bits_in;
+    tdefl_compress_block(d, MZ_TRUE);
+  }
+
+  if (flush)
+  {
+    if (flush == TDEFL_FINISH)
+    {
+      if (d->m_bits_in) { TDEFL_PUT_BITS(0, 8 - d->m_bits_in); }
+      if (d->m_flags & TDEFL_WRITE_ZLIB_HEADER) { mz_uint i, a = d->m_adler32; for (i = 0; i < 4; i++) { TDEFL_PUT_BITS((a >> 24) & 0xFF, 8); a <<= 8; } }
+    }
+    else
+    {
+      mz_uint i, z = 0; TDEFL_PUT_BITS(0, 3); if (d->m_bits_in) { TDEFL_PUT_BITS(0, 8 - d->m_bits_in); } for (i = 2; i; --i, z ^= 0xFFFF) { TDEFL_PUT_BITS(z & 0xFFFF, 16); }
+    }
+  }
+
+  MZ_ASSERT(d->m_pOutput_buf < d->m_pOutput_buf_end);
+
+  memset(&d->m_huff_count[0][0], 0, sizeof(d->m_huff_count[0][0]) * TDEFL_MAX_HUFF_SYMBOLS_0);
+  memset(&d->m_huff_count[1][0], 0, sizeof(d->m_huff_count[1][0]) * TDEFL_MAX_HUFF_SYMBOLS_1);
+
+  d->m_pLZ_code_buf = d->m_lz_code_buf + 1; d->m_pLZ_flags = d->m_lz_code_buf; d->m_num_flags_left = 8; d->m_lz_code_buf_dict_pos += d->m_total_lz_bytes; d->m_total_lz_bytes = 0; d->m_block_index++;
+
+  if ((n = (int)(d->m_pOutput_buf - pOutput_buf_start)) != 0)
+  {
+    if (d->m_pPut_buf_func)
+    {
+      *d->m_pIn_buf_size = d->m_pSrc - (const mz_uint8 *)d->m_pIn_buf;
+      if (!(*d->m_pPut_buf_func)(d->m_output_buf, n, d->m_pPut_buf_user))
+        return (d->m_prev_return_status = TDEFL_STATUS_PUT_BUF_FAILED);
+    }
+    else if (pOutput_buf_start == d->m_output_buf)
+    {
+      int bytes_to_copy = (int)MZ_MIN((size_t)n, (size_t)(*d->m_pOut_buf_size - d->m_out_buf_ofs));
+      memcpy((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs, d->m_output_buf, bytes_to_copy);
+      d->m_out_buf_ofs += bytes_to_copy;
+      if ((n -= bytes_to_copy) != 0)
+      {
+        d->m_output_flush_ofs = bytes_to_copy;
+        d->m_output_flush_remaining = n;
+      }
+    }
+    else
+    {
+      d->m_out_buf_ofs += n;
+    }
+  }
+
+  return d->m_output_flush_remaining;
+}
+
+#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES
+#define TDEFL_READ_UNALIGNED_WORD(p) *(const mz_uint16*)(p)
+static MZ_FORCEINLINE void tdefl_find_match(tdefl_compressor *d, mz_uint lookahead_pos, mz_uint max_dist, mz_uint max_match_len, mz_uint *pMatch_dist, mz_uint *pMatch_len)
+{
+  mz_uint dist, pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK, match_len = *pMatch_len, probe_pos = pos, next_probe_pos, probe_len;
+  mz_uint num_probes_left = d->m_max_probes[match_len >= 32];
+  const mz_uint16 *s = (const mz_uint16*)(d->m_dict + pos), *p, *q;
+  mz_uint16 c01 = TDEFL_READ_UNALIGNED_WORD(&d->m_dict[pos + match_len - 1]), s01 = TDEFL_READ_UNALIGNED_WORD(s);
+  MZ_ASSERT(max_match_len <= TDEFL_MAX_MATCH_LEN); if (max_match_len <= match_len) return;
+  for ( ; ; )
+  {
+    for ( ; ; )
+    {
+      if (--num_probes_left == 0) return;
+      #define TDEFL_PROBE \
+        next_probe_pos = d->m_next[probe_pos]; \
+        if ((!next_probe_pos) || ((dist = (mz_uint16)(lookahead_pos - next_probe_pos)) > max_dist)) return; \
+        probe_pos = next_probe_pos & TDEFL_LZ_DICT_SIZE_MASK; \
+        if (TDEFL_READ_UNALIGNED_WORD(&d->m_dict[probe_pos + match_len - 1]) == c01) break;
+      TDEFL_PROBE
+      TDEFL_PROBE
+      TDEFL_PROBE
+    }
+    if (!dist) break;
+    q = (const mz_uint16*)(d->m_dict + probe_pos);
+    if (TDEFL_READ_UNALIGNED_WORD(q) != s01) continue;
+    p = s; probe_len = 32;
+    do { } while ( (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) &&
+                   (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (--probe_len > 0) );
+    if (!probe_len)
+    {
+      *pMatch_dist = dist; *pMatch_len = MZ_MIN(max_match_len, TDEFL_MAX_MATCH_LEN); break;
+    }
+    else if ((probe_len = ((mz_uint)(p - s) * 2) + (mz_uint)(*(const mz_uint8*)p == *(const mz_uint8*)q)) > match_len)
+    {
+      *pMatch_dist = dist; if ((*pMatch_len = match_len = MZ_MIN(max_match_len, probe_len)) == max_match_len) break;
+      c01 = TDEFL_READ_UNALIGNED_WORD(&d->m_dict[pos + match_len - 1]);
+    }
+  }
+}
+#else
+static MZ_FORCEINLINE void tdefl_find_match(tdefl_compressor *d, mz_uint lookahead_pos, mz_uint max_dist, mz_uint max_match_len, mz_uint *pMatch_dist, mz_uint *pMatch_len)
+{
+  mz_uint dist, pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK, match_len = *pMatch_len, probe_pos = pos, next_probe_pos, probe_len;
+  mz_uint num_probes_left = d->m_max_probes[match_len >= 32];
+  const mz_uint8 *s = d->m_dict + pos, *p, *q;
+  mz_uint8 c0 = d->m_dict[pos + match_len], c1 = d->m_dict[pos + match_len - 1];
+  MZ_ASSERT(max_match_len <= TDEFL_MAX_MATCH_LEN); if (max_match_len <= match_len) return;
+  for ( ; ; )
+  {
+    for ( ; ; )
+    {
+      if (--num_probes_left == 0) return;
+      #define TDEFL_PROBE \
+        next_probe_pos = d->m_next[probe_pos]; \
+        if ((!next_probe_pos) || ((dist = (mz_uint16)(lookahead_pos - next_probe_pos)) > max_dist)) return; \
+        probe_pos = next_probe_pos & TDEFL_LZ_DICT_SIZE_MASK; \
+        if ((d->m_dict[probe_pos + match_len] == c0) && (d->m_dict[probe_pos + match_len - 1] == c1)) break;
+      TDEFL_PROBE
+      TDEFL_PROBE
+      TDEFL_PROBE
+    }
+    if (!dist) break;
+    p = s; q = d->m_dict + probe_pos;
+    for (probe_len = 0; probe_len < max_match_len; probe_len++) {
+        if (*p++ != *q++) break;
+    }
+    if (probe_len > match_len)
+    {
+      *pMatch_dist = dist;
+      if ((*pMatch_len = match_len = probe_len) == max_match_len) return;
+      c0 = d->m_dict[pos + match_len]; c1 = d->m_dict[pos + match_len - 1];
+    }
+  }
+}
+#endif // #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES
+
+#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN
+static mz_bool tdefl_compress_fast(tdefl_compressor *d)
+{
+  // Faster, minimally featured LZRW1-style match+parse loop with better register utilization. Intended for applications where raw throughput is valued more highly than ratio.
+  mz_uint lookahead_pos = d->m_lookahead_pos, lookahead_size = d->m_lookahead_size, dict_size = d->m_dict_size, total_lz_bytes = d->m_total_lz_bytes, num_flags_left = d->m_num_flags_left;
+  mz_uint8 *pLZ_code_buf = d->m_pLZ_code_buf, *pLZ_flags = d->m_pLZ_flags;
+  mz_uint cur_pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK;
+
+  while ((d->m_src_buf_left) || ((d->m_flush) && (lookahead_size)))
+  {
+    const mz_uint TDEFL_COMP_FAST_LOOKAHEAD_SIZE = 4096;
+    mz_uint dst_pos = (lookahead_pos + lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK;
+    mz_uint num_bytes_to_process = (mz_uint)MZ_MIN(d->m_src_buf_left, TDEFL_COMP_FAST_LOOKAHEAD_SIZE - lookahead_size);
+    d->m_src_buf_left -= num_bytes_to_process;
+    lookahead_size += num_bytes_to_process;
+
+    while (num_bytes_to_process)
+    {
+      mz_uint32 n = MZ_MIN(TDEFL_LZ_DICT_SIZE - dst_pos, num_bytes_to_process);
+      memcpy(d->m_dict + dst_pos, d->m_pSrc, n);
+      if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1))
+        memcpy(d->m_dict + TDEFL_LZ_DICT_SIZE + dst_pos, d->m_pSrc, MZ_MIN(n, (TDEFL_MAX_MATCH_LEN - 1) - dst_pos));
+      d->m_pSrc += n;
+      dst_pos = (dst_pos + n) & TDEFL_LZ_DICT_SIZE_MASK;
+      num_bytes_to_process -= n;
+    }
+
+    dict_size = MZ_MIN(TDEFL_LZ_DICT_SIZE - lookahead_size, dict_size);
+    if ((!d->m_flush) && (lookahead_size < TDEFL_COMP_FAST_LOOKAHEAD_SIZE)) break;
+
+    while (lookahead_size >= 4)
+    {
+      mz_uint cur_match_dist, cur_match_len = 1;
+      mz_uint8 *pCur_dict = d->m_dict + cur_pos;
+      mz_uint first_trigram = (*(const mz_uint32 *)pCur_dict) & 0xFFFFFF;
+      mz_uint hash = (first_trigram ^ (first_trigram >> (24 - (TDEFL_LZ_HASH_BITS - 8)))) & TDEFL_LEVEL1_HASH_SIZE_MASK;
+      mz_uint probe_pos = d->m_hash[hash];
+      d->m_hash[hash] = (mz_uint16)lookahead_pos;
+
+      if (((cur_match_dist = (mz_uint16)(lookahead_pos - probe_pos)) <= dict_size) && ((*(const mz_uint32 *)(d->m_dict + (probe_pos &= TDEFL_LZ_DICT_SIZE_MASK)) & 0xFFFFFF) == first_trigram))
+      {
+        const mz_uint16 *p = (const mz_uint16 *)pCur_dict;
+        const mz_uint16 *q = (const mz_uint16 *)(d->m_dict + probe_pos);
+        mz_uint32 probe_len = 32;
+        do { } while ( (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) &&
+          (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (--probe_len > 0) );
+        cur_match_len = ((mz_uint)(p - (const mz_uint16 *)pCur_dict) * 2) + (mz_uint)(*(const mz_uint8 *)p == *(const mz_uint8 *)q);
+        if (!probe_len)
+          cur_match_len = cur_match_dist ? TDEFL_MAX_MATCH_LEN : 0;
+
+        if ((cur_match_len < TDEFL_MIN_MATCH_LEN) || ((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U*1024U)))
+        {
+          cur_match_len = 1;
+          *pLZ_code_buf++ = (mz_uint8)first_trigram;
+          *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1);
+          d->m_huff_count[0][(mz_uint8)first_trigram]++;
+        }
+        else
+        {
+          mz_uint32 s0, s1;
+          cur_match_len = MZ_MIN(cur_match_len, lookahead_size);
+
+          MZ_ASSERT((cur_match_len >= TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 1) && (cur_match_dist <= TDEFL_LZ_DICT_SIZE));
+
+          cur_match_dist--;
+
+          pLZ_code_buf[0] = (mz_uint8)(cur_match_len - TDEFL_MIN_MATCH_LEN);
+          *(mz_uint16 *)(&pLZ_code_buf[1]) = (mz_uint16)cur_match_dist;
+          pLZ_code_buf += 3;
+          *pLZ_flags = (mz_uint8)((*pLZ_flags >> 1) | 0x80);
+
+          s0 = s_tdefl_small_dist_sym[cur_match_dist & 511];
+          s1 = s_tdefl_large_dist_sym[cur_match_dist >> 8];
+          d->m_huff_count[1][(cur_match_dist < 512) ? s0 : s1]++;
+
+          d->m_huff_count[0][s_tdefl_len_sym[cur_match_len - TDEFL_MIN_MATCH_LEN]]++;
+        }
+      }
+      else
+      {
+        *pLZ_code_buf++ = (mz_uint8)first_trigram;
+        *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1);
+        d->m_huff_count[0][(mz_uint8)first_trigram]++;
+      }
+
+      if (--num_flags_left == 0) { num_flags_left = 8; pLZ_flags = pLZ_code_buf++; }
+
+      total_lz_bytes += cur_match_len;
+      lookahead_pos += cur_match_len;
+      dict_size = MZ_MIN(dict_size + cur_match_len, TDEFL_LZ_DICT_SIZE);
+      cur_pos = (cur_pos + cur_match_len) & TDEFL_LZ_DICT_SIZE_MASK;
+      MZ_ASSERT(lookahead_size >= cur_match_len);
+      lookahead_size -= cur_match_len;
+
+      if (pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8])
+      {
+        int n;
+        d->m_lookahead_pos = lookahead_pos; d->m_lookahead_size = lookahead_size; d->m_dict_size = dict_size;
+        d->m_total_lz_bytes = total_lz_bytes; d->m_pLZ_code_buf = pLZ_code_buf; d->m_pLZ_flags = pLZ_flags; d->m_num_flags_left = num_flags_left;
+        if ((n = tdefl_flush_block(d, 0)) != 0)
+          return (n < 0) ? MZ_FALSE : MZ_TRUE;
+        total_lz_bytes = d->m_total_lz_bytes; pLZ_code_buf = d->m_pLZ_code_buf; pLZ_flags = d->m_pLZ_flags; num_flags_left = d->m_num_flags_left;
+      }
+    }
+
+    while (lookahead_size)
+    {
+      mz_uint8 lit = d->m_dict[cur_pos];
+
+      total_lz_bytes++;
+      *pLZ_code_buf++ = lit;
+      *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1);
+      if (--num_flags_left == 0) { num_flags_left = 8; pLZ_flags = pLZ_code_buf++; }
+
+      d->m_huff_count[0][lit]++;
+
+      lookahead_pos++;
+      dict_size = MZ_MIN(dict_size + 1, TDEFL_LZ_DICT_SIZE);
+      cur_pos = (cur_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK;
+      lookahead_size--;
+
+      if (pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8])
+      {
+        int n;
+        d->m_lookahead_pos = lookahead_pos; d->m_lookahead_size = lookahead_size; d->m_dict_size = dict_size;
+        d->m_total_lz_bytes = total_lz_bytes; d->m_pLZ_code_buf = pLZ_code_buf; d->m_pLZ_flags = pLZ_flags; d->m_num_flags_left = num_flags_left;
+        if ((n = tdefl_flush_block(d, 0)) != 0)
+          return (n < 0) ? MZ_FALSE : MZ_TRUE;
+        total_lz_bytes = d->m_total_lz_bytes; pLZ_code_buf = d->m_pLZ_code_buf; pLZ_flags = d->m_pLZ_flags; num_flags_left = d->m_num_flags_left;
+      }
+    }
+  }
+
+  d->m_lookahead_pos = lookahead_pos; d->m_lookahead_size = lookahead_size; d->m_dict_size = dict_size;
+  d->m_total_lz_bytes = total_lz_bytes; d->m_pLZ_code_buf = pLZ_code_buf; d->m_pLZ_flags = pLZ_flags; d->m_num_flags_left = num_flags_left;
+  return MZ_TRUE;
+}
+#endif // MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN
+
+static MZ_FORCEINLINE void tdefl_record_literal(tdefl_compressor *d, mz_uint8 lit)
+{
+  d->m_total_lz_bytes++;
+  *d->m_pLZ_code_buf++ = lit;
+  *d->m_pLZ_flags = (mz_uint8)(*d->m_pLZ_flags >> 1); if (--d->m_num_flags_left == 0) { d->m_num_flags_left = 8; d->m_pLZ_flags = d->m_pLZ_code_buf++; }
+  d->m_huff_count[0][lit]++;
+}
+
+static MZ_FORCEINLINE void tdefl_record_match(tdefl_compressor *d, mz_uint match_len, mz_uint match_dist)
+{
+  mz_uint32 s0, s1;
+
+  MZ_ASSERT((match_len >= TDEFL_MIN_MATCH_LEN) && (match_dist >= 1) && (match_dist <= TDEFL_LZ_DICT_SIZE));
+
+  d->m_total_lz_bytes += match_len;
+
+  d->m_pLZ_code_buf[0] = (mz_uint8)(match_len - TDEFL_MIN_MATCH_LEN);
+
+  match_dist -= 1;
+  d->m_pLZ_code_buf[1] = (mz_uint8)(match_dist & 0xFF);
+  d->m_pLZ_code_buf[2] = (mz_uint8)(match_dist >> 8); d->m_pLZ_code_buf += 3;
+
+  *d->m_pLZ_flags = (mz_uint8)((*d->m_pLZ_flags >> 1) | 0x80); if (--d->m_num_flags_left == 0) { d->m_num_flags_left = 8; d->m_pLZ_flags = d->m_pLZ_code_buf++; }
+
+  s0 = s_tdefl_small_dist_sym[match_dist & 511]; s1 = s_tdefl_large_dist_sym[(match_dist >> 8) & 127];
+  d->m_huff_count[1][(match_dist < 512) ? s0 : s1]++;
+
+  if (match_len >= TDEFL_MIN_MATCH_LEN) d->m_huff_count[0][s_tdefl_len_sym[match_len - TDEFL_MIN_MATCH_LEN]]++;
+}
+
+static mz_bool tdefl_compress_normal(tdefl_compressor *d)
+{
+  const mz_uint8 *pSrc = d->m_pSrc; size_t src_buf_left = d->m_src_buf_left;
+  tdefl_flush flush = d->m_flush;
+
+  while ((src_buf_left) || ((flush) && (d->m_lookahead_size)))
+  {
+    mz_uint len_to_move, cur_match_dist, cur_match_len, cur_pos;
+    // Update dictionary and hash chains. Keeps the lookahead size equal to TDEFL_MAX_MATCH_LEN.
+    if ((d->m_lookahead_size + d->m_dict_size) >= (TDEFL_MIN_MATCH_LEN - 1))
+    {
+      mz_uint dst_pos = (d->m_lookahead_pos + d->m_lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK, ins_pos = d->m_lookahead_pos + d->m_lookahead_size - 2;
+      mz_uint hash = (d->m_dict[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] << TDEFL_LZ_HASH_SHIFT) ^ d->m_dict[(ins_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK];
+      mz_uint num_bytes_to_process = (mz_uint)MZ_MIN(src_buf_left, TDEFL_MAX_MATCH_LEN - d->m_lookahead_size);
+      const mz_uint8 *pSrc_end = pSrc ? pSrc + num_bytes_to_process : NULL;
+      src_buf_left -= num_bytes_to_process;
+      d->m_lookahead_size += num_bytes_to_process;
+      while (pSrc != pSrc_end)
+      {
+        mz_uint8 c = *pSrc++; d->m_dict[dst_pos] = c; if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1)) d->m_dict[TDEFL_LZ_DICT_SIZE + dst_pos] = c;
+        hash = ((hash << TDEFL_LZ_HASH_SHIFT) ^ c) & (TDEFL_LZ_HASH_SIZE - 1);
+        d->m_next[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] = d->m_hash[hash]; d->m_hash[hash] = (mz_uint16)(ins_pos);
+        dst_pos = (dst_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK; ins_pos++;
+      }
+    }
+    else
+    {
+      while ((src_buf_left) && (d->m_lookahead_size < TDEFL_MAX_MATCH_LEN))
+      {
+        mz_uint8 c = *pSrc++;
+        mz_uint dst_pos = (d->m_lookahead_pos + d->m_lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK;
+        src_buf_left--;
+        d->m_dict[dst_pos] = c;
+        if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1))
+          d->m_dict[TDEFL_LZ_DICT_SIZE + dst_pos] = c;
+        if ((++d->m_lookahead_size + d->m_dict_size) >= TDEFL_MIN_MATCH_LEN)
+        {
+          mz_uint ins_pos = d->m_lookahead_pos + (d->m_lookahead_size - 1) - 2;
+          mz_uint hash = ((d->m_dict[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] << (TDEFL_LZ_HASH_SHIFT * 2)) ^ (d->m_dict[(ins_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK] << TDEFL_LZ_HASH_SHIFT) ^ c) & (TDEFL_LZ_HASH_SIZE - 1);
+          d->m_next[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] = d->m_hash[hash]; d->m_hash[hash] = (mz_uint16)(ins_pos);
+        }
+      }
+    }
+    d->m_dict_size = MZ_MIN(TDEFL_LZ_DICT_SIZE - d->m_lookahead_size, d->m_dict_size);
+    if ((!flush) && (d->m_lookahead_size < TDEFL_MAX_MATCH_LEN))
+      break;
+
+    // Simple lazy/greedy parsing state machine.
+    len_to_move = 1; cur_match_dist = 0; cur_match_len = d->m_saved_match_len ? d->m_saved_match_len : (TDEFL_MIN_MATCH_LEN - 1); cur_pos = d->m_lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK;
+    if (d->m_flags & (TDEFL_RLE_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS))
+    {
+      if ((d->m_dict_size) && (!(d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS)))
+      {
+        mz_uint8 c = d->m_dict[(cur_pos - 1) & TDEFL_LZ_DICT_SIZE_MASK];
+        cur_match_len = 0; while (cur_match_len < d->m_lookahead_size) { if (d->m_dict[cur_pos + cur_match_len] != c) break; cur_match_len++; }
+        if (cur_match_len < TDEFL_MIN_MATCH_LEN) cur_match_len = 0; else cur_match_dist = 1;
+      }
+    }
+    else
+    {
+      tdefl_find_match(d, d->m_lookahead_pos, d->m_dict_size, d->m_lookahead_size, &cur_match_dist, &cur_match_len);
+    }
+    if (((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U*1024U)) || (cur_pos == cur_match_dist) || ((d->m_flags & TDEFL_FILTER_MATCHES) && (cur_match_len <= 5)))
+    {
+      cur_match_dist = cur_match_len = 0;
+    }
+    if (d->m_saved_match_len)
+    {
+      if (cur_match_len > d->m_saved_match_len)
+      {
+        tdefl_record_literal(d, (mz_uint8)d->m_saved_lit);
+        if (cur_match_len >= 128)
+        {
+          tdefl_record_match(d, cur_match_len, cur_match_dist);
+          d->m_saved_match_len = 0; len_to_move = cur_match_len;
+        }
+        else
+        {
+          d->m_saved_lit = d->m_dict[cur_pos]; d->m_saved_match_dist = cur_match_dist; d->m_saved_match_len = cur_match_len;
+        }
+      }
+      else
+      {
+        tdefl_record_match(d, d->m_saved_match_len, d->m_saved_match_dist);
+        len_to_move = d->m_saved_match_len - 1; d->m_saved_match_len = 0;
+      }
+    }
+    else if (!cur_match_dist)
+      tdefl_record_literal(d, d->m_dict[MZ_MIN(cur_pos, sizeof(d->m_dict) - 1)]);
+    else if ((d->m_greedy_parsing) || (d->m_flags & TDEFL_RLE_MATCHES) || (cur_match_len >= 128))
+    {
+      tdefl_record_match(d, cur_match_len, cur_match_dist);
+      len_to_move = cur_match_len;
+    }
+    else
+    {
+      d->m_saved_lit = d->m_dict[MZ_MIN(cur_pos, sizeof(d->m_dict) - 1)]; d->m_saved_match_dist = cur_match_dist; d->m_saved_match_len = cur_match_len;
+    }
+    // Move the lookahead forward by len_to_move bytes.
+    d->m_lookahead_pos += len_to_move;
+    MZ_ASSERT(d->m_lookahead_size >= len_to_move);
+    d->m_lookahead_size -= len_to_move;
+    d->m_dict_size = MZ_MIN(d->m_dict_size + len_to_move, TDEFL_LZ_DICT_SIZE);
+    // Check if it's time to flush the current LZ codes to the internal output buffer.
+    if ( (d->m_pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8]) ||
+         ( (d->m_total_lz_bytes > 31*1024) && (((((mz_uint)(d->m_pLZ_code_buf - d->m_lz_code_buf) * 115) >> 7) >= d->m_total_lz_bytes) || (d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS))) )
+    {
+      int n;
+      d->m_pSrc = pSrc; d->m_src_buf_left = src_buf_left;
+      if ((n = tdefl_flush_block(d, 0)) != 0)
+        return (n < 0) ? MZ_FALSE : MZ_TRUE;
+    }
+  }
+
+  d->m_pSrc = pSrc; d->m_src_buf_left = src_buf_left;
+  return MZ_TRUE;
+}
+
+static tdefl_status tdefl_flush_output_buffer(tdefl_compressor *d)
+{
+  if (d->m_pIn_buf_size)
+  {
+    *d->m_pIn_buf_size = d->m_pSrc - (const mz_uint8 *)d->m_pIn_buf;
+  }
+
+  if (d->m_pOut_buf_size)
+  {
+    size_t n = MZ_MIN(*d->m_pOut_buf_size - d->m_out_buf_ofs, d->m_output_flush_remaining);
+    memcpy((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs, d->m_output_buf + d->m_output_flush_ofs, n);
+    d->m_output_flush_ofs += (mz_uint)n;
+    d->m_output_flush_remaining -= (mz_uint)n;
+    d->m_out_buf_ofs += n;
+
+    *d->m_pOut_buf_size = d->m_out_buf_ofs;
+  }
+
+  return (d->m_finished && !d->m_output_flush_remaining) ? TDEFL_STATUS_DONE : TDEFL_STATUS_OKAY;
+}
+
+tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush)
+{
+  if (!d)
+  {
+    if (pIn_buf_size) *pIn_buf_size = 0;
+    if (pOut_buf_size) *pOut_buf_size = 0;
+    return TDEFL_STATUS_BAD_PARAM;
+  }
+
+  d->m_pIn_buf = pIn_buf; d->m_pIn_buf_size = pIn_buf_size;
+  d->m_pOut_buf = pOut_buf; d->m_pOut_buf_size = pOut_buf_size;
+  d->m_pSrc = (const mz_uint8 *)(pIn_buf); d->m_src_buf_left = pIn_buf_size ? *pIn_buf_size : 0;
+  d->m_out_buf_ofs = 0;
+  d->m_flush = flush;
+
+  if ( ((d->m_pPut_buf_func != NULL) == ((pOut_buf != NULL) || (pOut_buf_size != NULL))) || (d->m_prev_return_status != TDEFL_STATUS_OKAY) ||
+        (d->m_wants_to_finish && (flush != TDEFL_FINISH)) || (pIn_buf_size && *pIn_buf_size && !pIn_buf) || (pOut_buf_size && *pOut_buf_size && !pOut_buf) )
+  {
+    if (pIn_buf_size) *pIn_buf_size = 0;
+    if (pOut_buf_size) *pOut_buf_size = 0;
+    return (d->m_prev_return_status = TDEFL_STATUS_BAD_PARAM);
+  }
+  d->m_wants_to_finish |= (flush == TDEFL_FINISH);
+
+  if ((d->m_output_flush_remaining) || (d->m_finished))
+    return (d->m_prev_return_status = tdefl_flush_output_buffer(d));
+
+#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN
+  if (((d->m_flags & TDEFL_MAX_PROBES_MASK) == 1) &&
+      ((d->m_flags & TDEFL_GREEDY_PARSING_FLAG) != 0) &&
+      ((d->m_flags & (TDEFL_FILTER_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS | TDEFL_RLE_MATCHES)) == 0))
+  {
+    if (!tdefl_compress_fast(d))
+      return d->m_prev_return_status;
+  }
+  else
+#endif // #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN
+  {
+    if (!tdefl_compress_normal(d))
+      return d->m_prev_return_status;
+  }
+
+  if ((d->m_flags & (TDEFL_WRITE_ZLIB_HEADER | TDEFL_COMPUTE_ADLER32)) && (pIn_buf))
+    d->m_adler32 = (mz_uint32)mz_adler32(d->m_adler32, (const mz_uint8 *)pIn_buf, d->m_pSrc - (const mz_uint8 *)pIn_buf);
+
+  if ((flush) && (!d->m_lookahead_size) && (!d->m_src_buf_left) && (!d->m_output_flush_remaining))
+  {
+    if (tdefl_flush_block(d, flush) < 0)
+      return d->m_prev_return_status;
+    d->m_finished = (flush == TDEFL_FINISH);
+    if (flush == TDEFL_FULL_FLUSH) { MZ_CLEAR_ARR(d->m_hash); MZ_CLEAR_ARR(d->m_next); d->m_dict_size = 0; }
+  }
+
+  return (d->m_prev_return_status = tdefl_flush_output_buffer(d));
+}
+
+tdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush)
+{
+  MZ_ASSERT(d->m_pPut_buf_func); return tdefl_compress(d, pIn_buf, &in_buf_size, NULL, NULL, flush);
+}
+
+tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags)
+{
+  d->m_pPut_buf_func = pPut_buf_func; d->m_pPut_buf_user = pPut_buf_user;
+  d->m_flags = (mz_uint)(flags); d->m_max_probes[0] = 1 + ((flags & 0xFFF) + 2) / 3; d->m_greedy_parsing = (flags & TDEFL_GREEDY_PARSING_FLAG) != 0;
+  d->m_max_probes[1] = 1 + (((flags & 0xFFF) >> 2) + 2) / 3;
+  if (!(flags & TDEFL_NONDETERMINISTIC_PARSING_FLAG)) MZ_CLEAR_ARR(d->m_hash);
+  d->m_lookahead_pos = d->m_lookahead_size = d->m_dict_size = d->m_total_lz_bytes = d->m_lz_code_buf_dict_pos = d->m_bits_in = 0;
+  d->m_output_flush_ofs = d->m_output_flush_remaining = d->m_finished = d->m_block_index = d->m_bit_buffer = d->m_wants_to_finish = 0;
+  d->m_pLZ_code_buf = d->m_lz_code_buf + 1; d->m_pLZ_flags = d->m_lz_code_buf; d->m_num_flags_left = 8;
+  d->m_pOutput_buf = d->m_output_buf; d->m_pOutput_buf_end = d->m_output_buf; d->m_prev_return_status = TDEFL_STATUS_OKAY;
+  d->m_saved_match_dist = d->m_saved_match_len = d->m_saved_lit = 0; d->m_adler32 = 1;
+  d->m_pIn_buf = NULL; d->m_pOut_buf = NULL;
+  d->m_pIn_buf_size = NULL; d->m_pOut_buf_size = NULL;
+  d->m_flush = TDEFL_NO_FLUSH; d->m_pSrc = NULL; d->m_src_buf_left = 0; d->m_out_buf_ofs = 0;
+  memset(&d->m_huff_count[0][0], 0, sizeof(d->m_huff_count[0][0]) * TDEFL_MAX_HUFF_SYMBOLS_0);
+  memset(&d->m_huff_count[1][0], 0, sizeof(d->m_huff_count[1][0]) * TDEFL_MAX_HUFF_SYMBOLS_1);
+  return TDEFL_STATUS_OKAY;
+}
+
+#ifndef MINIZ_SDL_NOUNUSED
+tdefl_status tdefl_get_prev_return_status(tdefl_compressor *d)
+{
+  return d->m_prev_return_status;
+}
+
+mz_uint32 tdefl_get_adler32(tdefl_compressor *d)
+{
+  return d->m_adler32;
+}
+
+mz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags)
+{
+  tdefl_compressor *pComp; mz_bool succeeded; if (((buf_len) && (!pBuf)) || (!pPut_buf_func)) return MZ_FALSE;
+  pComp = (tdefl_compressor*)MZ_MALLOC(sizeof(tdefl_compressor)); if (!pComp) return MZ_FALSE;
+  succeeded = (tdefl_init(pComp, pPut_buf_func, pPut_buf_user, flags) == TDEFL_STATUS_OKAY);
+  succeeded = succeeded && (tdefl_compress_buffer(pComp, pBuf, buf_len, TDEFL_FINISH) == TDEFL_STATUS_DONE);
+  MZ_FREE(pComp); return succeeded;
+}
+#endif /* MINIZ_SDL_NOUNUSED */
+
+typedef struct
+{
+  size_t m_size, m_capacity;
+  mz_uint8 *m_pBuf;
+  mz_bool m_expandable;
+} tdefl_output_buffer;
+
+static mz_bool tdefl_output_buffer_putter(const void *pBuf, int len, void *pUser)
+{
+  tdefl_output_buffer *p = (tdefl_output_buffer *)pUser;
+  size_t new_size = p->m_size + len;
+  if (new_size > p->m_capacity)
+  {
+    size_t new_capacity = p->m_capacity; mz_uint8 *pNew_buf; if (!p->m_expandable) return MZ_FALSE;
+    do { new_capacity = MZ_MAX(128U, new_capacity << 1U); } while (new_size > new_capacity);
+    pNew_buf = (mz_uint8*)MZ_REALLOC(p->m_pBuf, new_capacity); if (!pNew_buf) return MZ_FALSE;
+    p->m_pBuf = pNew_buf; p->m_capacity = new_capacity;
+  }
+  memcpy((mz_uint8*)p->m_pBuf + p->m_size, pBuf, len); p->m_size = new_size;
+  return MZ_TRUE;
+}
+
+#ifndef MINIZ_SDL_NOUNUSED
+void *tdefl_compress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags)
+{
+  tdefl_output_buffer out_buf; MZ_CLEAR_OBJ(out_buf);
+  if (!pOut_len) return MZ_FALSE; else *pOut_len = 0;
+  out_buf.m_expandable = MZ_TRUE;
+  if (!tdefl_compress_mem_to_output(pSrc_buf, src_buf_len, tdefl_output_buffer_putter, &out_buf, flags)) return NULL;
+  *pOut_len = out_buf.m_size; return out_buf.m_pBuf;
+}
+
+size_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags)
+{
+  tdefl_output_buffer out_buf; MZ_CLEAR_OBJ(out_buf);
+  if (!pOut_buf) return 0;
+  out_buf.m_pBuf = (mz_uint8*)pOut_buf; out_buf.m_capacity = out_buf_len;
+  if (!tdefl_compress_mem_to_output(pSrc_buf, src_buf_len, tdefl_output_buffer_putter, &out_buf, flags)) return 0;
+  return out_buf.m_size;
+}
+#endif /* MINIZ_SDL_NOUNUSED */
+
+#ifndef MINIZ_NO_ZLIB_APIS
+static const mz_uint s_tdefl_num_probes[11] = { 0, 1, 6, 32,  16, 32, 128, 256,  512, 768, 1500 };
+
+// level may actually range from [0,10] (10 is a "hidden" max level, where we want a bit more compression and it's fine if throughput to fall off a cliff on some files).
+mz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy)
+{
+  mz_uint comp_flags = s_tdefl_num_probes[(level >= 0) ? MZ_MIN(10, level) : MZ_DEFAULT_LEVEL] | ((level <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0);
+  if (window_bits > 0) comp_flags |= TDEFL_WRITE_ZLIB_HEADER;
+
+  if (!level) comp_flags |= TDEFL_FORCE_ALL_RAW_BLOCKS;
+  else if (strategy == MZ_FILTERED) comp_flags |= TDEFL_FILTER_MATCHES;
+  else if (strategy == MZ_HUFFMAN_ONLY) comp_flags &= ~TDEFL_MAX_PROBES_MASK;
+  else if (strategy == MZ_FIXED) comp_flags |= TDEFL_FORCE_ALL_STATIC_BLOCKS;
+  else if (strategy == MZ_RLE) comp_flags |= TDEFL_RLE_MATCHES;
+
+  return comp_flags;
+}
+#endif //MINIZ_NO_ZLIB_APIS
+
+#ifdef _MSC_VER
+#pragma warning (push)
+#pragma warning (disable:4204) // nonstandard extension used : non-constant aggregate initializer (also supported by GNU C and C99, so no big deal)
+#endif
+
+// Simple PNG writer function by Alex Evans, 2011. Released into the public domain: https://gist.github.com/908299, more context at
+// http://altdevblogaday.org/2011/04/06/a-smaller-jpg-encoder/.
+// This is actually a modification of Alex's original code so PNG files generated by this function pass pngcheck.
+MINIZ_STATIC void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int h, int num_chans, int bpl, size_t *pLen_out, mz_uint level, mz_bool flip, mz_uint8 *plte, int plte_size, mz_uint8 *trns, int trns_size)
+{
+  // Using a local copy of this array here in case MINIZ_NO_ZLIB_APIS was defined.
+  static const mz_uint s_tdefl_png_num_probes[11] = { 0, 1, 6, 32,  16, 32, 128, 256,  512, 768, 1500 };
+  tdefl_compressor *pComp = (tdefl_compressor *)MZ_MALLOC(sizeof(tdefl_compressor)); tdefl_output_buffer out_buf; int i, y, z = 0; mz_uint32 c; *pLen_out = 0; size_t data_start, data_size;
+  if (!pComp) return NULL;
+  MZ_CLEAR_OBJ(out_buf); out_buf.m_expandable = MZ_TRUE;
+  out_buf.m_capacity = 57+MZ_MAX(64, (1+w*num_chans)*h);
+  if (plte_size > 0)
+    out_buf.m_capacity += 12+plte_size;
+  if (trns_size > 0)
+    out_buf.m_capacity += 12+trns_size;
+  if (NULL == (out_buf.m_pBuf = (mz_uint8*)MZ_MALLOC(out_buf.m_capacity))) { MZ_FREE(pComp); return NULL; }
+  // write header
+  {
+    static const mz_uint8 chans[] = {0x00, 0x00, 0x04, 0x02, 0x06};
+
+    mz_uint8 pnghdr[33]={ 0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a,
+                          0x00,0x00,0x00,0x0d,0x49,0x48,0x44,0x52,
+                          0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+                          0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};
+
+    pnghdr[18] = (mz_uint8)(w>>8);
+    pnghdr[19] = (mz_uint8)(w>>0);
+
+    pnghdr[22] = (mz_uint8)(h>>8);
+    pnghdr[23] = (mz_uint8)(h>>0);
+
+    if (num_chans == 1 && plte_size > 0)
+      pnghdr[25] = 3;
+    else
+      pnghdr[25] = (chans[num_chans]);
+
+    c = (mz_uint32)mz_crc32(MZ_CRC32_INIT,pnghdr+12,17);
+    for (i=0; i<4; ++i, c<<=8) ((mz_uint8*)(pnghdr+29))[i] = (mz_uint8)(c>>24);
+    if (!tdefl_output_buffer_putter(pnghdr, sizeof(pnghdr), &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+  }
+  // write PLTE chunk
+  if (plte_size > 0)
+  {
+    mz_uint8 hdr[8]={0x00,0x00,0x00,0x00,0x50,0x4c,0x54,0x45};
+
+    hdr[0] = (mz_uint8)(plte_size>>24);
+    hdr[1] = (mz_uint8)(plte_size>>16);
+    hdr[2] = (mz_uint8)(plte_size>> 8);
+    hdr[3] = (mz_uint8)(plte_size>> 0);
+    if (!tdefl_output_buffer_putter(hdr, sizeof(hdr), &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+    data_start = out_buf.m_size;
+    if (!tdefl_output_buffer_putter(plte, plte_size, &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+    if (!tdefl_output_buffer_putter("\0\0\0\0", 4, &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+    c = (mz_uint32)mz_crc32(MZ_CRC32_INIT,out_buf.m_pBuf+data_start-4, plte_size+4);
+    for (i=0; i<4; ++i, c<<=8) (out_buf.m_pBuf+out_buf.m_size-4)[i] = (mz_uint8)(c >> 24);
+  }
+  // write tRNS chunk
+  if (trns_size > 0)
+  {
+    mz_uint8 hdr[8]={0x00,0x00,0x00,0x00,0x74,0x52,0x4E,0x53};
+
+    hdr[0] = (mz_uint8)(trns_size>>24);
+    hdr[1] = (mz_uint8)(trns_size>>16);
+    hdr[2] = (mz_uint8)(trns_size>> 8);
+    hdr[3] = (mz_uint8)(trns_size>> 0);
+    if (!tdefl_output_buffer_putter(hdr, sizeof(hdr), &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+    data_start = out_buf.m_size;
+    if (!tdefl_output_buffer_putter(trns, trns_size, &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+    if (!tdefl_output_buffer_putter("\0\0\0\0", 4, &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+    c = (mz_uint32)mz_crc32(MZ_CRC32_INIT,out_buf.m_pBuf+data_start-4, trns_size+4);
+    for (i=0; i<4; ++i, c<<=8) (out_buf.m_pBuf+out_buf.m_size-4)[i] = (mz_uint8)(c >> 24);
+  }
+  // write IDAT chunk
+  if (!tdefl_output_buffer_putter("\0\0\0\0\x49\x44\x41\x54", 8, &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+  data_start = out_buf.m_size;
+  // compress image data
+  tdefl_init(pComp, tdefl_output_buffer_putter, &out_buf, s_tdefl_png_num_probes[MZ_MIN(10, level)] | TDEFL_WRITE_ZLIB_HEADER);
+  for (y = 0; y < h; ++y) { tdefl_compress_buffer(pComp, &z, 1, TDEFL_NO_FLUSH); tdefl_compress_buffer(pComp, (mz_uint8*)pImage + (flip ? (h - 1 - y) : y) * bpl, w * num_chans, TDEFL_NO_FLUSH); }
+  if (tdefl_compress_buffer(pComp, NULL, 0, TDEFL_FINISH) != TDEFL_STATUS_DONE) { MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+  // write IDAT size
+  data_size = out_buf.m_size-data_start;
+  (out_buf.m_pBuf+data_start-8)[0] = (mz_uint8)(data_size>>24);
+  (out_buf.m_pBuf+data_start-8)[1] = (mz_uint8)(data_size>>16);
+  (out_buf.m_pBuf+data_start-8)[2] = (mz_uint8)(data_size>> 8);
+  (out_buf.m_pBuf+data_start-8)[3] = (mz_uint8)(data_size>> 0);
+  // write footer (IDAT CRC-32, followed by IEND chunk)
+  if (!tdefl_output_buffer_putter("\0\0\0\0\0\0\0\0\x49\x45\x4e\x44\xae\x42\x60\x82", 16, &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; }
+  c = (mz_uint32)mz_crc32(MZ_CRC32_INIT,out_buf.m_pBuf+data_start-4, data_size+4);
+  for (i=0; i<4; ++i, c<<=8) (out_buf.m_pBuf+out_buf.m_size-16)[i] = (mz_uint8)(c >> 24);
+  // compute final size of file, grab compressed data buffer and return
+  *pLen_out = out_buf.m_size;
+  MZ_FREE(pComp);
+  return out_buf.m_pBuf;
+}
+#ifndef MINIZ_SDL_NOUNUSED
+MINIZ_STATIC void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, int bpl, size_t *pLen_out)
+{
+  // Level 6 corresponds to TDEFL_DEFAULT_MAX_PROBES or MZ_DEFAULT_LEVEL (but we can't depend on MZ_DEFAULT_LEVEL being available in case the zlib API's where #defined out)
+  return tdefl_write_image_to_png_file_in_memory_ex(pImage, w, h, num_chans, bpl, pLen_out, 6, MZ_FALSE, NULL, 0, NULL, 0);
+}
+#endif
+
+#ifdef _MSC_VER
+#pragma warning (pop)
+#endif
+#endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/
+
+// ------------------- .ZIP archive reading
+
+#ifndef MINIZ_NO_ARCHIVE_APIS
+
+#ifdef MINIZ_NO_STDIO
+  #define MZ_FILE void *
+#else
+  #include <stdio.h>
+  #include <sys/stat.h>
+
+  #if defined(_MSC_VER) || defined(__MINGW64__)
+    static FILE *mz_fopen(const char *pFilename, const char *pMode)
+    {
+      FILE* pFile = NULL;
+      fopen_s(&pFile, pFilename, pMode);
+      return pFile;
+    }
+    static FILE *mz_freopen(const char *pPath, const char *pMode, FILE *pStream)
+    {
+      FILE* pFile = NULL;
+      if (freopen_s(&pFile, pPath, pMode, pStream))
+        return NULL;
+      return pFile;
+    }
+    #ifndef MINIZ_NO_TIME
+      #include <sys/utime.h>
+    #endif
+    #define MZ_FILE FILE
+    #define MZ_FOPEN mz_fopen
+    #define MZ_FCLOSE fclose
+    #define MZ_FREAD fread
+    #define MZ_FWRITE fwrite
+    #define MZ_FTELL64 _ftelli64
+    #define MZ_FSEEK64 _fseeki64
+    #define MZ_FILE_STAT_STRUCT _stat
+    #define MZ_FILE_STAT _stat
+    #define MZ_FFLUSH fflush
+    #define MZ_FREOPEN mz_freopen
+    #define MZ_DELETE_FILE remove
+  #elif defined(__MINGW32__)
+    #ifndef MINIZ_NO_TIME
+      #include <sys/utime.h>
+    #endif
+    #define MZ_FILE FILE
+    #define MZ_FOPEN(f, m) fopen(f, m)
+    #define MZ_FCLOSE fclose
+    #define MZ_FREAD fread
+    #define MZ_FWRITE fwrite
+    #define MZ_FTELL64 ftello64
+    #define MZ_FSEEK64 fseeko64
+    #define MZ_FILE_STAT_STRUCT _stat
+    #define MZ_FILE_STAT _stat
+    #define MZ_FFLUSH fflush
+    #define MZ_FREOPEN(f, m, s) freopen(f, m, s)
+    #define MZ_DELETE_FILE remove
+  #elif defined(__TINYC__)
+    #ifndef MINIZ_NO_TIME
+      #include <sys/utime.h>
+    #endif
+    #define MZ_FILE FILE
+    #define MZ_FOPEN(f, m) fopen(f, m)
+    #define MZ_FCLOSE fclose
+    #define MZ_FREAD fread
+    #define MZ_FWRITE fwrite
+    #define MZ_FTELL64 ftell
+    #define MZ_FSEEK64 fseek
+    #define MZ_FILE_STAT_STRUCT stat
+    #define MZ_FILE_STAT stat
+    #define MZ_FFLUSH fflush
+    #define MZ_FREOPEN(f, m, s) freopen(f, m, s)
+    #define MZ_DELETE_FILE remove
+  #elif defined(__GNUC__) && _LARGEFILE64_SOURCE
+    #ifndef MINIZ_NO_TIME
+      #include <utime.h>
+    #endif
+    #define MZ_FILE FILE
+    #define MZ_FOPEN(f, m) fopen64(f, m)
+    #define MZ_FCLOSE fclose
+    #define MZ_FREAD fread
+    #define MZ_FWRITE fwrite
+    #define MZ_FTELL64 ftello64
+    #define MZ_FSEEK64 fseeko64
+    #define MZ_FILE_STAT_STRUCT stat64
+    #define MZ_FILE_STAT stat64
+    #define MZ_FFLUSH fflush
+    #define MZ_FREOPEN(p, m, s) freopen64(p, m, s)
+    #define MZ_DELETE_FILE remove
+  #else
+    #ifndef MINIZ_NO_TIME
+      #include <utime.h>
+    #endif
+    #define MZ_FILE FILE
+    #define MZ_FOPEN(f, m) fopen(f, m)
+    #define MZ_FCLOSE fclose
+    #define MZ_FREAD fread
+    #define MZ_FWRITE fwrite
+    #define MZ_FTELL64 ftello
+    #define MZ_FSEEK64 fseeko
+    #define MZ_FILE_STAT_STRUCT stat
+    #define MZ_FILE_STAT stat
+    #define MZ_FFLUSH fflush
+    #define MZ_FREOPEN(f, m, s) freopen(f, m, s)
+    #define MZ_DELETE_FILE remove
+  #endif // #ifdef _MSC_VER
+#endif // #ifdef MINIZ_NO_STDIO
+
+#define MZ_TOLOWER(c) ((((c) >= 'A') && ((c) <= 'Z')) ? ((c) - 'A' + 'a') : (c))
+
+// Various ZIP archive enums. To completely avoid cross platform compiler alignment and platform endian issues, miniz.c doesn't use structs for any of this stuff.
+enum
+{
+  // ZIP archive identifiers and record sizes
+  MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG = 0x06054b50, MZ_ZIP_CENTRAL_DIR_HEADER_SIG = 0x02014b50, MZ_ZIP_LOCAL_DIR_HEADER_SIG = 0x04034b50,
+  MZ_ZIP_LOCAL_DIR_HEADER_SIZE = 30, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE = 46, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE = 22,
+  // Central directory header record offsets
+  MZ_ZIP_CDH_SIG_OFS = 0, MZ_ZIP_CDH_VERSION_MADE_BY_OFS = 4, MZ_ZIP_CDH_VERSION_NEEDED_OFS = 6, MZ_ZIP_CDH_BIT_FLAG_OFS = 8,
+  MZ_ZIP_CDH_METHOD_OFS = 10, MZ_ZIP_CDH_FILE_TIME_OFS = 12, MZ_ZIP_CDH_FILE_DATE_OFS = 14, MZ_ZIP_CDH_CRC32_OFS = 16,
+  MZ_ZIP_CDH_COMPRESSED_SIZE_OFS = 20, MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS = 24, MZ_ZIP_CDH_FILENAME_LEN_OFS = 28, MZ_ZIP_CDH_EXTRA_LEN_OFS = 30,
+  MZ_ZIP_CDH_COMMENT_LEN_OFS = 32, MZ_ZIP_CDH_DISK_START_OFS = 34, MZ_ZIP_CDH_INTERNAL_ATTR_OFS = 36, MZ_ZIP_CDH_EXTERNAL_ATTR_OFS = 38, MZ_ZIP_CDH_LOCAL_HEADER_OFS = 42,
+  // Local directory header offsets
+  MZ_ZIP_LDH_SIG_OFS = 0, MZ_ZIP_LDH_VERSION_NEEDED_OFS = 4, MZ_ZIP_LDH_BIT_FLAG_OFS = 6, MZ_ZIP_LDH_METHOD_OFS = 8, MZ_ZIP_LDH_FILE_TIME_OFS = 10,
+  MZ_ZIP_LDH_FILE_DATE_OFS = 12, MZ_ZIP_LDH_CRC32_OFS = 14, MZ_ZIP_LDH_COMPRESSED_SIZE_OFS = 18, MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS = 22,
+  MZ_ZIP_LDH_FILENAME_LEN_OFS = 26, MZ_ZIP_LDH_EXTRA_LEN_OFS = 28,
+  // End of central directory offsets
+  MZ_ZIP_ECDH_SIG_OFS = 0, MZ_ZIP_ECDH_NUM_THIS_DISK_OFS = 4, MZ_ZIP_ECDH_NUM_DISK_CDIR_OFS = 6, MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS = 8,
+  MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS = 10, MZ_ZIP_ECDH_CDIR_SIZE_OFS = 12, MZ_ZIP_ECDH_CDIR_OFS_OFS = 16, MZ_ZIP_ECDH_COMMENT_SIZE_OFS = 20,
+};
+
+typedef struct
+{
+  void *m_p;
+  size_t m_size, m_capacity;
+  mz_uint m_element_size;
+} mz_zip_array;
+
+struct mz_zip_internal_state_tag
+{
+  mz_zip_array m_central_dir;
+  mz_zip_array m_central_dir_offsets;
+  mz_zip_array m_sorted_central_dir_offsets;
+  MZ_FILE *m_pFile;
+  void *m_pMem;
+  size_t m_mem_size;
+  size_t m_mem_capacity;
+};
+
+#define MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(array_ptr, element_size) (array_ptr)->m_element_size = element_size
+#define MZ_ZIP_ARRAY_ELEMENT(array_ptr, element_type, index) ((element_type *)((array_ptr)->m_p))[index]
+
+static MZ_FORCEINLINE void mz_zip_array_clear(mz_zip_archive *pZip, mz_zip_array *pArray)
+{
+  pZip->m_pFree(pZip->m_pAlloc_opaque, pArray->m_p);
+  memset(pArray, 0, sizeof(mz_zip_array));
+}
+
+static mz_bool mz_zip_array_ensure_capacity(mz_zip_archive *pZip, mz_zip_array *pArray, size_t min_new_capacity, mz_uint growing)
+{
+  void *pNew_p; size_t new_capacity = min_new_capacity; MZ_ASSERT(pArray->m_element_size); if (pArray->m_capacity >= min_new_capacity) return MZ_TRUE;
+  if (growing) { new_capacity = MZ_MAX(1, pArray->m_capacity); while (new_capacity < min_new_capacity) new_capacity *= 2; }
+  if (NULL == (pNew_p = pZip->m_pRealloc(pZip->m_pAlloc_opaque, pArray->m_p, pArray->m_element_size, new_capacity))) return MZ_FALSE;
+  pArray->m_p = pNew_p; pArray->m_capacity = new_capacity;
+  return MZ_TRUE;
+}
+
+static MZ_FORCEINLINE mz_bool mz_zip_array_reserve(mz_zip_archive *pZip, mz_zip_array *pArray, size_t new_capacity, mz_uint growing)
+{
+  if (new_capacity > pArray->m_capacity) { if (!mz_zip_array_ensure_capacity(pZip, pArray, new_capacity, growing)) return MZ_FALSE; }
+  return MZ_TRUE;
+}
+
+static MZ_FORCEINLINE mz_bool mz_zip_array_resize(mz_zip_archive *pZip, mz_zip_array *pArray, size_t new_size, mz_uint growing)
+{
+  if (new_size > pArray->m_capacity) { if (!mz_zip_array_ensure_capacity(pZip, pArray, new_size, growing)) return MZ_FALSE; }
+  pArray->m_size = new_size;
+  return MZ_TRUE;
+}
+
+static MZ_FORCEINLINE mz_bool mz_zip_array_ensure_room(mz_zip_archive *pZip, mz_zip_array *pArray, size_t n)
+{
+  return mz_zip_array_reserve(pZip, pArray, pArray->m_size + n, MZ_TRUE);
+}
+
+static MZ_FORCEINLINE mz_bool mz_zip_array_push_back(mz_zip_archive *pZip, mz_zip_array *pArray, const void *pElements, size_t n)
+{
+  size_t orig_size = pArray->m_size; if (!mz_zip_array_resize(pZip, pArray, orig_size + n, MZ_TRUE)) return MZ_FALSE;
+  memcpy((mz_uint8*)pArray->m_p + orig_size * pArray->m_element_size, pElements, n * pArray->m_element_size);
+  return MZ_TRUE;
+}
+
+#ifndef MINIZ_NO_TIME
+static time_t mz_zip_dos_to_time_t(int dos_time, int dos_date)
+{
+  struct tm tm;
+  memset(&tm, 0, sizeof(tm)); tm.tm_isdst = -1;
+  tm.tm_year = ((dos_date >> 9) & 127) + 1980 - 1900; tm.tm_mon = ((dos_date >> 5) & 15) - 1; tm.tm_mday = dos_date & 31;
+  tm.tm_hour = (dos_time >> 11) & 31; tm.tm_min = (dos_time >> 5) & 63; tm.tm_sec = (dos_time << 1) & 62;
+  return mktime(&tm);
+}
+
+static void mz_zip_time_to_dos_time(time_t time, mz_uint16 *pDOS_time, mz_uint16 *pDOS_date)
+{
+#ifdef _MSC_VER
+  struct tm tm_struct;
+  struct tm *tm = &tm_struct;
+  errno_t err = localtime_s(tm, &time);
+  if (err)
+  {
+    *pDOS_date = 0; *pDOS_time = 0;
+    return;
+  }
+#else
+  struct tm *tm = localtime(&time);
+#endif
+  *pDOS_time = (mz_uint16)(((tm->tm_hour) << 11) + ((tm->tm_min) << 5) + ((tm->tm_sec) >> 1));
+  *pDOS_date = (mz_uint16)(((tm->tm_year + 1900 - 1980) << 9) + ((tm->tm_mon + 1) << 5) + tm->tm_mday);
+}
+#endif
+
+#ifndef MINIZ_NO_STDIO
+static mz_bool mz_zip_get_file_modified_time(const char *pFilename, mz_uint16 *pDOS_time, mz_uint16 *pDOS_date)
+{
+#ifdef MINIZ_NO_TIME
+  (void)pFilename; *pDOS_date = *pDOS_time = 0;
+#else
+  struct MZ_FILE_STAT_STRUCT file_stat;
+  // On Linux with x86 glibc, this call will fail on large files (>= 0x80000000 bytes) unless you compiled with _LARGEFILE64_SOURCE. Argh.
+  if (MZ_FILE_STAT(pFilename, &file_stat) != 0)
+    return MZ_FALSE;
+  mz_zip_time_to_dos_time(file_stat.st_mtime, pDOS_time, pDOS_date);
+#endif // #ifdef MINIZ_NO_TIME
+  return MZ_TRUE;
+}
+
+#ifndef MINIZ_NO_TIME
+static mz_bool mz_zip_set_file_times(const char *pFilename, time_t access_time, time_t modified_time)
+{
+  struct utimbuf t; t.actime = access_time; t.modtime = modified_time;
+  return !utime(pFilename, &t);
+}
+#endif // #ifndef MINIZ_NO_TIME
+#endif // #ifndef MINIZ_NO_STDIO
+
+static mz_bool mz_zip_reader_init_internal(mz_zip_archive *pZip, mz_uint32 flags)
+{
+  (void)flags;
+  if ((!pZip) || (pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_INVALID))
+    return MZ_FALSE;
+
+  if (!pZip->m_pAlloc) pZip->m_pAlloc = def_alloc_func;
+  if (!pZip->m_pFree) pZip->m_pFree = def_free_func;
+  if (!pZip->m_pRealloc) pZip->m_pRealloc = def_realloc_func;
+
+  pZip->m_zip_mode = MZ_ZIP_MODE_READING;
+  pZip->m_archive_size = 0;
+  pZip->m_central_directory_file_ofs = 0;
+  pZip->m_total_files = 0;
+
+  if (NULL == (pZip->m_pState = (mz_zip_internal_state *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(mz_zip_internal_state))))
+    return MZ_FALSE;
+  memset(pZip->m_pState, 0, sizeof(mz_zip_internal_state));
+  MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir, sizeof(mz_uint8));
+  MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir_offsets, sizeof(mz_uint32));
+  MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_sorted_central_dir_offsets, sizeof(mz_uint32));
+  return MZ_TRUE;
+}
+
+static MZ_FORCEINLINE mz_bool mz_zip_reader_filename_less(const mz_zip_array *pCentral_dir_array, const mz_zip_array *pCentral_dir_offsets, mz_uint l_index, mz_uint r_index)
+{
+  const mz_uint8 *pL = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, l_index)), *pE;
+  const mz_uint8 *pR = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, r_index));
+  mz_uint l_len = MZ_READ_LE16(pL + MZ_ZIP_CDH_FILENAME_LEN_OFS), r_len = MZ_READ_LE16(pR + MZ_ZIP_CDH_FILENAME_LEN_OFS);
+  mz_uint8 l = 0, r = 0;
+  pL += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE; pR += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE;
+  pE = pL + MZ_MIN(l_len, r_len);
+  while (pL < pE)
+  {
+    if ((l = MZ_TOLOWER(*pL)) != (r = MZ_TOLOWER(*pR)))
+      break;
+    pL++; pR++;
+  }
+  return (pL == pE) ? (l_len < r_len) : (l < r);
+}
+
+#define MZ_SWAP_UINT32(a, b) do { mz_uint32 t = a; a = b; b = t; } MZ_MACRO_END
+
+// Heap sort of lowercased filenames, used to help accelerate plain central directory searches by mz_zip_reader_locate_file(). (Could also use qsort(), but it could allocate memory.)
+static void mz_zip_reader_sort_central_dir_offsets_by_filename(mz_zip_archive *pZip)
+{
+  mz_zip_internal_state *pState = pZip->m_pState;
+  const mz_zip_array *pCentral_dir_offsets = &pState->m_central_dir_offsets;
+  const mz_zip_array *pCentral_dir = &pState->m_central_dir;
+  mz_uint32 *pIndices = &MZ_ZIP_ARRAY_ELEMENT(&pState->m_sorted_central_dir_offsets, mz_uint32, 0);
+  const int size = pZip->m_total_files;
+  int start = (size - 2) >> 1, end;
+  while (start >= 0)
+  {
+    int child, root = start;
+    for ( ; ; )
+    {
+      if ((child = (root << 1) + 1) >= size)
+        break;
+      child += (((child + 1) < size) && (mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[child], pIndices[child + 1])));
+      if (!mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[root], pIndices[child]))
+        break;
+      MZ_SWAP_UINT32(pIndices[root], pIndices[child]); root = child;
+    }
+    start--;
+  }
+
+  end = size - 1;
+  while (end > 0)
+  {
+    int child, root = 0;
+    MZ_SWAP_UINT32(pIndices[end], pIndices[0]);
+    for ( ; ; )
+    {
+      if ((child = (root << 1) + 1) >= end)
+        break;
+      child += (((child + 1) < end) && mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[child], pIndices[child + 1]));
+      if (!mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[root], pIndices[child]))
+        break;
+      MZ_SWAP_UINT32(pIndices[root], pIndices[child]); root = child;
+    }
+    end--;
+  }
+}
+
+static mz_bool mz_zip_reader_read_central_dir(mz_zip_archive *pZip, mz_uint32 flags)
+{
+  mz_uint cdir_size, num_this_disk, cdir_disk_index;
+  mz_uint64 cdir_ofs;
+  mz_int64 cur_file_ofs;
+  const mz_uint8 *p;
+  mz_uint32 buf_u32[4096 / sizeof(mz_uint32)]; mz_uint8 *pBuf = (mz_uint8 *)buf_u32;
+  mz_bool sort_central_dir = ((flags & MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY) == 0);
+  // Basic sanity checks - reject files which are too small, and check the first 4 bytes of the file to make sure a local header is there.
+  if (pZip->m_archive_size < MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)
+    return MZ_FALSE;
+  // Find the end of central directory record by scanning the file from the end towards the beginning.
+  cur_file_ofs = MZ_MAX((mz_int64)pZip->m_archive_size - (mz_int64)sizeof(buf_u32), 0);
+  for ( ; ; )
+  {
+    int i, n = (int)MZ_MIN(sizeof(buf_u32), pZip->m_archive_size - cur_file_ofs);
+    if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, n) != (mz_uint)n)
+      return MZ_FALSE;
+    for (i = n - 4; i >= 0; --i)
+      if (MZ_READ_LE32(pBuf + i) == MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG)
+        break;
+    if (i >= 0)
+    {
+      cur_file_ofs += i;
+      break;
+    }
+    if ((!cur_file_ofs) || ((pZip->m_archive_size - cur_file_ofs) >= (0xFFFF + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)))
+      return MZ_FALSE;
+    cur_file_ofs = MZ_MAX(cur_file_ofs - (sizeof(buf_u32) - 3), 0);
+  }
+  // Read and verify the end of central directory record.
+  if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) != MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)
+    return MZ_FALSE;
+  if ((MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_SIG_OFS) != MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG) ||
+      ((pZip->m_total_files = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS)) != MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS)))
+    return MZ_FALSE;
+
+  num_this_disk = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_NUM_THIS_DISK_OFS);
+  cdir_disk_index = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_NUM_DISK_CDIR_OFS);
+  if (((num_this_disk | cdir_disk_index) != 0) && ((num_this_disk != 1) || (cdir_disk_index != 1)))
+    return MZ_FALSE;
+
+  if ((cdir_size = MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_CDIR_SIZE_OFS)) < pZip->m_total_files * MZ_ZIP_CENTRAL_DIR_HEADER_SIZE)
+    return MZ_FALSE;
+
+  cdir_ofs = MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_CDIR_OFS_OFS);
+  if ((cdir_ofs + (mz_uint64)cdir_size) > pZip->m_archive_size)
+    return MZ_FALSE;
+
+  pZip->m_central_directory_file_ofs = cdir_ofs;
+
+  if (pZip->m_total_files)
+  {
+     mz_uint i, n;
+
+    // Read the entire central directory into a heap block, and allocate another heap block to hold the unsorted central dir file record offsets, and another to hold the sorted indices.
+    if ((!mz_zip_array_resize(pZip, &pZip->m_pState->m_central_dir, cdir_size, MZ_FALSE)) ||
+        (!mz_zip_array_resize(pZip, &pZip->m_pState->m_central_dir_offsets, pZip->m_total_files, MZ_FALSE)))
+      return MZ_FALSE;
+
+    if (sort_central_dir)
+    {
+      if (!mz_zip_array_resize(pZip, &pZip->m_pState->m_sorted_central_dir_offsets, pZip->m_total_files, MZ_FALSE))
+        return MZ_FALSE;
+    }
+
+    if (pZip->m_pRead(pZip->m_pIO_opaque, cdir_ofs, pZip->m_pState->m_central_dir.m_p, cdir_size) != cdir_size)
+      return MZ_FALSE;
+
+    // Now create an index into the central directory file records, do some basic sanity checking on each record, and check for zip64 entries (which are not yet supported).
+    p = (const mz_uint8 *)pZip->m_pState->m_central_dir.m_p;
+    for (n = cdir_size, i = 0; i < pZip->m_total_files; ++i)
+    {
+      mz_uint total_header_size, comp_size, decomp_size, disk_index;
+      if ((n < MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) || (MZ_READ_LE32(p) != MZ_ZIP_CENTRAL_DIR_HEADER_SIG))
+        return MZ_FALSE;
+      MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, i) = (mz_uint32)(p - (const mz_uint8 *)pZip->m_pState->m_central_dir.m_p);
+      if (sort_central_dir)
+        MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_sorted_central_dir_offsets, mz_uint32, i) = i;
+      comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS);
+      decomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS);
+      if (((!MZ_READ_LE32(p + MZ_ZIP_CDH_METHOD_OFS)) && (decomp_size != comp_size)) || (decomp_size && !comp_size) || (decomp_size == 0xFFFFFFFF) || (comp_size == 0xFFFFFFFF))
+        return MZ_FALSE;
+      disk_index = MZ_READ_LE16(p + MZ_ZIP_CDH_DISK_START_OFS);
+      if ((disk_index != num_this_disk) && (disk_index != 1))
+        return MZ_FALSE;
+      if (((mz_uint64)MZ_READ_LE32(p + MZ_ZIP_CDH_LOCAL_HEADER_OFS) + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + comp_size) > pZip->m_archive_size)
+        return MZ_FALSE;
+      if ((total_header_size = MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_COMMENT_LEN_OFS)) > n)
+        return MZ_FALSE;
+      n -= total_header_size; p += total_header_size;
+    }
+  }
+
+  if (sort_central_dir)
+    mz_zip_reader_sort_central_dir_offsets_by_filename(pZip);
+
+  return MZ_TRUE;
+}
+
+mz_bool mz_zip_reader_init(mz_zip_archive *pZip, mz_uint64 size, mz_uint32 flags)
+{
+  if ((!pZip) || (!pZip->m_pRead))
+    return MZ_FALSE;
+  if (!mz_zip_reader_init_internal(pZip, flags))
+    return MZ_FALSE;
+  pZip->m_archive_size = size;
+  if (!mz_zip_reader_read_central_dir(pZip, flags))
+  {
+    mz_zip_reader_end(pZip);
+    return MZ_FALSE;
+  }
+  return MZ_TRUE;
+}
+
+static size_t mz_zip_mem_read_func(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n)
+{
+  mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;
+  size_t s = (file_ofs >= pZip->m_archive_size) ? 0 : (size_t)MZ_MIN(pZip->m_archive_size - file_ofs, n);
+  memcpy(pBuf, (const mz_uint8 *)pZip->m_pState->m_pMem + file_ofs, s);
+  return s;
+}
+
+mz_bool mz_zip_reader_init_mem(mz_zip_archive *pZip, const void *pMem, size_t size, mz_uint32 flags)
+{
+  if (!mz_zip_reader_init_internal(pZip, flags))
+    return MZ_FALSE;
+  pZip->m_archive_size = size;
+  pZip->m_pRead = mz_zip_mem_read_func;
+  pZip->m_pIO_opaque = pZip;
+#ifdef __cplusplus
+  pZip->m_pState->m_pMem = const_cast<void *>(pMem);
+#else
+  pZip->m_pState->m_pMem = (void *)pMem;
+#endif
+  pZip->m_pState->m_mem_size = size;
+  if (!mz_zip_reader_read_central_dir(pZip, flags))
+  {
+    mz_zip_reader_end(pZip);
+    return MZ_FALSE;
+  }
+  return MZ_TRUE;
+}
+
+#ifndef MINIZ_NO_STDIO
+static size_t mz_zip_file_read_func(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n)
+{
+  mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;
+  mz_int64 cur_ofs = MZ_FTELL64(pZip->m_pState->m_pFile);
+  if (((mz_int64)file_ofs < 0) || (((cur_ofs != (mz_int64)file_ofs)) && (MZ_FSEEK64(pZip->m_pState->m_pFile, (mz_int64)file_ofs, SEEK_SET))))
+    return 0;
+  return MZ_FREAD(pBuf, 1, n, pZip->m_pState->m_pFile);
+}
+
+mz_bool mz_zip_reader_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint32 flags)
+{
+  mz_uint64 file_size;
+  MZ_FILE *pFile = MZ_FOPEN(pFilename, "rb");
+  if (!pFile)
+    return MZ_FALSE;
+  if (MZ_FSEEK64(pFile, 0, SEEK_END))
+  {
+    MZ_FCLOSE(pFile);
+    return MZ_FALSE;
+  }
+  file_size = MZ_FTELL64(pFile);
+  if (!mz_zip_reader_init_internal(pZip, flags))
+  {
+    MZ_FCLOSE(pFile);
+    return MZ_FALSE;
+  }
+  pZip->m_pRead = mz_zip_file_read_func;
+  pZip->m_pIO_opaque = pZip;
+  pZip->m_pState->m_pFile = pFile;
+  pZip->m_archive_size = file_size;
+  if (!mz_zip_reader_read_central_dir(pZip, flags))
+  {
+    mz_zip_reader_end(pZip);
+    return MZ_FALSE;
+  }
+  return MZ_TRUE;
+}
+#endif // #ifndef MINIZ_NO_STDIO
+
+mz_uint mz_zip_reader_get_num_files(mz_zip_archive *pZip)
+{
+  return pZip ? pZip->m_total_files : 0;
+}
+
+static MZ_FORCEINLINE const mz_uint8 *mz_zip_reader_get_cdh(mz_zip_archive *pZip, mz_uint file_index)
+{
+  if ((!pZip) || (!pZip->m_pState) || (file_index >= pZip->m_total_files) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING))
+    return NULL;
+  return &MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index));
+}
+
+mz_bool mz_zip_reader_is_file_encrypted(mz_zip_archive *pZip, mz_uint file_index)
+{
+  mz_uint m_bit_flag;
+  const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index);
+  if (!p)
+    return MZ_FALSE;
+  m_bit_flag = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS);
+  return (m_bit_flag & 1);
+}
+
+mz_bool mz_zip_reader_is_file_a_directory(mz_zip_archive *pZip, mz_uint file_index)
+{
+  mz_uint filename_len, external_attr;
+  const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index);
+  if (!p)
+    return MZ_FALSE;
+
+  // First see if the filename ends with a '/' character.
+  filename_len = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS);
+  if (filename_len)
+  {
+    if (*(p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_len - 1) == '/')
+      return MZ_TRUE;
+  }
+
+  // Bugfix: This code was also checking if the internal attribute was non-zero, which wasn't correct.
+  // Most/all zip writers (hopefully) set DOS file/directory attributes in the low 16-bits, so check for the DOS directory flag and ignore the source OS ID in the created by field.
+  // FIXME: Remove this check? Is it necessary - we already check the filename.
+  external_attr = MZ_READ_LE32(p + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS);
+  if ((external_attr & 0x10) != 0)
+    return MZ_TRUE;
+
+  return MZ_FALSE;
+}
+
+mz_bool mz_zip_reader_file_stat(mz_zip_archive *pZip, mz_uint file_index, mz_zip_archive_file_stat *pStat)
+{
+  mz_uint n;
+  const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index);
+  if ((!p) || (!pStat))
+    return MZ_FALSE;
+
+  // Unpack the central directory record.
+  pStat->m_file_index = file_index;
+  pStat->m_central_dir_ofs = MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index);
+  pStat->m_version_made_by = MZ_READ_LE16(p + MZ_ZIP_CDH_VERSION_MADE_BY_OFS);
+  pStat->m_version_needed = MZ_READ_LE16(p + MZ_ZIP_CDH_VERSION_NEEDED_OFS);
+  pStat->m_bit_flag = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS);
+  pStat->m_method = MZ_READ_LE16(p + MZ_ZIP_CDH_METHOD_OFS);
+#ifndef MINIZ_NO_TIME
+  pStat->m_time = mz_zip_dos_to_time_t(MZ_READ_LE16(p + MZ_ZIP_CDH_FILE_TIME_OFS), MZ_READ_LE16(p + MZ_ZIP_CDH_FILE_DATE_OFS));
+#endif
+  pStat->m_crc32 = MZ_READ_LE32(p + MZ_ZIP_CDH_CRC32_OFS);
+  pStat->m_comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS);
+  pStat->m_uncomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS);
+  pStat->m_internal_attr = MZ_READ_LE16(p + MZ_ZIP_CDH_INTERNAL_ATTR_OFS);
+  pStat->m_external_attr = MZ_READ_LE32(p + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS);
+  pStat->m_local_header_ofs = MZ_READ_LE32(p + MZ_ZIP_CDH_LOCAL_HEADER_OFS);
+
+  // Copy as much of the filename and comment as possible.
+  n = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS); n = MZ_MIN(n, MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE - 1);
+  memcpy(pStat->m_filename, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n); pStat->m_filename[n] = '\0';
+
+  n = MZ_READ_LE16(p + MZ_ZIP_CDH_COMMENT_LEN_OFS); n = MZ_MIN(n, MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE - 1);
+  pStat->m_comment_size = n;
+  memcpy(pStat->m_comment, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS), n); pStat->m_comment[n] = '\0';
+
+  return MZ_TRUE;
+}
+
+mz_uint mz_zip_reader_get_filename(mz_zip_archive *pZip, mz_uint file_index, char *pFilename, mz_uint filename_buf_size)
+{
+  mz_uint n;
+  const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index);
+  if (!p) { if (filename_buf_size) pFilename[0] = '\0'; return 0; }
+  n = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS);
+  if (filename_buf_size)
+  {
+    n = MZ_MIN(n, filename_buf_size - 1);
+    memcpy(pFilename, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n);
+    pFilename[n] = '\0';
+  }
+  return n + 1;
+}
+
+static MZ_FORCEINLINE mz_bool mz_zip_reader_string_equal(const char *pA, const char *pB, mz_uint len, mz_uint flags)
+{
+  mz_uint i;
+  if (flags & MZ_ZIP_FLAG_CASE_SENSITIVE)
+    return 0 == memcmp(pA, pB, len);
+  for (i = 0; i < len; ++i)
+    if (MZ_TOLOWER(pA[i]) != MZ_TOLOWER(pB[i]))
+      return MZ_FALSE;
+  return MZ_TRUE;
+}
+
+static MZ_FORCEINLINE int mz_zip_reader_filename_compare(const mz_zip_array *pCentral_dir_array, const mz_zip_array *pCentral_dir_offsets, mz_uint l_index, const char *pR, mz_uint r_len)
+{
+  const mz_uint8 *pL = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, l_index)), *pE;
+  mz_uint l_len = MZ_READ_LE16(pL + MZ_ZIP_CDH_FILENAME_LEN_OFS);
+  mz_uint8 l = 0, r = 0;
+  pL += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE;
+  pE = pL + MZ_MIN(l_len, r_len);
+  while (pL < pE)
+  {
+    if ((l = MZ_TOLOWER(*pL)) != (r = MZ_TOLOWER(*pR)))
+      break;
+    pL++; pR++;
+  }
+  return (pL == pE) ? (int)(l_len - r_len) : (l - r);
+}
+
+static int mz_zip_reader_locate_file_binary_search(mz_zip_archive *pZip, const char *pFilename)
+{
+  mz_zip_internal_state *pState = pZip->m_pState;
+  const mz_zip_array *pCentral_dir_offsets = &pState->m_central_dir_offsets;
+  const mz_zip_array *pCentral_dir = &pState->m_central_dir;
+  mz_uint32 *pIndices = &MZ_ZIP_ARRAY_ELEMENT(&pState->m_sorted_central_dir_offsets, mz_uint32, 0);
+  const int size = pZip->m_total_files;
+  const mz_uint filename_len = (mz_uint)strlen(pFilename);
+  int l = 0, h = size - 1;
+  while (l <= h)
+  {
+    int m = (l + h) >> 1, file_index = pIndices[m], comp = mz_zip_reader_filename_compare(pCentral_dir, pCentral_dir_offsets, file_index, pFilename, filename_len);
+    if (!comp)
+      return file_index;
+    else if (comp < 0)
+      l = m + 1;
+    else
+      h = m - 1;
+  }
+  return -1;
+}
+
+int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags)
+{
+  mz_uint file_index; size_t name_len, comment_len;
+  if ((!pZip) || (!pZip->m_pState) || (!pName) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING))
+    return -1;
+  if (((flags & (MZ_ZIP_FLAG_IGNORE_PATH | MZ_ZIP_FLAG_CASE_SENSITIVE)) == 0) && (!pComment) && (pZip->m_pState->m_sorted_central_dir_offsets.m_size))
+    return mz_zip_reader_locate_file_binary_search(pZip, pName);
+  name_len = strlen(pName); if (name_len > 0xFFFF) return -1;
+  comment_len = pComment ? strlen(pComment) : 0; if (comment_len > 0xFFFF) return -1;
+  for (file_index = 0; file_index < pZip->m_total_files; file_index++)
+  {
+    const mz_uint8 *pHeader = &MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index));
+    mz_uint filename_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_FILENAME_LEN_OFS);
+    const char *pFilename = (const char *)pHeader + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE;
+    if (filename_len < name_len)
+      continue;
+    if (comment_len)
+    {
+      mz_uint file_extra_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_EXTRA_LEN_OFS), file_comment_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_COMMENT_LEN_OFS);
+      const char *pFile_comment = pFilename + filename_len + file_extra_len;
+      if ((file_comment_len != comment_len) || (!mz_zip_reader_string_equal(pComment, pFile_comment, file_comment_len, flags)))
+        continue;
+    }
+    if ((flags & MZ_ZIP_FLAG_IGNORE_PATH) && (filename_len))
+    {
+      int ofs = filename_len - 1;
+      do
+      {
+        if ((pFilename[ofs] == '/') || (pFilename[ofs] == '\\') || (pFilename[ofs] == ':'))
+          break;
+      } while (--ofs >= 0);
+      ofs++;
+      pFilename += ofs; filename_len -= ofs;
+    }
+    if ((filename_len == name_len) && (mz_zip_reader_string_equal(pName, pFilename, filename_len, flags)))
+      return file_index;
+  }
+  return -1;
+}
+
+mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size)
+{
+  int status = TINFL_STATUS_DONE;
+  mz_uint64 needed_size, cur_file_ofs, comp_remaining, out_buf_ofs = 0, read_buf_size, read_buf_ofs = 0, read_buf_avail;
+  mz_zip_archive_file_stat file_stat;
+  void *pRead_buf;
+  mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)]; mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;
+  tinfl_decompressor inflator;
+
+  if ((buf_size) && (!pBuf))
+    return MZ_FALSE;
+
+  if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat))
+    return MZ_FALSE;
+
+  // Empty file, or a directory (but not always a directory - I've seen odd zips with directories that have compressed data which inflates to 0 bytes)
+  if (!file_stat.m_comp_size)
+    return MZ_TRUE;
+
+  // Entry is a subdirectory (I've seen old zips with dir entries which have compressed deflate data which inflates to 0 bytes, but these entries claim to uncompress to 512 bytes in the headers).
+  // I'm torn how to handle this case - should it fail instead?
+  if (mz_zip_reader_is_file_a_directory(pZip, file_index))
+    return MZ_TRUE;
+
+  // Encryption and patch files are not supported.
+  if (file_stat.m_bit_flag & (1 | 32))
+    return MZ_FALSE;
+
+  // This function only supports stored and deflate.
+  if ((!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (file_stat.m_method != 0) && (file_stat.m_method != MZ_DEFLATED))
+    return MZ_FALSE;
+
+  // Ensure supplied output buffer is large enough.
+  needed_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? file_stat.m_comp_size : file_stat.m_uncomp_size;
+  if (buf_size < needed_size)
+    return MZ_FALSE;
+
+  // Read and parse the local directory entry.
+  cur_file_ofs = file_stat.m_local_header_ofs;
+  if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)
+    return MZ_FALSE;
+  if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)
+    return MZ_FALSE;
+
+  cur_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);
+  if ((cur_file_ofs + file_stat.m_comp_size) > pZip->m_archive_size)
+    return MZ_FALSE;
+
+  if ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!file_stat.m_method))
+  {
+    // The file is stored or the caller has requested the compressed data.
+    if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, (size_t)needed_size) != needed_size)
+      return MZ_FALSE;
+    return ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) != 0) || (mz_crc32(MZ_CRC32_INIT, (const mz_uint8 *)pBuf, (size_t)file_stat.m_uncomp_size) == file_stat.m_crc32);
+  }
+
+  // Decompress the file either directly from memory or from a file input buffer.
+  tinfl_init(&inflator);
+
+  if (pZip->m_pState->m_pMem)
+  {
+    // Read directly from the archive in memory.
+    pRead_buf = (mz_uint8 *)pZip->m_pState->m_pMem + cur_file_ofs;
+    read_buf_size = read_buf_avail = file_stat.m_comp_size;
+    comp_remaining = 0;
+  }
+  else if (pUser_read_buf)
+  {
+    // Use a user provided read buffer.
+    if (!user_read_buf_size)
+      return MZ_FALSE;
+    pRead_buf = (mz_uint8 *)pUser_read_buf;
+    read_buf_size = user_read_buf_size;
+    read_buf_avail = 0;
+    comp_remaining = file_stat.m_comp_size;
+  }
+  else
+  {
+    // Temporarily allocate a read buffer.
+    read_buf_size = MZ_MIN(file_stat.m_comp_size, MZ_ZIP_MAX_IO_BUF_SIZE);
+#ifdef _MSC_VER
+    if (((0, sizeof(size_t) == sizeof(mz_uint32))) && (read_buf_size > 0x7FFFFFFF))
+#else
+    if (((sizeof(size_t) == sizeof(mz_uint32))) && (read_buf_size > 0x7FFFFFFF))
+#endif
+      return MZ_FALSE;
+    if (NULL == (pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)read_buf_size)))
+      return MZ_FALSE;
+    read_buf_avail = 0;
+    comp_remaining = file_stat.m_comp_size;
+  }
+
+  do
+  {
+    size_t in_buf_size, out_buf_size = (size_t)(file_stat.m_uncomp_size - out_buf_ofs);
+    if ((!read_buf_avail) && (!pZip->m_pState->m_pMem))
+    {
+      read_buf_avail = MZ_MIN(read_buf_size, comp_remaining);
+      if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)
+      {
+        status = TINFL_STATUS_FAILED;
+        break;
+      }
+      cur_file_ofs += read_buf_avail;
+      comp_remaining -= read_buf_avail;
+      read_buf_ofs = 0;
+    }
+    in_buf_size = (size_t)read_buf_avail;
+    status = tinfl_decompress(&inflator, (mz_uint8 *)pRead_buf + read_buf_ofs, &in_buf_size, (mz_uint8 *)pBuf, (mz_uint8 *)pBuf + out_buf_ofs, &out_buf_size, TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF | (comp_remaining ? TINFL_FLAG_HAS_MORE_INPUT : 0));
+    read_buf_avail -= in_buf_size;
+    read_buf_ofs += in_buf_size;
+    out_buf_ofs += out_buf_size;
+  } while (status == TINFL_STATUS_NEEDS_MORE_INPUT);
+
+  if (status == TINFL_STATUS_DONE)
+  {
+    // Make sure the entire file was decompressed, and check its CRC.
+    if ((out_buf_ofs != file_stat.m_uncomp_size) || (mz_crc32(MZ_CRC32_INIT, (const mz_uint8 *)pBuf, (size_t)file_stat.m_uncomp_size) != file_stat.m_crc32))
+      status = TINFL_STATUS_FAILED;
+  }
+
+  if ((!pZip->m_pState->m_pMem) && (!pUser_read_buf))
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);
+
+  return status == TINFL_STATUS_DONE;
+}
+
+mz_bool mz_zip_reader_extract_file_to_mem_no_alloc(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size)
+{
+  int file_index = mz_zip_reader_locate_file(pZip, pFilename, NULL, flags);
+  if (file_index < 0)
+    return MZ_FALSE;
+  return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, pUser_read_buf, user_read_buf_size);
+}
+
+mz_bool mz_zip_reader_extract_to_mem(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags)
+{
+  return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, NULL, 0);
+}
+
+mz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags)
+{
+  return mz_zip_reader_extract_file_to_mem_no_alloc(pZip, pFilename, pBuf, buf_size, flags, NULL, 0);
+}
+
+void *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, size_t *pSize, mz_uint flags)
+{
+  mz_uint64 comp_size, uncomp_size, alloc_size;
+  const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index);
+  void *pBuf;
+
+  if (pSize)
+    *pSize = 0;
+  if (!p)
+    return NULL;
+
+  comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS);
+  uncomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS);
+
+  alloc_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? comp_size : uncomp_size;
+#ifdef _MSC_VER
+  if (((0, sizeof(size_t) == sizeof(mz_uint32))) && (alloc_size > 0x7FFFFFFF))
+#else
+  if (((sizeof(size_t) == sizeof(mz_uint32))) && (alloc_size > 0x7FFFFFFF))
+#endif
+    return NULL;
+  if (NULL == (pBuf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)alloc_size)))
+    return NULL;
+
+  if (!mz_zip_reader_extract_to_mem(pZip, file_index, pBuf, (size_t)alloc_size, flags))
+  {
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);
+    return NULL;
+  }
+
+  if (pSize) *pSize = (size_t)alloc_size;
+  return pBuf;
+}
+
+void *mz_zip_reader_extract_file_to_heap(mz_zip_archive *pZip, const char *pFilename, size_t *pSize, mz_uint flags)
+{
+  int file_index = mz_zip_reader_locate_file(pZip, pFilename, NULL, flags);
+  if (file_index < 0)
+  {
+    if (pSize) *pSize = 0;
+    return MZ_FALSE;
+  }
+  return mz_zip_reader_extract_to_heap(pZip, file_index, pSize, flags);
+}
+
+mz_bool mz_zip_reader_extract_to_callback(mz_zip_archive *pZip, mz_uint file_index, mz_file_write_func pCallback, void *pOpaque, mz_uint flags)
+{
+  int status = TINFL_STATUS_DONE; mz_uint file_crc32 = MZ_CRC32_INIT;
+  mz_uint64 read_buf_size, read_buf_ofs = 0, read_buf_avail, comp_remaining, out_buf_ofs = 0, cur_file_ofs;
+  mz_zip_archive_file_stat file_stat;
+  void *pRead_buf = NULL; void *pWrite_buf = NULL;
+  mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)]; mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;
+
+  if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat))
+    return MZ_FALSE;
+
+  // Empty file, or a directory (but not always a directory - I've seen odd zips with directories that have compressed data which inflates to 0 bytes)
+  if (!file_stat.m_comp_size)
+    return MZ_TRUE;
+
+  // Entry is a subdirectory (I've seen old zips with dir entries which have compressed deflate data which inflates to 0 bytes, but these entries claim to uncompress to 512 bytes in the headers).
+  // I'm torn how to handle this case - should it fail instead?
+  if (mz_zip_reader_is_file_a_directory(pZip, file_index))
+    return MZ_TRUE;
+
+  // Encryption and patch files are not supported.
+  if (file_stat.m_bit_flag & (1 | 32))
+    return MZ_FALSE;
+
+  // This function only supports stored and deflate.
+  if ((!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (file_stat.m_method != 0) && (file_stat.m_method != MZ_DEFLATED))
+    return MZ_FALSE;
+
+  // Read and parse the local directory entry.
+  cur_file_ofs = file_stat.m_local_header_ofs;
+  if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)
+    return MZ_FALSE;
+  if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)
+    return MZ_FALSE;
+
+  cur_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);
+  if ((cur_file_ofs + file_stat.m_comp_size) > pZip->m_archive_size)
+    return MZ_FALSE;
+
+  // Decompress the file either directly from memory or from a file input buffer.
+  if (pZip->m_pState->m_pMem)
+  {
+    pRead_buf = (mz_uint8 *)pZip->m_pState->m_pMem + cur_file_ofs;
+    read_buf_size = read_buf_avail = file_stat.m_comp_size;
+    comp_remaining = 0;
+  }
+  else
+  {
+    read_buf_size = MZ_MIN(file_stat.m_comp_size, MZ_ZIP_MAX_IO_BUF_SIZE);
+    if (NULL == (pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)read_buf_size)))
+      return MZ_FALSE;
+    read_buf_avail = 0;
+    comp_remaining = file_stat.m_comp_size;
+  }
+
+  if ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!file_stat.m_method))
+  {
+    // The file is stored or the caller has requested the compressed data.
+    if (pZip->m_pState->m_pMem)
+    {
+#ifdef _MSC_VER
+      if (((0, sizeof(size_t) == sizeof(mz_uint32))) && (file_stat.m_comp_size > 0xFFFFFFFF))
+#else
+      if (((sizeof(size_t) == sizeof(mz_uint32))) && (file_stat.m_comp_size > 0xFFFFFFFF))
+#endif
+        return MZ_FALSE;
+      if (pCallback(pOpaque, out_buf_ofs, pRead_buf, (size_t)file_stat.m_comp_size) != file_stat.m_comp_size)
+        status = TINFL_STATUS_FAILED;
+      else if (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA))
+        file_crc32 = (mz_uint32)mz_crc32(file_crc32, (const mz_uint8 *)pRead_buf, (size_t)file_stat.m_comp_size);
+      cur_file_ofs += file_stat.m_comp_size;
+      out_buf_ofs += file_stat.m_comp_size;
+      comp_remaining = 0;
+    }
+    else
+    {
+      while (comp_remaining)
+      {
+        read_buf_avail = MZ_MIN(read_buf_size, comp_remaining);
+        if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)
+        {
+          status = TINFL_STATUS_FAILED;
+          break;
+        }
+
+        if (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA))
+          file_crc32 = (mz_uint32)mz_crc32(file_crc32, (const mz_uint8 *)pRead_buf, (size_t)read_buf_avail);
+
+        if (pCallback(pOpaque, out_buf_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)
+        {
+          status = TINFL_STATUS_FAILED;
+          break;
+        }
+        cur_file_ofs += read_buf_avail;
+        out_buf_ofs += read_buf_avail;
+        comp_remaining -= read_buf_avail;
+      }
+    }
+  }
+  else
+  {
+    tinfl_decompressor inflator;
+    tinfl_init(&inflator);
+
+    if (NULL == (pWrite_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, TINFL_LZ_DICT_SIZE)))
+      status = TINFL_STATUS_FAILED;
+    else
+    {
+      do
+      {
+        mz_uint8 *pWrite_buf_cur = (mz_uint8 *)pWrite_buf + (out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1));
+        size_t in_buf_size, out_buf_size = TINFL_LZ_DICT_SIZE - (out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1));
+        if ((!read_buf_avail) && (!pZip->m_pState->m_pMem))
+        {
+          read_buf_avail = MZ_MIN(read_buf_size, comp_remaining);
+          if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)
+          {
+            status = TINFL_STATUS_FAILED;
+            break;
+          }
+          cur_file_ofs += read_buf_avail;
+          comp_remaining -= read_buf_avail;
+          read_buf_ofs = 0;
+        }
+
+        in_buf_size = (size_t)read_buf_avail;
+        status = tinfl_decompress(&inflator, (const mz_uint8 *)pRead_buf + read_buf_ofs, &in_buf_size, (mz_uint8 *)pWrite_buf, pWrite_buf_cur, &out_buf_size, comp_remaining ? TINFL_FLAG_HAS_MORE_INPUT : 0);
+        read_buf_avail -= in_buf_size;
+        read_buf_ofs += in_buf_size;
+
+        if (out_buf_size)
+        {
+          if (pCallback(pOpaque, out_buf_ofs, pWrite_buf_cur, out_buf_size) != out_buf_size)
+          {
+            status = TINFL_STATUS_FAILED;
+            break;
+          }
+          file_crc32 = (mz_uint32)mz_crc32(file_crc32, pWrite_buf_cur, out_buf_size);
+          if ((out_buf_ofs += out_buf_size) > file_stat.m_uncomp_size)
+          {
+            status = TINFL_STATUS_FAILED;
+            break;
+          }
+        }
+      } while ((status == TINFL_STATUS_NEEDS_MORE_INPUT) || (status == TINFL_STATUS_HAS_MORE_OUTPUT));
+    }
+  }
+
+  if ((status == TINFL_STATUS_DONE) && (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)))
+  {
+    // Make sure the entire file was decompressed, and check its CRC.
+    if ((out_buf_ofs != file_stat.m_uncomp_size) || (file_crc32 != file_stat.m_crc32))
+      status = TINFL_STATUS_FAILED;
+  }
+
+  if (!pZip->m_pState->m_pMem)
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);
+  if (pWrite_buf)
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pWrite_buf);
+
+  return status == TINFL_STATUS_DONE;
+}
+
+mz_bool mz_zip_reader_extract_file_to_callback(mz_zip_archive *pZip, const char *pFilename, mz_file_write_func pCallback, void *pOpaque, mz_uint flags)
+{
+  int file_index = mz_zip_reader_locate_file(pZip, pFilename, NULL, flags);
+  if (file_index < 0)
+    return MZ_FALSE;
+  return mz_zip_reader_extract_to_callback(pZip, file_index, pCallback, pOpaque, flags);
+}
+
+#ifndef MINIZ_NO_STDIO
+static size_t mz_zip_file_write_callback(void *pOpaque, mz_uint64 ofs, const void *pBuf, size_t n)
+{
+  (void)ofs; return MZ_FWRITE(pBuf, 1, n, (MZ_FILE*)pOpaque);
+}
+
+mz_bool mz_zip_reader_extract_to_file(mz_zip_archive *pZip, mz_uint file_index, const char *pDst_filename, mz_uint flags)
+{
+  mz_bool status;
+  mz_zip_archive_file_stat file_stat;
+  MZ_FILE *pFile;
+  if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat))
+    return MZ_FALSE;
+  pFile = MZ_FOPEN(pDst_filename, "wb");
+  if (!pFile)
+    return MZ_FALSE;
+  status = mz_zip_reader_extract_to_callback(pZip, file_index, mz_zip_file_write_callback, pFile, flags);
+  if (MZ_FCLOSE(pFile) == EOF)
+    return MZ_FALSE;
+#ifndef MINIZ_NO_TIME
+  if (status)
+    mz_zip_set_file_times(pDst_filename, file_stat.m_time, file_stat.m_time);
+#endif
+  return status;
+}
+#endif // #ifndef MINIZ_NO_STDIO
+
+mz_bool mz_zip_reader_end(mz_zip_archive *pZip)
+{
+  if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING))
+    return MZ_FALSE;
+
+  if (pZip->m_pState)
+  {
+    mz_zip_internal_state *pState = pZip->m_pState; pZip->m_pState = NULL;
+    mz_zip_array_clear(pZip, &pState->m_central_dir);
+    mz_zip_array_clear(pZip, &pState->m_central_dir_offsets);
+    mz_zip_array_clear(pZip, &pState->m_sorted_central_dir_offsets);
+
+#ifndef MINIZ_NO_STDIO
+    if (pState->m_pFile)
+    {
+      MZ_FCLOSE(pState->m_pFile);
+      pState->m_pFile = NULL;
+    }
+#endif // #ifndef MINIZ_NO_STDIO
+
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pState);
+  }
+  pZip->m_zip_mode = MZ_ZIP_MODE_INVALID;
+
+  return MZ_TRUE;
+}
+
+#ifndef MINIZ_NO_STDIO
+mz_bool mz_zip_reader_extract_file_to_file(mz_zip_archive *pZip, const char *pArchive_filename, const char *pDst_filename, mz_uint flags)
+{
+  int file_index = mz_zip_reader_locate_file(pZip, pArchive_filename, NULL, flags);
+  if (file_index < 0)
+    return MZ_FALSE;
+  return mz_zip_reader_extract_to_file(pZip, file_index, pDst_filename, flags);
+}
+#endif
+
+// ------------------- .ZIP archive writing
+
+#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS
+
+static void mz_write_le16(mz_uint8 *p, mz_uint16 v) { p[0] = (mz_uint8)v; p[1] = (mz_uint8)(v >> 8); }
+static void mz_write_le32(mz_uint8 *p, mz_uint32 v) { p[0] = (mz_uint8)v; p[1] = (mz_uint8)(v >> 8); p[2] = (mz_uint8)(v >> 16); p[3] = (mz_uint8)(v >> 24); }
+#define MZ_WRITE_LE16(p, v) mz_write_le16((mz_uint8 *)(p), (mz_uint16)(v))
+#define MZ_WRITE_LE32(p, v) mz_write_le32((mz_uint8 *)(p), (mz_uint32)(v))
+
+mz_bool mz_zip_writer_init(mz_zip_archive *pZip, mz_uint64 existing_size)
+{
+  if ((!pZip) || (pZip->m_pState) || (!pZip->m_pWrite) || (pZip->m_zip_mode != MZ_ZIP_MODE_INVALID))
+    return MZ_FALSE;
+
+  if (pZip->m_file_offset_alignment)
+  {
+    // Ensure user specified file offset alignment is a power of 2.
+    if (pZip->m_file_offset_alignment & (pZip->m_file_offset_alignment - 1))
+      return MZ_FALSE;
+  }
+
+  if (!pZip->m_pAlloc) pZip->m_pAlloc = def_alloc_func;
+  if (!pZip->m_pFree) pZip->m_pFree = def_free_func;
+  if (!pZip->m_pRealloc) pZip->m_pRealloc = def_realloc_func;
+
+  pZip->m_zip_mode = MZ_ZIP_MODE_WRITING;
+  pZip->m_archive_size = existing_size;
+  pZip->m_central_directory_file_ofs = 0;
+  pZip->m_total_files = 0;
+
+  if (NULL == (pZip->m_pState = (mz_zip_internal_state *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(mz_zip_internal_state))))
+    return MZ_FALSE;
+  memset(pZip->m_pState, 0, sizeof(mz_zip_internal_state));
+  MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir, sizeof(mz_uint8));
+  MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir_offsets, sizeof(mz_uint32));
+  MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_sorted_central_dir_offsets, sizeof(mz_uint32));
+  return MZ_TRUE;
+}
+
+static size_t mz_zip_heap_write_func(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n)
+{
+  mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;
+  mz_zip_internal_state *pState = pZip->m_pState;
+  mz_uint64 new_size = MZ_MAX(file_ofs + n, pState->m_mem_size);
+#ifdef _MSC_VER
+  if ((!n) || ((0, sizeof(size_t) == sizeof(mz_uint32)) && (new_size > 0x7FFFFFFF)))
+#else
+  if ((!n) || ((sizeof(size_t) == sizeof(mz_uint32)) && (new_size > 0x7FFFFFFF)))
+#endif
+    return 0;
+  if (new_size > pState->m_mem_capacity)
+  {
+    void *pNew_block;
+    size_t new_capacity = MZ_MAX(64, pState->m_mem_capacity); while (new_capacity < new_size) new_capacity *= 2;
+    if (NULL == (pNew_block = pZip->m_pRealloc(pZip->m_pAlloc_opaque, pState->m_pMem, 1, new_capacity)))
+      return 0;
+    pState->m_pMem = pNew_block; pState->m_mem_capacity = new_capacity;
+  }
+  memcpy((mz_uint8 *)pState->m_pMem + file_ofs, pBuf, n);
+  pState->m_mem_size = (size_t)new_size;
+  return n;
+}
+
+mz_bool mz_zip_writer_init_heap(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size)
+{
+  pZip->m_pWrite = mz_zip_heap_write_func;
+  pZip->m_pIO_opaque = pZip;
+  if (!mz_zip_writer_init(pZip, size_to_reserve_at_beginning))
+    return MZ_FALSE;
+  if (0 != (initial_allocation_size = MZ_MAX(initial_allocation_size, size_to_reserve_at_beginning)))
+  {
+    if (NULL == (pZip->m_pState->m_pMem = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, initial_allocation_size)))
+    {
+      mz_zip_writer_end(pZip);
+      return MZ_FALSE;
+    }
+    pZip->m_pState->m_mem_capacity = initial_allocation_size;
+  }
+  return MZ_TRUE;
+}
+
+#ifndef MINIZ_NO_STDIO
+static size_t mz_zip_file_write_func(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n)
+{
+  mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;
+  mz_int64 cur_ofs = MZ_FTELL64(pZip->m_pState->m_pFile);
+  if (((mz_int64)file_ofs < 0) || (((cur_ofs != (mz_int64)file_ofs)) && (MZ_FSEEK64(pZip->m_pState->m_pFile, (mz_int64)file_ofs, SEEK_SET))))
+    return 0;
+  return MZ_FWRITE(pBuf, 1, n, pZip->m_pState->m_pFile);
+}
+
+mz_bool mz_zip_writer_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning)
+{
+  MZ_FILE *pFile;
+  pZip->m_pWrite = mz_zip_file_write_func;
+  pZip->m_pIO_opaque = pZip;
+  if (!mz_zip_writer_init(pZip, size_to_reserve_at_beginning))
+    return MZ_FALSE;
+  if (NULL == (pFile = MZ_FOPEN(pFilename, "wb")))
+  {
+    mz_zip_writer_end(pZip);
+    return MZ_FALSE;
+  }
+  pZip->m_pState->m_pFile = pFile;
+  if (size_to_reserve_at_beginning)
+  {
+    mz_uint64 cur_ofs = 0; char buf[4096]; MZ_CLEAR_ARR(buf);
+    do
+    {
+      size_t n = (size_t)MZ_MIN(sizeof(buf), size_to_reserve_at_beginning);
+      if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_ofs, buf, n) != n)
+      {
+        mz_zip_writer_end(pZip);
+        return MZ_FALSE;
+      }
+      cur_ofs += n; size_to_reserve_at_beginning -= n;
+    } while (size_to_reserve_at_beginning);
+  }
+  return MZ_TRUE;
+}
+#endif // #ifndef MINIZ_NO_STDIO
+
+mz_bool mz_zip_writer_init_from_reader(mz_zip_archive *pZip, const char *pFilename)
+{
+  mz_zip_internal_state *pState;
+  if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING))
+    return MZ_FALSE;
+  // No sense in trying to write to an archive that's already at the support max size
+  if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_ZIP_LOCAL_DIR_HEADER_SIZE) > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  pState = pZip->m_pState;
+
+  if (pState->m_pFile)
+  {
+#ifdef MINIZ_NO_STDIO
+    pFilename; return MZ_FALSE;
+#else
+    // Archive is being read from stdio - try to reopen as writable.
+    if (pZip->m_pIO_opaque != pZip)
+      return MZ_FALSE;
+    if (!pFilename)
+      return MZ_FALSE;
+    pZip->m_pWrite = mz_zip_file_write_func;
+    if (NULL == (pState->m_pFile = MZ_FREOPEN(pFilename, "r+b", pState->m_pFile)))
+    {
+      // The mz_zip_archive is now in a bogus state because pState->m_pFile is NULL, so just close it.
+      mz_zip_reader_end(pZip);
+      return MZ_FALSE;
+    }
+#endif // #ifdef MINIZ_NO_STDIO
+  }
+  else if (pState->m_pMem)
+  {
+    // Archive lives in a memory block. Assume it's from the heap that we can resize using the realloc callback.
+    if (pZip->m_pIO_opaque != pZip)
+      return MZ_FALSE;
+    pState->m_mem_capacity = pState->m_mem_size;
+    pZip->m_pWrite = mz_zip_heap_write_func;
+  }
+  // Archive is being read via a user provided read function - make sure the user has specified a write function too.
+  else if (!pZip->m_pWrite)
+    return MZ_FALSE;
+
+  // Start writing new files at the archive's current central directory location.
+  pZip->m_archive_size = pZip->m_central_directory_file_ofs;
+  pZip->m_zip_mode = MZ_ZIP_MODE_WRITING;
+  pZip->m_central_directory_file_ofs = 0;
+
+  return MZ_TRUE;
+}
+
+mz_bool mz_zip_writer_add_mem(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, mz_uint level_and_flags)
+{
+  return mz_zip_writer_add_mem_ex(pZip, pArchive_name, pBuf, buf_size, NULL, 0, level_and_flags, 0, 0);
+}
+
+typedef struct
+{
+  mz_zip_archive *m_pZip;
+  mz_uint64 m_cur_archive_file_ofs;
+  mz_uint64 m_comp_size;
+} mz_zip_writer_add_state;
+
+static mz_bool mz_zip_writer_add_put_buf_callback(const void* pBuf, int len, void *pUser)
+{
+  mz_zip_writer_add_state *pState = (mz_zip_writer_add_state *)pUser;
+  if ((int)pState->m_pZip->m_pWrite(pState->m_pZip->m_pIO_opaque, pState->m_cur_archive_file_ofs, pBuf, len) != len)
+    return MZ_FALSE;
+  pState->m_cur_archive_file_ofs += len;
+  pState->m_comp_size += len;
+  return MZ_TRUE;
+}
+
+static mz_bool mz_zip_writer_create_local_dir_header(mz_zip_archive *pZip, mz_uint8 *pDst, mz_uint16 filename_size, mz_uint16 extra_size, mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32, mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date)
+{
+  (void)pZip;
+  memset(pDst, 0, MZ_ZIP_LOCAL_DIR_HEADER_SIZE);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_SIG_OFS, MZ_ZIP_LOCAL_DIR_HEADER_SIG);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_VERSION_NEEDED_OFS, method ? 20 : 0);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_BIT_FLAG_OFS, bit_flags);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_METHOD_OFS, method);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILE_TIME_OFS, dos_time);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILE_DATE_OFS, dos_date);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_CRC32_OFS, uncomp_crc32);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_COMPRESSED_SIZE_OFS, comp_size);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS, uncomp_size);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILENAME_LEN_OFS, filename_size);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_EXTRA_LEN_OFS, extra_size);
+  return MZ_TRUE;
+}
+
+static mz_bool mz_zip_writer_create_central_dir_header(mz_zip_archive *pZip, mz_uint8 *pDst, mz_uint16 filename_size, mz_uint16 extra_size, mz_uint16 comment_size, mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32, mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date, mz_uint64 local_header_ofs, mz_uint32 ext_attributes)
+{
+  (void)pZip;
+  memset(pDst, 0, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_SIG_OFS, MZ_ZIP_CENTRAL_DIR_HEADER_SIG);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_VERSION_NEEDED_OFS, method ? 20 : 0);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_BIT_FLAG_OFS, bit_flags);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_METHOD_OFS, method);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILE_TIME_OFS, dos_time);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILE_DATE_OFS, dos_date);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_CRC32_OFS, uncomp_crc32);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS, comp_size);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS, uncomp_size);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILENAME_LEN_OFS, filename_size);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_EXTRA_LEN_OFS, extra_size);
+  MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_COMMENT_LEN_OFS, comment_size);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS, ext_attributes);
+  MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_LOCAL_HEADER_OFS, local_header_ofs);
+  return MZ_TRUE;
+}
+
+static mz_bool mz_zip_writer_add_to_central_dir(mz_zip_archive *pZip, const char *pFilename, mz_uint16 filename_size, const void *pExtra, mz_uint16 extra_size, const void *pComment, mz_uint16 comment_size, mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32, mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date, mz_uint64 local_header_ofs, mz_uint32 ext_attributes)
+{
+  mz_zip_internal_state *pState = pZip->m_pState;
+  mz_uint32 central_dir_ofs = (mz_uint32)pState->m_central_dir.m_size;
+  size_t orig_central_dir_size = pState->m_central_dir.m_size;
+  mz_uint8 central_dir_header[MZ_ZIP_CENTRAL_DIR_HEADER_SIZE];
+
+  // No zip64 support yet
+  if ((local_header_ofs > 0xFFFFFFFF) || (((mz_uint64)pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_size + extra_size + comment_size) > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  if (!mz_zip_writer_create_central_dir_header(pZip, central_dir_header, filename_size, extra_size, comment_size, uncomp_size, comp_size, uncomp_crc32, method, bit_flags, dos_time, dos_date, local_header_ofs, ext_attributes))
+    return MZ_FALSE;
+
+  if ((!mz_zip_array_push_back(pZip, &pState->m_central_dir, central_dir_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE)) ||
+      (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pFilename, filename_size)) ||
+      (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pExtra, extra_size)) ||
+      (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pComment, comment_size)) ||
+      (!mz_zip_array_push_back(pZip, &pState->m_central_dir_offsets, &central_dir_ofs, 1)))
+  {
+    // Try to push the central directory array back into its original state.
+    mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);
+    return MZ_FALSE;
+  }
+
+  return MZ_TRUE;
+}
+
+static mz_bool mz_zip_writer_validate_archive_name(const char *pArchive_name)
+{
+  // Basic ZIP archive filename validity checks: Valid filenames cannot start with a forward slash, cannot contain a drive letter, and cannot use DOS-style backward slashes.
+  if (*pArchive_name == '/')
+    return MZ_FALSE;
+  while (*pArchive_name)
+  {
+    if ((*pArchive_name == '\\') || (*pArchive_name == ':'))
+      return MZ_FALSE;
+    pArchive_name++;
+  }
+  return MZ_TRUE;
+}
+
+static mz_uint mz_zip_writer_compute_padding_needed_for_file_alignment(mz_zip_archive *pZip)
+{
+  mz_uint32 n;
+  if (!pZip->m_file_offset_alignment)
+    return 0;
+  n = (mz_uint32)(pZip->m_archive_size & (pZip->m_file_offset_alignment - 1));
+  return (pZip->m_file_offset_alignment - n) & (pZip->m_file_offset_alignment - 1);
+}
+
+static mz_bool mz_zip_writer_write_zeros(mz_zip_archive *pZip, mz_uint64 cur_file_ofs, mz_uint32 n)
+{
+  char buf[4096];
+  memset(buf, 0, MZ_MIN(sizeof(buf), n));
+  while (n)
+  {
+    mz_uint32 s = MZ_MIN(sizeof(buf), n);
+    if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_file_ofs, buf, s) != s)
+      return MZ_FALSE;
+    cur_file_ofs += s; n -= s;
+  }
+  return MZ_TRUE;
+}
+
+mz_bool mz_zip_writer_add_mem_ex(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, mz_uint64 uncomp_size, mz_uint32 uncomp_crc32)
+{
+  mz_uint16 method = 0, dos_time = 0, dos_date = 0;
+  mz_uint level, ext_attributes = 0, num_alignment_padding_bytes;
+  mz_uint64 local_dir_header_ofs = pZip->m_archive_size, cur_archive_file_ofs = pZip->m_archive_size, comp_size = 0;
+  size_t archive_name_size;
+  mz_uint8 local_dir_header[MZ_ZIP_LOCAL_DIR_HEADER_SIZE];
+  tdefl_compressor *pComp = NULL;
+  mz_bool store_data_uncompressed;
+  mz_zip_internal_state *pState;
+
+  if ((int)level_and_flags < 0)
+    level_and_flags = MZ_DEFAULT_LEVEL;
+  level = level_and_flags & 0xF;
+  store_data_uncompressed = ((!level) || (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA));
+
+  if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || ((buf_size) && (!pBuf)) || (!pArchive_name) || ((comment_size) && (!pComment)) || (pZip->m_total_files == 0xFFFF) || (level > MZ_UBER_COMPRESSION))
+    return MZ_FALSE;
+
+  pState = pZip->m_pState;
+
+  if ((!(level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (uncomp_size))
+    return MZ_FALSE;
+  // No zip64 support yet
+  if ((buf_size > 0xFFFFFFFF) || (uncomp_size > 0xFFFFFFFF))
+    return MZ_FALSE;
+  if (!mz_zip_writer_validate_archive_name(pArchive_name))
+    return MZ_FALSE;
+
+#ifndef MINIZ_NO_TIME
+  {
+    time_t cur_time; time(&cur_time);
+    mz_zip_time_to_dos_time(cur_time, &dos_time, &dos_date);
+  }
+#endif // #ifndef MINIZ_NO_TIME
+
+  archive_name_size = strlen(pArchive_name);
+  if (archive_name_size > 0xFFFF)
+    return MZ_FALSE;
+
+  num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip);
+
+  // no zip64 support yet
+  if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + comment_size + archive_name_size) > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  if ((archive_name_size) && (pArchive_name[archive_name_size - 1] == '/'))
+  {
+    // Set DOS Subdirectory attribute bit.
+    ext_attributes |= 0x10;
+    // Subdirectories cannot contain data.
+    if ((buf_size) || (uncomp_size))
+      return MZ_FALSE;
+  }
+
+  // Try to do any allocations before writing to the archive, so if an allocation fails the file remains unmodified. (A good idea if we're doing an in-place modification.)
+  if ((!mz_zip_array_ensure_room(pZip, &pState->m_central_dir, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + archive_name_size + comment_size)) || (!mz_zip_array_ensure_room(pZip, &pState->m_central_dir_offsets, 1)))
+    return MZ_FALSE;
+
+  if ((!store_data_uncompressed) && (buf_size))
+  {
+    if (NULL == (pComp = (tdefl_compressor *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(tdefl_compressor))))
+      return MZ_FALSE;
+  }
+
+  if (!mz_zip_writer_write_zeros(pZip, cur_archive_file_ofs, num_alignment_padding_bytes + sizeof(local_dir_header)))
+  {
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);
+    return MZ_FALSE;
+  }
+  local_dir_header_ofs += num_alignment_padding_bytes;
+  if (pZip->m_file_offset_alignment) { MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0); }
+  cur_archive_file_ofs += num_alignment_padding_bytes + sizeof(local_dir_header);
+
+  MZ_CLEAR_ARR(local_dir_header);
+  if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size)
+  {
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);
+    return MZ_FALSE;
+  }
+  cur_archive_file_ofs += archive_name_size;
+
+  if (!(level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA))
+  {
+    uncomp_crc32 = (mz_uint32)mz_crc32(MZ_CRC32_INIT, (const mz_uint8*)pBuf, buf_size);
+    uncomp_size = buf_size;
+    if (uncomp_size <= 3)
+    {
+      level = 0;
+      store_data_uncompressed = MZ_TRUE;
+    }
+  }
+
+  if (store_data_uncompressed)
+  {
+    if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pBuf, buf_size) != buf_size)
+    {
+      pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);
+      return MZ_FALSE;
+    }
+
+    cur_archive_file_ofs += buf_size;
+    comp_size = buf_size;
+
+    if (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)
+      method = MZ_DEFLATED;
+  }
+  else if (buf_size)
+  {
+    mz_zip_writer_add_state state;
+
+    state.m_pZip = pZip;
+    state.m_cur_archive_file_ofs = cur_archive_file_ofs;
+    state.m_comp_size = 0;
+
+    if ((tdefl_init(pComp, mz_zip_writer_add_put_buf_callback, &state, tdefl_create_comp_flags_from_zip_params(level, -15, MZ_DEFAULT_STRATEGY)) != TDEFL_STATUS_OKAY) ||
+        (tdefl_compress_buffer(pComp, pBuf, buf_size, TDEFL_FINISH) != TDEFL_STATUS_DONE))
+    {
+      pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);
+      return MZ_FALSE;
+    }
+
+    comp_size = state.m_comp_size;
+    cur_archive_file_ofs = state.m_cur_archive_file_ofs;
+
+    method = MZ_DEFLATED;
+  }
+
+  pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);
+  pComp = NULL;
+
+  // no zip64 support yet
+  if ((comp_size > 0xFFFFFFFF) || (cur_archive_file_ofs > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, 0, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date))
+    return MZ_FALSE;
+
+  if (pZip->m_pWrite(pZip->m_pIO_opaque, local_dir_header_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))
+    return MZ_FALSE;
+
+  if (!mz_zip_writer_add_to_central_dir(pZip, pArchive_name, (mz_uint16)archive_name_size, NULL, 0, pComment, comment_size, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date, local_dir_header_ofs, ext_attributes))
+    return MZ_FALSE;
+
+  pZip->m_total_files++;
+  pZip->m_archive_size = cur_archive_file_ofs;
+
+  return MZ_TRUE;
+}
+
+#ifndef MINIZ_NO_STDIO
+mz_bool mz_zip_writer_add_file(mz_zip_archive *pZip, const char *pArchive_name, const char *pSrc_filename, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags)
+{
+  mz_uint uncomp_crc32 = MZ_CRC32_INIT, level, num_alignment_padding_bytes;
+  mz_uint16 method = 0, dos_time = 0, dos_date = 0, ext_attributes = 0;
+  mz_uint64 local_dir_header_ofs = pZip->m_archive_size, cur_archive_file_ofs = pZip->m_archive_size, uncomp_size = 0, comp_size = 0;
+  size_t archive_name_size;
+  mz_uint8 local_dir_header[MZ_ZIP_LOCAL_DIR_HEADER_SIZE];
+  MZ_FILE *pSrc_file = NULL;
+
+  if ((int)level_and_flags < 0)
+    level_and_flags = MZ_DEFAULT_LEVEL;
+  level = level_and_flags & 0xF;
+
+  if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || (!pArchive_name) || ((comment_size) && (!pComment)) || (level > MZ_UBER_COMPRESSION))
+    return MZ_FALSE;
+  if (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)
+    return MZ_FALSE;
+  if (!mz_zip_writer_validate_archive_name(pArchive_name))
+    return MZ_FALSE;
+
+  archive_name_size = strlen(pArchive_name);
+  if (archive_name_size > 0xFFFF)
+    return MZ_FALSE;
+
+  num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip);
+
+  // no zip64 support yet
+  if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + comment_size + archive_name_size) > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  if (!mz_zip_get_file_modified_time(pSrc_filename, &dos_time, &dos_date))
+    return MZ_FALSE;
+
+  pSrc_file = MZ_FOPEN(pSrc_filename, "rb");
+  if (!pSrc_file)
+    return MZ_FALSE;
+  MZ_FSEEK64(pSrc_file, 0, SEEK_END);
+  uncomp_size = MZ_FTELL64(pSrc_file);
+  MZ_FSEEK64(pSrc_file, 0, SEEK_SET);
+
+  if (uncomp_size > 0xFFFFFFFF)
+  {
+    // No zip64 support yet
+    MZ_FCLOSE(pSrc_file);
+    return MZ_FALSE;
+  }
+  if (uncomp_size <= 3)
+    level = 0;
+
+  if (!mz_zip_writer_write_zeros(pZip, cur_archive_file_ofs, num_alignment_padding_bytes + sizeof(local_dir_header)))
+  {
+    MZ_FCLOSE(pSrc_file);
+    return MZ_FALSE;
+  }
+  local_dir_header_ofs += num_alignment_padding_bytes;
+  if (pZip->m_file_offset_alignment) { MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0); }
+  cur_archive_file_ofs += num_alignment_padding_bytes + sizeof(local_dir_header);
+
+  MZ_CLEAR_ARR(local_dir_header);
+  if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size)
+  {
+    MZ_FCLOSE(pSrc_file);
+    return MZ_FALSE;
+  }
+  cur_archive_file_ofs += archive_name_size;
+
+  if (uncomp_size)
+  {
+    mz_uint64 uncomp_remaining = uncomp_size;
+    void *pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, MZ_ZIP_MAX_IO_BUF_SIZE);
+    if (!pRead_buf)
+    {
+      MZ_FCLOSE(pSrc_file);
+      return MZ_FALSE;
+    }
+
+    if (!level)
+    {
+      while (uncomp_remaining)
+      {
+        mz_uint n = (mz_uint)MZ_MIN(MZ_ZIP_MAX_IO_BUF_SIZE, uncomp_remaining);
+        if ((MZ_FREAD(pRead_buf, 1, n, pSrc_file) != n) || (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pRead_buf, n) != n))
+        {
+          pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);
+          MZ_FCLOSE(pSrc_file);
+          return MZ_FALSE;
+        }
+        uncomp_crc32 = (mz_uint32)mz_crc32(uncomp_crc32, (const mz_uint8 *)pRead_buf, n);
+        uncomp_remaining -= n;
+        cur_archive_file_ofs += n;
+      }
+      comp_size = uncomp_size;
+    }
+    else
+    {
+      mz_bool result = MZ_FALSE;
+      mz_zip_writer_add_state state;
+      tdefl_compressor *pComp = (tdefl_compressor *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(tdefl_compressor));
+      if (!pComp)
+      {
+        pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);
+        MZ_FCLOSE(pSrc_file);
+        return MZ_FALSE;
+      }
+
+      state.m_pZip = pZip;
+      state.m_cur_archive_file_ofs = cur_archive_file_ofs;
+      state.m_comp_size = 0;
+
+      if (tdefl_init(pComp, mz_zip_writer_add_put_buf_callback, &state, tdefl_create_comp_flags_from_zip_params(level, -15, MZ_DEFAULT_STRATEGY)) != TDEFL_STATUS_OKAY)
+      {
+        pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);
+        pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);
+        MZ_FCLOSE(pSrc_file);
+        return MZ_FALSE;
+      }
+
+      for ( ; ; )
+      {
+        size_t in_buf_size = (mz_uint32)MZ_MIN(uncomp_remaining, MZ_ZIP_MAX_IO_BUF_SIZE);
+        tdefl_status status;
+
+        if (MZ_FREAD(pRead_buf, 1, in_buf_size, pSrc_file) != in_buf_size)
+          break;
+
+        uncomp_crc32 = (mz_uint32)mz_crc32(uncomp_crc32, (const mz_uint8 *)pRead_buf, in_buf_size);
+        uncomp_remaining -= in_buf_size;
+
+        status = tdefl_compress_buffer(pComp, pRead_buf, in_buf_size, uncomp_remaining ? TDEFL_NO_FLUSH : TDEFL_FINISH);
+        if (status == TDEFL_STATUS_DONE)
+        {
+          result = MZ_TRUE;
+          break;
+        }
+        else if (status != TDEFL_STATUS_OKAY)
+          break;
+      }
+
+      pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);
+
+      if (!result)
+      {
+        pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);
+        MZ_FCLOSE(pSrc_file);
+        return MZ_FALSE;
+      }
+
+      comp_size = state.m_comp_size;
+      cur_archive_file_ofs = state.m_cur_archive_file_ofs;
+
+      method = MZ_DEFLATED;
+    }
+
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);
+  }
+
+  MZ_FCLOSE(pSrc_file); pSrc_file = NULL;
+
+  // no zip64 support yet
+  if ((comp_size > 0xFFFFFFFF) || (cur_archive_file_ofs > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, 0, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date))
+    return MZ_FALSE;
+
+  if (pZip->m_pWrite(pZip->m_pIO_opaque, local_dir_header_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))
+    return MZ_FALSE;
+
+  if (!mz_zip_writer_add_to_central_dir(pZip, pArchive_name, (mz_uint16)archive_name_size, NULL, 0, pComment, comment_size, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date, local_dir_header_ofs, ext_attributes))
+    return MZ_FALSE;
+
+  pZip->m_total_files++;
+  pZip->m_archive_size = cur_archive_file_ofs;
+
+  return MZ_TRUE;
+}
+#endif // #ifndef MINIZ_NO_STDIO
+
+mz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive *pSource_zip, mz_uint file_index)
+{
+  mz_uint n, bit_flags, num_alignment_padding_bytes;
+  mz_uint64 comp_bytes_remaining, local_dir_header_ofs;
+  mz_uint64 cur_src_file_ofs, cur_dst_file_ofs;
+  mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)]; mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;
+  mz_uint8 central_header[MZ_ZIP_CENTRAL_DIR_HEADER_SIZE];
+  size_t orig_central_dir_size;
+  mz_zip_internal_state *pState;
+  void *pBuf; const mz_uint8 *pSrc_central_header;
+
+  if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING))
+    return MZ_FALSE;
+  if (NULL == (pSrc_central_header = mz_zip_reader_get_cdh(pSource_zip, file_index)))
+    return MZ_FALSE;
+  pState = pZip->m_pState;
+
+  num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip);
+
+  // no zip64 support yet
+  if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  cur_src_file_ofs = MZ_READ_LE32(pSrc_central_header + MZ_ZIP_CDH_LOCAL_HEADER_OFS);
+  cur_dst_file_ofs = pZip->m_archive_size;
+
+  if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)
+    return MZ_FALSE;
+  if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)
+    return MZ_FALSE;
+  cur_src_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE;
+
+  if (!mz_zip_writer_write_zeros(pZip, cur_dst_file_ofs, num_alignment_padding_bytes))
+    return MZ_FALSE;
+  cur_dst_file_ofs += num_alignment_padding_bytes;
+  local_dir_header_ofs = cur_dst_file_ofs;
+  if (pZip->m_file_offset_alignment) { MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0); }
+
+  if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)
+    return MZ_FALSE;
+  cur_dst_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE;
+
+  n = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);
+  comp_bytes_remaining = n + MZ_READ_LE32(pSrc_central_header + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS);
+
+  if (NULL == (pBuf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)MZ_MAX(sizeof(mz_uint32) * 4, MZ_MIN(MZ_ZIP_MAX_IO_BUF_SIZE, comp_bytes_remaining)))))
+    return MZ_FALSE;
+
+  while (comp_bytes_remaining)
+  {
+    n = (mz_uint)MZ_MIN(MZ_ZIP_MAX_IO_BUF_SIZE, comp_bytes_remaining);
+    if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pBuf, n) != n)
+    {
+      pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);
+      return MZ_FALSE;
+    }
+    cur_src_file_ofs += n;
+
+    if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pBuf, n) != n)
+    {
+      pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);
+      return MZ_FALSE;
+    }
+    cur_dst_file_ofs += n;
+
+    comp_bytes_remaining -= n;
+  }
+
+  bit_flags = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_BIT_FLAG_OFS);
+  if (bit_flags & 8)
+  {
+    // Copy data descriptor
+    if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pBuf, sizeof(mz_uint32) * 4) != sizeof(mz_uint32) * 4)
+    {
+      pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);
+      return MZ_FALSE;
+    }
+
+    n = sizeof(mz_uint32) * ((MZ_READ_LE32(pBuf) == 0x08074b50) ? 4 : 3);
+    if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pBuf, n) != n)
+    {
+      pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);
+      return MZ_FALSE;
+    }
+
+    cur_src_file_ofs += n;
+    cur_dst_file_ofs += n;
+  }
+  pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);
+
+  // no zip64 support yet
+  if (cur_dst_file_ofs > 0xFFFFFFFF)
+    return MZ_FALSE;
+
+  orig_central_dir_size = pState->m_central_dir.m_size;
+
+  memcpy(central_header, pSrc_central_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE);
+  MZ_WRITE_LE32(central_header + MZ_ZIP_CDH_LOCAL_HEADER_OFS, local_dir_header_ofs);
+  if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, central_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE))
+    return MZ_FALSE;
+
+  n = MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_EXTRA_LEN_OFS) + MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_COMMENT_LEN_OFS);
+  if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pSrc_central_header + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n))
+  {
+    mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);
+    return MZ_FALSE;
+  }
+
+  if (pState->m_central_dir.m_size > 0xFFFFFFFF)
+    return MZ_FALSE;
+  n = (mz_uint32)orig_central_dir_size;
+  if (!mz_zip_array_push_back(pZip, &pState->m_central_dir_offsets, &n, 1))
+  {
+    mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);
+    return MZ_FALSE;
+  }
+
+  pZip->m_total_files++;
+  pZip->m_archive_size = cur_dst_file_ofs;
+
+  return MZ_TRUE;
+}
+
+mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip)
+{
+  mz_zip_internal_state *pState;
+  mz_uint64 central_dir_ofs, central_dir_size;
+  mz_uint8 hdr[MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE];
+
+  if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING))
+    return MZ_FALSE;
+
+  pState = pZip->m_pState;
+
+  // no zip64 support yet
+  if ((pZip->m_total_files > 0xFFFF) || ((pZip->m_archive_size + pState->m_central_dir.m_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) > 0xFFFFFFFF))
+    return MZ_FALSE;
+
+  central_dir_ofs = 0;
+  central_dir_size = 0;
+  if (pZip->m_total_files)
+  {
+    // Write central directory
+    central_dir_ofs = pZip->m_archive_size;
+    central_dir_size = pState->m_central_dir.m_size;
+    pZip->m_central_directory_file_ofs = central_dir_ofs;
+    if (pZip->m_pWrite(pZip->m_pIO_opaque, central_dir_ofs, pState->m_central_dir.m_p, (size_t)central_dir_size) != central_dir_size)
+      return MZ_FALSE;
+    pZip->m_archive_size += central_dir_size;
+  }
+
+  // Write end of central directory record
+  MZ_CLEAR_ARR(hdr);
+  MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_SIG_OFS, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG);
+  MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS, pZip->m_total_files);
+  MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS, pZip->m_total_files);
+  MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_SIZE_OFS, central_dir_size);
+  MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_OFS_OFS, central_dir_ofs);
+
+  if (pZip->m_pWrite(pZip->m_pIO_opaque, pZip->m_archive_size, hdr, sizeof(hdr)) != sizeof(hdr))
+    return MZ_FALSE;
+#ifndef MINIZ_NO_STDIO
+  if ((pState->m_pFile) && (MZ_FFLUSH(pState->m_pFile) == EOF))
+    return MZ_FALSE;
+#endif // #ifndef MINIZ_NO_STDIO
+
+  pZip->m_archive_size += sizeof(hdr);
+
+  pZip->m_zip_mode = MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED;
+  return MZ_TRUE;
+}
+
+mz_bool mz_zip_writer_finalize_heap_archive(mz_zip_archive *pZip, void **pBuf, size_t *pSize)
+{
+  if ((!pZip) || (!pZip->m_pState) || (!pBuf) || (!pSize))
+    return MZ_FALSE;
+  if (pZip->m_pWrite != mz_zip_heap_write_func)
+    return MZ_FALSE;
+  if (!mz_zip_writer_finalize_archive(pZip))
+    return MZ_FALSE;
+
+  *pBuf = pZip->m_pState->m_pMem;
+  *pSize = pZip->m_pState->m_mem_size;
+  pZip->m_pState->m_pMem = NULL;
+  pZip->m_pState->m_mem_size = pZip->m_pState->m_mem_capacity = 0;
+  return MZ_TRUE;
+}
+
+mz_bool mz_zip_writer_end(mz_zip_archive *pZip)
+{
+  mz_zip_internal_state *pState;
+  mz_bool status = MZ_TRUE;
+  if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || ((pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) && (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED)))
+    return MZ_FALSE;
+
+  pState = pZip->m_pState;
+  pZip->m_pState = NULL;
+  mz_zip_array_clear(pZip, &pState->m_central_dir);
+  mz_zip_array_clear(pZip, &pState->m_central_dir_offsets);
+  mz_zip_array_clear(pZip, &pState->m_sorted_central_dir_offsets);
+
+#ifndef MINIZ_NO_STDIO
+  if (pState->m_pFile)
+  {
+    MZ_FCLOSE(pState->m_pFile);
+    pState->m_pFile = NULL;
+  }
+#endif // #ifndef MINIZ_NO_STDIO
+
+  if ((pZip->m_pWrite == mz_zip_heap_write_func) && (pState->m_pMem))
+  {
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pState->m_pMem);
+    pState->m_pMem = NULL;
+  }
+
+  pZip->m_pFree(pZip->m_pAlloc_opaque, pState);
+  pZip->m_zip_mode = MZ_ZIP_MODE_INVALID;
+  return status;
+}
+
+#ifndef MINIZ_NO_STDIO
+mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags)
+{
+  mz_bool status, created_new_archive = MZ_FALSE;
+  mz_zip_archive zip_archive;
+  struct MZ_FILE_STAT_STRUCT file_stat;
+  MZ_CLEAR_OBJ(zip_archive);
+  if ((int)level_and_flags < 0)
+     level_and_flags = MZ_DEFAULT_LEVEL;
+  if ((!pZip_filename) || (!pArchive_name) || ((buf_size) && (!pBuf)) || ((comment_size) && (!pComment)) || ((level_and_flags & 0xF) > MZ_UBER_COMPRESSION))
+    return MZ_FALSE;
+  if (!mz_zip_writer_validate_archive_name(pArchive_name))
+    return MZ_FALSE;
+  if (MZ_FILE_STAT(pZip_filename, &file_stat) != 0)
+  {
+    // Create a new archive.
+    if (!mz_zip_writer_init_file(&zip_archive, pZip_filename, 0))
+      return MZ_FALSE;
+    created_new_archive = MZ_TRUE;
+  }
+  else
+  {
+    // Append to an existing archive.
+    if (!mz_zip_reader_init_file(&zip_archive, pZip_filename, level_and_flags | MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY))
+      return MZ_FALSE;
+    if (!mz_zip_writer_init_from_reader(&zip_archive, pZip_filename))
+    {
+      mz_zip_reader_end(&zip_archive);
+      return MZ_FALSE;
+    }
+  }
+  status = mz_zip_writer_add_mem_ex(&zip_archive, pArchive_name, pBuf, buf_size, pComment, comment_size, level_and_flags, 0, 0);
+  // Always finalize, even if adding failed for some reason, so we have a valid central directory. (This may not always succeed, but we can try.)
+  if (!mz_zip_writer_finalize_archive(&zip_archive))
+    status = MZ_FALSE;
+  if (!mz_zip_writer_end(&zip_archive))
+    status = MZ_FALSE;
+  if ((!status) && (created_new_archive))
+  {
+    // It's a new archive and something went wrong, so just delete it.
+    int ignoredStatus = MZ_DELETE_FILE(pZip_filename);
+    (void)ignoredStatus;
+  }
+  return status;
+}
+
+void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, size_t *pSize, mz_uint flags)
+{
+  int file_index;
+  mz_zip_archive zip_archive;
+  void *p = NULL;
+
+  if (pSize)
+    *pSize = 0;
+
+  if ((!pZip_filename) || (!pArchive_name))
+    return NULL;
+
+  MZ_CLEAR_OBJ(zip_archive);
+  if (!mz_zip_reader_init_file(&zip_archive, pZip_filename, flags | MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY))
+    return NULL;
+
+  if ((file_index = mz_zip_reader_locate_file(&zip_archive, pArchive_name, NULL, flags)) >= 0)
+    p = mz_zip_reader_extract_to_heap(&zip_archive, file_index, pSize, flags);
+
+  mz_zip_reader_end(&zip_archive);
+  return p;
+}
+
+#endif // #ifndef MINIZ_NO_STDIO
+
+#endif // #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS
+
+#endif // #ifndef MINIZ_NO_ARCHIVE_APIS
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef _MSC_VER
+#pragma warning (pop)
+#endif
+
+#endif // MINIZ_HEADER_FILE_ONLY
+
+/*
+  This is free and unencumbered software released into the public domain.
+
+  Anyone is free to copy, modify, publish, use, compile, sell, or
+  distribute this software, either in source code form or as a compiled
+  binary, for any purpose, commercial or non-commercial, and by any
+  means.
+
+  In jurisdictions that recognize copyright laws, the author or authors
+  of this software dedicate any and all copyright interest in the
+  software to the public domain. We make this dedication for the benefit
+  of the public at large and to the detriment of our heirs and
+  successors. We intend this dedication to be an overt act of
+  relinquishment in perpetuity of all present and future rights to this
+  software under copyright law.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+  OTHER DEALINGS IN THE SOFTWARE.
+
+  For more information, please refer to <http://unlicense.org/>
+*/
diff --git a/thirdparty/plume b/thirdparty/plume
new file mode 160000
index 0000000..de0f70f
--- /dev/null
+++ b/thirdparty/plume
@@ -0,0 +1 @@
+Subproject commit de0f70f75087242ce92516dd587ecadff25385db
diff --git a/thirdparty/zstd/LICENSE b/thirdparty/zstd/LICENSE
new file mode 100644
index 0000000..7580028
--- /dev/null
+++ b/thirdparty/zstd/LICENSE
@@ -0,0 +1,30 @@
+BSD License
+
+For Zstandard software
+
+Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook, nor Meta, nor the names of its contributors may
+   be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h
new file mode 100644
index 0000000..a78b6db
--- /dev/null
+++ b/thirdparty/zstd/zstd.h
@@ -0,0 +1,3142 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef ZSTD_H_235446
+#define ZSTD_H_235446
+
+/* ======   Dependencies   ======*/
+#include <stddef.h>   /* size_t */
+
+
+/* =====   ZSTDLIB_API : control library symbols visibility   ===== */
+#ifndef ZSTDLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDLIB_VISIBILITY
+#    define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
+#  else
+#    define ZSTDLIB_VISIBLE
+#  endif
+#endif
+
+#ifndef ZSTDLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDLIB_HIDDEN
+#  endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDLIB_API ZSTDLIB_VISIBLE
+#endif
+
+/* Deprecation warnings :
+ * Should these warnings be a problem, it is generally possible to disable them,
+ * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
+ * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
+ */
+#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
+#  define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
+#else
+#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#    define ZSTD_DEPRECATED(message) [[deprecated(message)]]
+#  elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__)
+#    define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif defined(__GNUC__) && (__GNUC__ >= 3)
+#    define ZSTD_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
+#    define ZSTD_DEPRECATED(message)
+#  endif
+#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
+
+
+/*******************************************************************************
+  Introduction
+
+  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
+  real-time compression scenarios at zlib-level and better compression ratios.
+  The zstd compression library provides in-memory compression and decompression
+  functions.
+
+  The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
+  which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
+  caution, as they require more memory. The library also offers negative
+  compression levels, which extend the range of speed vs. ratio preferences.
+  The lower the level, the faster the speed (at the cost of compression).
+
+  Compression can be done in:
+    - a single step (described as Simple API)
+    - a single step, reusing a context (described as Explicit context)
+    - unbounded multiple steps (described as Streaming compression)
+
+  The compression ratio achievable on small data can be highly improved using
+  a dictionary. Dictionary compression can be performed in:
+    - a single step (described as Simple dictionary API)
+    - a single step, reusing a dictionary (described as Bulk-processing
+      dictionary API)
+
+  Advanced experimental functions can be accessed using
+  `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
+
+  Advanced experimental APIs should never be used with a dynamically-linked
+  library. They are not "stable"; their definitions or signatures may change in
+  the future. Only static linking is allowed.
+*******************************************************************************/
+
+/*------   Version   ------*/
+#define ZSTD_VERSION_MAJOR    1
+#define ZSTD_VERSION_MINOR    5
+#define ZSTD_VERSION_RELEASE  7
+#define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+
+/*! ZSTD_versionNumber() :
+ *  Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
+ZSTDLIB_API unsigned ZSTD_versionNumber(void);
+
+#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
+#define ZSTD_QUOTE(str) #str
+#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
+#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
+
+/*! ZSTD_versionString() :
+ *  Return runtime library version, like "1.4.5". Requires v1.3.0+. */
+ZSTDLIB_API const char* ZSTD_versionString(void);
+
+/* *************************************
+ *  Default constant
+ ***************************************/
+#ifndef ZSTD_CLEVEL_DEFAULT
+#  define ZSTD_CLEVEL_DEFAULT 3
+#endif
+
+/* *************************************
+ *  Constants
+ ***************************************/
+
+/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
+#define ZSTD_MAGICNUMBER            0xFD2FB528    /* valid since v0.8.0 */
+#define ZSTD_MAGIC_DICTIONARY       0xEC30A437    /* valid since v0.7.0 */
+#define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50    /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
+#define ZSTD_MAGIC_SKIPPABLE_MASK   0xFFFFFFF0
+
+#define ZSTD_BLOCKSIZELOG_MAX  17
+#define ZSTD_BLOCKSIZE_MAX     (1<<ZSTD_BLOCKSIZELOG_MAX)
+
+
+/***************************************
+*  Simple Core API
+***************************************/
+/*! ZSTD_compress() :
+ *  Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
+ *  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
+ *        enough space to successfully compress the data.
+ *  @return : compressed size written into `dst` (<= `dstCapacity),
+ *            or an error code if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                                  int compressionLevel);
+
+/*! ZSTD_decompress() :
+ * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
+ *  Multiple compressed frames can be decompressed at once with this method.
+ *  The result will be the concatenation of all decompressed frames, back to back.
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
+ *  First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
+ *  If maximum upper bound isn't known, prefer using streaming mode to decompress data.
+ * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ *           or an errorCode if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+                              const void* src, size_t compressedSize);
+
+
+/*======  Decompression helper functions  ======*/
+
+/*! ZSTD_getFrameContentSize() : requires v1.3.0+
+ * `src` should point to the start of a ZSTD encoded frame.
+ * `srcSize` must be at least as large as the frame header.
+ *           hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
+ * @return : - decompressed size of `src` frame content, if known
+ *           - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ *           - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
+ *  note 1 : a 0 return value means the frame is valid but "empty".
+ *  note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode).
+ *           When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ *           In which case, it's necessary to use streaming mode to decompress data.
+ *           Optionally, application can rely on some implicit limit,
+ *           as ZSTD_decompress() only needs an upper bound of decompressed size.
+ *           (For example, data could be necessarily cut into blocks <= 16 KB).
+ *  note 3 : decompressed size is always present when compression is completed using single-pass functions,
+ *           such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
+ *  note 4 : decompressed size can be very large (64-bits value),
+ *           potentially larger than what local system can handle as a single memory segment.
+ *           In which case, it's necessary to use streaming mode to decompress data.
+ *  note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ *           Always ensure return value fits within application's authorized limits.
+ *           Each application can set its own limits.
+ *  note 6 : This function replaces ZSTD_getDecompressedSize() */
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+
+/*! ZSTD_getDecompressedSize() :
+ *  NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
+ *  Both functions work the same way, but ZSTD_getDecompressedSize() blends
+ *  "empty", "unknown" and "error" results to the same return value (0),
+ *  while ZSTD_getFrameContentSize() gives them separate return values.
+ * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
+ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
+ * `src` should point to the start of a ZSTD frame or skippable frame.
+ * `srcSize` must be >= first frame size
+ * @return : the compressed size of the first frame starting at `src`,
+ *           suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ *        or an error code if input is invalid */
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
+
+/*======  Compression helper functions  ======*/
+
+/*! ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()`, or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize is too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+
+
+/*======  Error helper functions  ======*/
+#include "zstd_errors.h" /* list of errors */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
+ZSTDLIB_API unsigned     ZSTD_isError(size_t result);      /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */
+ZSTDLIB_API const char*  ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */
+ZSTDLIB_API int          ZSTD_minCLevel(void);             /*!< minimum negative compression level allowed, requires v1.4.0+ */
+ZSTDLIB_API int          ZSTD_maxCLevel(void);             /*!< maximum compression level available */
+ZSTDLIB_API int          ZSTD_defaultCLevel(void);         /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
+
+
+/***************************************
+*  Explicit context
+***************************************/
+/*= Compression context
+ *  When compressing many times,
+ *  it is recommended to allocate a compression context just once,
+ *  and reuse it for each successive compression operation.
+ *  This will make the workload easier for system's memory.
+ *  Note : re-using context is just a speed / resource optimization.
+ *         It doesn't change the compression ratio, which remains identical.
+ *  Note 2: For parallel execution in multi-threaded environments,
+ *         use one different context per thread .
+ */
+typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
+ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  /* compatible with NULL pointer */
+
+/*! ZSTD_compressCCtx() :
+ *  Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
+ *  Important : in order to mirror `ZSTD_compress()` behavior,
+ *  this function compresses at the requested compression level,
+ *  __ignoring any other advanced parameter__ .
+ *  If any advanced parameter was set using the advanced API,
+ *  they will all be reset. Only @compressionLevel remains.
+ */
+ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
+                                     void* dst, size_t dstCapacity,
+                               const void* src, size_t srcSize,
+                                     int compressionLevel);
+
+/*= Decompression context
+ *  When decompressing many times,
+ *  it is recommended to allocate a context only once,
+ *  and reuse it for each successive compression operation.
+ *  This will make workload friendlier for system's memory.
+ *  Use one context per thread for parallel execution. */
+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
+ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  /* accept NULL pointer */
+
+/*! ZSTD_decompressDCtx() :
+ *  Same as ZSTD_decompress(),
+ *  requires an allocated ZSTD_DCtx.
+ *  Compatible with sticky parameters (see below).
+ */
+ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
+                                       void* dst, size_t dstCapacity,
+                                 const void* src, size_t srcSize);
+
+
+/*********************************************
+*  Advanced compression API (Requires v1.4.0+)
+**********************************************/
+
+/* API design :
+ *   Parameters are pushed one by one into an existing context,
+ *   using ZSTD_CCtx_set*() functions.
+ *   Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
+ *   "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
+ *   __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ .
+ *
+ *   It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
+ *
+ *   This API supersedes all other "advanced" API entry points in the experimental section.
+ *   In the future, we expect to remove API entry points from experimental which are redundant with this API.
+ */
+
+
+/* Compression strategies, listed from fastest to strongest */
+typedef enum { ZSTD_fast=1,
+               ZSTD_dfast=2,
+               ZSTD_greedy=3,
+               ZSTD_lazy=4,
+               ZSTD_lazy2=5,
+               ZSTD_btlazy2=6,
+               ZSTD_btopt=7,
+               ZSTD_btultra=8,
+               ZSTD_btultra2=9
+               /* note : new strategies _might_ be added in the future.
+                         Only the order (from fast to strong) is guaranteed */
+} ZSTD_strategy;
+
+typedef enum {
+
+    /* compression parameters
+     * Note: When compressing with a ZSTD_CDict these parameters are superseded
+     * by the parameters used to construct the ZSTD_CDict.
+     * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
+    ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
+                              * Note that exact compression parameters are dynamically determined,
+                              * depending on both compression level and srcSize (when known).
+                              * Default level is ZSTD_CLEVEL_DEFAULT==3.
+                              * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
+                              * Note 1 : it's possible to pass a negative compression level.
+                              * Note 2 : setting a level does not automatically set all other compression parameters
+                              *   to default. Setting this will however eventually dynamically impact the compression
+                              *   parameters which have not been manually set. The manually set
+                              *   ones will 'stick'. */
+    /* Advanced compression parameters :
+     * It's possible to pin down compression parameters to some specific values.
+     * In which case, these values are no longer dynamically selected by the compressor */
+    ZSTD_c_windowLog=101,    /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * This will set a memory budget for streaming decompression,
+                              * with larger values requiring more memory
+                              * and typically compressing more.
+                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+                              * Special: value 0 means "use default windowLog".
+                              * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
+                              *       requires explicitly allowing such size at streaming decompression stage. */
+    ZSTD_c_hashLog=102,      /* Size of the initial probe table, as a power of 2.
+                              * Resulting memory usage is (1 << (hashLog+2)).
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+                              * Larger tables improve compression ratio of strategies <= dFast,
+                              * and improve speed of strategies > dFast.
+                              * Special: value 0 means "use default hashLog". */
+    ZSTD_c_chainLog=103,     /* Size of the multi-probe search table, as a power of 2.
+                              * Resulting memory usage is (1 << (chainLog+2)).
+                              * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
+                              * Larger tables result in better and slower compression.
+                              * This parameter is useless for "fast" strategy.
+                              * It's still useful when using "dfast" strategy,
+                              * in which case it defines a secondary probe table.
+                              * Special: value 0 means "use default chainLog". */
+    ZSTD_c_searchLog=104,    /* Number of search attempts, as a power of 2.
+                              * More attempts result in better and slower compression.
+                              * This parameter is useless for "fast" and "dFast" strategies.
+                              * Special: value 0 means "use default searchLog". */
+    ZSTD_c_minMatch=105,     /* Minimum size of searched matches.
+                              * Note that Zstandard can still find matches of smaller size,
+                              * it just tweaks its search algorithm to look for this size and larger.
+                              * Larger values increase compression and decompression speed, but decrease ratio.
+                              * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
+                              * Note that currently, for all strategies < btopt, effective minimum is 4.
+                              *                    , for all strategies > fast, effective maximum is 6.
+                              * Special: value 0 means "use default minMatchLength". */
+    ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
+                              * For strategies btopt, btultra & btultra2:
+                              *     Length of Match considered "good enough" to stop search.
+                              *     Larger values make compression stronger, and slower.
+                              * For strategy fast:
+                              *     Distance between match sampling.
+                              *     Larger values make compression faster, and weaker.
+                              * Special: value 0 means "use default targetLength". */
+    ZSTD_c_strategy=107,     /* See ZSTD_strategy enum definition.
+                              * The higher the value of selected strategy, the more complex it is,
+                              * resulting in stronger and slower compression.
+                              * Special: value 0 means "use default strategy". */
+
+    ZSTD_c_targetCBlockSize=130, /* v1.5.6+
+                                  * Attempts to fit compressed block size into approximately targetCBlockSize.
+                                  * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
+                                  * Note that it's not a guarantee, just a convergence target (default:0).
+                                  * No target when targetCBlockSize == 0.
+                                  * This is helpful in low bandwidth streaming environments to improve end-to-end latency,
+                                  * when a client can make use of partial documents (a prominent example being Chrome).
+                                  * Note: this parameter is stable since v1.5.6.
+                                  * It was present as an experimental parameter in earlier versions,
+                                  * but it's not recommended using it with earlier library versions
+                                  * due to massive performance regressions.
+                                  */
+    /* LDM mode parameters */
+    ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
+                                     * This parameter is designed to improve compression ratio
+                                     * for large inputs, by finding large matches at long distance.
+                                     * It increases memory usage and window size.
+                                     * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
+                                     * except when expressly set to a different value.
+                                     * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
+                                     * compression strategy >= ZSTD_btopt (== compression level 16+) */
+    ZSTD_c_ldmHashLog=161,   /* Size of the table for long distance matching, as a power of 2.
+                              * Larger values increase memory usage and compression ratio,
+                              * but decrease compression speed.
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+                              * default: windowlog - 7.
+                              * Special: value 0 means "automatically determine hashlog". */
+    ZSTD_c_ldmMinMatch=162,  /* Minimum match size for long distance matcher.
+                              * Larger/too small values usually decrease compression ratio.
+                              * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
+                              * Special: value 0 means "use default value" (default: 64). */
+    ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
+                              * Larger values improve collision resolution but decrease compression speed.
+                              * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
+                              * Special: value 0 means "use default value" (default: 3). */
+    ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
+                              * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
+                              * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
+                              * Larger values improve compression speed.
+                              * Deviating far from default value will likely result in a compression ratio decrease.
+                              * Special: value 0 means "automatically determine hashRateLog". */
+
+    /* frame parameters */
+    ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+                              * Content size must be known at the beginning of compression.
+                              * This is automatically the case when using ZSTD_compress2(),
+                              * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
+    ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
+    ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
+
+    /* multi-threading parameters */
+    /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+     * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
+     * In a situation where it's unknown if the linked library supports multi-threading or not,
+     * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
+     */
+    ZSTD_c_nbWorkers=400,    /* Select how many threads will be spawned to compress in parallel.
+                              * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
+                              * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
+                              * while compression is performed in parallel, within worker thread(s).
+                              * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
+                              *  in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
+                              * More workers improve speed, but also increase memory usage.
+                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
+                              * compression is performed inside Caller's thread, and all invocations are blocking */
+    ZSTD_c_jobSize=401,      /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
+                              * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
+                              * 0 means default, which is dynamically determined based on compression parameters.
+                              * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
+                              * The minimum size is automatically and transparently enforced. */
+    ZSTD_c_overlapLog=402,   /* Control the overlap size, as a fraction of window size.
+                              * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
+                              * It helps preserve compression ratio, while each job is compressed in parallel.
+                              * This value is enforced only when nbWorkers >= 1.
+                              * Larger values increase compression ratio, but decrease speed.
+                              * Possible values range from 0 to 9 :
+                              * - 0 means "default" : value will be determined by the library, depending on strategy
+                              * - 1 means "no overlap"
+                              * - 9 means "full overlap", using a full window size.
+                              * Each intermediate rank increases/decreases load size by a factor 2 :
+                              * 9: full window;  8: w/2;  7: w/4;  6: w/8;  5:w/16;  4: w/32;  3:w/64;  2:w/128;  1:no overlap;  0:default
+                              * default value varies between 6 and 9, depending on strategy */
+
+    /* note : additional experimental parameters are also available
+     * within the experimental section of the API.
+     * At the time of this writing, they include :
+     * ZSTD_c_rsyncable
+     * ZSTD_c_format
+     * ZSTD_c_forceMaxWindow
+     * ZSTD_c_forceAttachDict
+     * ZSTD_c_literalCompressionMode
+     * ZSTD_c_srcSizeHint
+     * ZSTD_c_enableDedicatedDictSearch
+     * ZSTD_c_stableInBuffer
+     * ZSTD_c_stableOutBuffer
+     * ZSTD_c_blockDelimiters
+     * ZSTD_c_validateSequences
+     * ZSTD_c_blockSplitterLevel
+     * ZSTD_c_splitAfterSequences
+     * ZSTD_c_useRowMatchFinder
+     * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableSeqProducerFallback
+     * ZSTD_c_maxBlockSize
+     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+     * note : never ever use experimentalParam? names directly;
+     *        also, the enums values themselves are unstable and can still change.
+     */
+     ZSTD_c_experimentalParam1=500,
+     ZSTD_c_experimentalParam2=10,
+     ZSTD_c_experimentalParam3=1000,
+     ZSTD_c_experimentalParam4=1001,
+     ZSTD_c_experimentalParam5=1002,
+     /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */
+     ZSTD_c_experimentalParam7=1004,
+     ZSTD_c_experimentalParam8=1005,
+     ZSTD_c_experimentalParam9=1006,
+     ZSTD_c_experimentalParam10=1007,
+     ZSTD_c_experimentalParam11=1008,
+     ZSTD_c_experimentalParam12=1009,
+     ZSTD_c_experimentalParam13=1010,
+     ZSTD_c_experimentalParam14=1011,
+     ZSTD_c_experimentalParam15=1012,
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015,
+     ZSTD_c_experimentalParam19=1016,
+     ZSTD_c_experimentalParam20=1017
+} ZSTD_cParameter;
+
+typedef struct {
+    size_t error;
+    int lowerBound;
+    int upperBound;
+} ZSTD_bounds;
+
+/*! ZSTD_cParam_getBounds() :
+ *  All parameters must belong to an interval with lower and upper bounds,
+ *  otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ *         - an error status field, which must be tested using ZSTD_isError()
+ *         - lower and upper bounds, both inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
+
+/*! ZSTD_CCtx_setParameter() :
+ *  Set one compression parameter, selected by enum ZSTD_cParameter.
+ *  All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
+ *  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ *  Setting a parameter is generally only possible during frame initialization (before starting compression).
+ *  Exception : when using multi-threading mode (nbWorkers >= 1),
+ *              the following parameters can be updated _during_ compression (within same frame):
+ *              => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
+ *              new parameters will be active for next job only (after a flush()).
+ * @return : an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtx_setPledgedSrcSize() :
+ *  Total input data size to be compressed as a single frame.
+ *  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+ *  This value will also be controlled at end of frame, and trigger an error if not respected.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
+ *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ *           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
+ *  Note 2 : pledgedSrcSize is only valid once, for the next frame.
+ *           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
+ *  Note 3 : Whenever all input data is provided and consumed in a single round,
+ *           for example with ZSTD_compress2(),
+ *           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
+ *           this value is automatically overridden by srcSize instead.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
+typedef enum {
+    ZSTD_reset_session_only = 1,
+    ZSTD_reset_parameters = 2,
+    ZSTD_reset_session_and_parameters = 3
+} ZSTD_ResetDirective;
+
+/*! ZSTD_CCtx_reset() :
+ *  There are 2 different things that can be reset, independently or jointly :
+ *  - The session : will stop compressing current frame, and make CCtx ready to start a new one.
+ *                  Useful after an error, or to interrupt any ongoing compression.
+ *                  Any internal data not yet flushed is cancelled.
+ *                  Compression parameters and dictionary remain unchanged.
+ *                  They will be used to compress next frame.
+ *                  Resetting session never fails.
+ *  - The parameters : changes all parameters back to "default".
+ *                  This also removes any reference to any dictionary or external sequence producer.
+ *                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
+ *                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
+ *  - Both : similar to resetting the session, followed by resetting parameters.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
+
+/*! ZSTD_compress2() :
+ *  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+ *  (note that this entry point doesn't even expose a compression level parameter).
+ *  ZSTD_compress2() always starts a new frame.
+ *  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ *  - The function is always blocking, returns when compression is completed.
+ *  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
+ *        enough space to successfully compress the data, though it is possible it fails for other reasons.
+ * @return : compressed size written into `dst` (<= `dstCapacity),
+ *           or an error code if it fails (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
+                                   void* dst, size_t dstCapacity,
+                             const void* src, size_t srcSize);
+
+
+/***********************************************
+*  Advanced decompression API (Requires v1.4.0+)
+************************************************/
+
+/* The advanced API pushes parameters one by one into an existing DCtx context.
+ * Parameters are sticky, and remain valid for all following frames
+ * using the same DCtx context.
+ * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
+ * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
+ *        Therefore, no new decompression function is necessary.
+ */
+
+typedef enum {
+
+    ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
+                              * the streaming API will refuse to allocate memory buffer
+                              * in order to protect the host from unreasonable memory requirements.
+                              * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+                              * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
+                              * Special: value 0 means "use default maximum windowLog". */
+
+    /* note : additional experimental parameters are also available
+     * within the experimental section of the API.
+     * At the time of this writing, they include :
+     * ZSTD_d_format
+     * ZSTD_d_stableOutBuffer
+     * ZSTD_d_forceIgnoreChecksum
+     * ZSTD_d_refMultipleDDicts
+     * ZSTD_d_disableHuffmanAssembly
+     * ZSTD_d_maxBlockSize
+     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+     * note : never ever use experimentalParam? names directly
+     */
+     ZSTD_d_experimentalParam1=1000,
+     ZSTD_d_experimentalParam2=1001,
+     ZSTD_d_experimentalParam3=1002,
+     ZSTD_d_experimentalParam4=1003,
+     ZSTD_d_experimentalParam5=1004,
+     ZSTD_d_experimentalParam6=1005
+
+} ZSTD_dParameter;
+
+/*! ZSTD_dParam_getBounds() :
+ *  All parameters must belong to an interval with lower and upper bounds,
+ *  otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ *         - an error status field, which must be tested using ZSTD_isError()
+ *         - both lower and upper bounds, inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
+
+/*! ZSTD_DCtx_setParameter() :
+ *  Set one compression parameter, selected by enum ZSTD_dParameter.
+ *  All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+ *  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ *  Setting a parameter is only possible during frame initialization (before starting decompression).
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
+
+/*! ZSTD_DCtx_reset() :
+ *  Return a DCtx to clean state.
+ *  Session and parameters can be reset jointly or separately.
+ *  Parameters can only be reset when no active frame is being decompressed.
+ * @return : 0, or an error code, which can be tested with ZSTD_isError()
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
+
+
+/****************************
+*  Streaming
+****************************/
+
+typedef struct ZSTD_inBuffer_s {
+  const void* src;    /**< start of input buffer */
+  size_t size;        /**< size of input buffer */
+  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+  void*  dst;         /**< start of output buffer */
+  size_t size;        /**< size of output buffer */
+  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+
+
+/*-***********************************************************************
+*  Streaming compression - HowTo
+*
+*  A ZSTD_CStream object is required to track streaming operation.
+*  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+*  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+*  It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+*
+*  For parallel execution, use one separate ZSTD_CStream per thread.
+*
+*  note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+*
+*  Parameters are sticky : when starting a new compression on the same context,
+*  it will reuse the same sticky parameters as previous compression session.
+*  When in doubt, it's recommended to fully initialize the context before usage.
+*  Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
+*  ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
+*  set more specific parameters, the pledged source size, or load a dictionary.
+*
+*  Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
+*  consume input stream. The function will automatically update both `pos`
+*  fields within `input` and `output`.
+*  Note that the function may not consume the entire input, for example, because
+*  the output buffer is already full, in which case `input.pos < input.size`.
+*  The caller must check if input has been entirely consumed.
+*  If not, the caller must make some room to receive more compressed data,
+*  and then present again remaining input data.
+*  note: ZSTD_e_continue is guaranteed to make some forward progress when called,
+*        but doesn't guarantee maximal forward progress. This is especially relevant
+*        when compressing with multiple threads. The call won't block if it can
+*        consume some input, but if it can't it will wait for some, but not all,
+*        output to be flushed.
+* @return : provides a minimum amount of data remaining to be flushed from internal buffers
+*           or an error code, which can be tested using ZSTD_isError().
+*
+*  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
+*  using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
+*  Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
+*  In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
+*  You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
+*  operation.
+*  note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
+*        block until the flush is complete or the output buffer is full.
+*  @return : 0 if internal buffers are entirely flushed,
+*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+*  Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
+*  It will perform a flush and write frame epilogue.
+*  The epilogue is required for decoders to consider a frame completed.
+*  flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
+*  You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
+*  start a new frame.
+*  note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
+*        block until the flush is complete or the output buffer is full.
+*  @return : 0 if frame fully completed and fully flushed,
+*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************/
+
+typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
+                                 /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
+/*===== ZSTD_CStream management functions =====*/
+ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
+ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);  /* accept NULL pointer */
+
+/*===== Streaming compression functions =====*/
+typedef enum {
+    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+    ZSTD_e_flush=1,    /* flush any data provided so far,
+                        * it creates (at least) one new block, that can be decoded immediately on reception;
+                        * frame will continue: any future data can still reference previously compressed data, improving compression.
+                        * note : multithreaded compression will block to flush as much output as possible. */
+    ZSTD_e_end=2       /* flush any remaining data _and_ close current frame.
+                        * note that frame is only closed after compressed data is fully flushed (return value == 0).
+                        * After that point, any additional data starts a new frame.
+                        * note : each frame is independent (does not reference any content from previous frame).
+                        : note : multithreaded compression will block to flush as much output as possible. */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream2() : Requires v1.4.0+
+ *  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ *  - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+ *  - output->pos must be <= dstCapacity, input->pos must be <= srcSize
+ *  - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ *  - endOp must be a valid directive
+ *  - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ *  - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
+ *                                                  and then immediately returns, just indicating that there is some data remaining to be flushed.
+ *                                                  The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ *  - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+ *  - @return provides a minimum amount of data remaining to be flushed from internal buffers
+ *            or an error code, which can be tested using ZSTD_isError().
+ *            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ *            This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+ *            For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+ *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ *            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+ *            Before starting a new compression job, or changing compression parameters,
+ *            it is required to fully flush internal buffers.
+ *  - note: if an operation ends with an error, it may leave @cctx in an undefined state.
+ *          Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state.
+ *          In order to be re-employed after an error, a state must be reset,
+ *          which can be done explicitly (ZSTD_CCtx_reset()),
+ *          or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx())
+ */
+ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+                                         ZSTD_outBuffer* output,
+                                         ZSTD_inBuffer* input,
+                                         ZSTD_EndDirective endOp);
+
+
+/* These buffer sizes are softly recommended.
+ * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
+ * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
+ * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
+ *
+ * However, note that these recommendations are from the perspective of a C caller program.
+ * If the streaming interface is invoked from some other language,
+ * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
+ * a major performance rule is to reduce crossing such interface to an absolute minimum.
+ * It's not rare that performance ends being spent more into the interface, rather than compression itself.
+ * In which cases, prefer using large buffers, as large as practical,
+ * for both input and output, to reduce the nb of roundtrips.
+ */
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
+
+
+/* *****************************************************************************
+ * This following is a legacy streaming API, available since v1.0+ .
+ * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
+ * It is redundant, but remains fully supported.
+ ******************************************************************************/
+
+/*!
+ * Equivalent to:
+ *
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ * to compress with a dictionary.
+ */
+ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+/*!
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
+ */
+ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
+ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
+ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+
+/*-***************************************************************************
+*  Streaming decompression - HowTo
+*
+*  A ZSTD_DStream object is required to track streaming operations.
+*  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+*  ZSTD_DStream objects can be re-employed multiple times.
+*
+*  Use ZSTD_initDStream() to start a new decompression operation.
+* @return : recommended first input size
+*  Alternatively, use advanced API to set specific properties.
+*
+*  Use ZSTD_decompressStream() repetitively to consume your input.
+*  The function will update both `pos` fields.
+*  If `input.pos < input.size`, some input has not been consumed.
+*  It's up to the caller to present again remaining data.
+*
+*  The function tries to flush all data decoded immediately, respecting output buffer size.
+*  If `output.pos < output.size`, decoder has flushed everything it could.
+*
+*  However, when `output.pos == output.size`, it's more difficult to know.
+*  If @return > 0, the frame is not complete, meaning
+*  either there is still some data left to flush within internal buffers,
+*  or there is more input to read to complete the frame (or both).
+*  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+*  Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+* @return : 0 when a frame is completely decoded and fully flushed,
+*        or an error code, which can be tested using ZSTD_isError(),
+*        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
+*                                the return value is a suggested next input size (just a hint for better latency)
+*                                that will never request more than the remaining content of the compressed frame.
+* *******************************************************************************/
+
+typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+                                 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
+/*===== ZSTD_DStream management functions =====*/
+ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
+ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer */
+
+/*===== Streaming decompression functions =====*/
+
+/*! ZSTD_initDStream() :
+ * Initialize/reset DStream state for new decompression operation.
+ * Call before new decompression operation using same DStream.
+ *
+ * Note : This function is redundant with the advanced API and equivalent to:
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_refDDict(zds, NULL);
+ */
+ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+
+/*! ZSTD_decompressStream() :
+ * Streaming decompression function.
+ * Call repetitively to consume full input updating it as necessary.
+ * Function will update both input and output `pos` fields exposing current state via these fields:
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
+ *   on the next call.
+ * - `output.pos < output.size`, decoder flushed internal output buffer.
+ * - `output.pos == output.size`, unflushed data potentially present in the internal buffers,
+ *   check ZSTD_decompressStream() @return value,
+ *   if > 0, invoke it again to flush remaining data to output.
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * @return : 0 when a frame is completely decoded and fully flushed,
+ *           or an error code, which can be tested using ZSTD_isError(),
+ *           or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ *
+ * Note: when an operation returns with an error code, the @zds state may be left in undefined state.
+ *       It's UB to invoke `ZSTD_decompressStream()` on such a state.
+ *       In order to re-use such a state, it must be first reset,
+ *       which can be done explicitly (`ZSTD_DCtx_reset()`),
+ *       or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`)
+ */
+ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+
+
+/**************************
+*  Simple dictionary API
+***************************/
+/*! ZSTD_compress_usingDict() :
+ *  Compression at an explicit compression level using a Dictionary.
+ *  A dictionary can be any arbitrary data segment (also called a prefix),
+ *  or a buffer with specified information (see zdict.h).
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *         It's intended for a dictionary used only once.
+ *  Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+                                           void* dst, size_t dstCapacity,
+                                     const void* src, size_t srcSize,
+                                     const void* dict,size_t dictSize,
+                                           int compressionLevel);
+
+/*! ZSTD_decompress_usingDict() :
+ *  Decompression using a known Dictionary.
+ *  Dictionary must be identical to the one used during compression.
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *         It's intended for a dictionary used only once.
+ *  Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+                                             void* dst, size_t dstCapacity,
+                                       const void* src, size_t srcSize,
+                                       const void* dict,size_t dictSize);
+
+
+/***********************************
+ *  Bulk processing dictionary API
+ **********************************/
+typedef struct ZSTD_CDict_s ZSTD_CDict;
+
+/*! ZSTD_createCDict() :
+ *  When compressing multiple messages or blocks using the same dictionary,
+ *  it's recommended to digest the dictionary only once, since it's a costly operation.
+ *  ZSTD_createCDict() will create a state from digesting a dictionary.
+ *  The resulting state can be used for future compression operations with very limited startup cost.
+ *  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ *  Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+ *  Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+ *      in which case the only thing that it transports is the @compressionLevel.
+ *      This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+ *      expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+                                         int compressionLevel);
+
+/*! ZSTD_freeCDict() :
+ *  Function frees memory allocated by ZSTD_createCDict().
+ *  If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
+
+/*! ZSTD_compress_usingCDict() :
+ *  Compression using a digested Dictionary.
+ *  Recommended when same dictionary is used multiple times.
+ *  Note : compression level is _decided at dictionary creation time_,
+ *     and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
+ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+                                            void* dst, size_t dstCapacity,
+                                      const void* src, size_t srcSize,
+                                      const ZSTD_CDict* cdict);
+
+
+typedef struct ZSTD_DDict_s ZSTD_DDict;
+
+/*! ZSTD_createDDict() :
+ *  Create a digested dictionary, ready to start decompression operation without startup delay.
+ *  dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_freeDDict() :
+ *  Function frees memory allocated with ZSTD_createDDict()
+ *  If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
+
+/*! ZSTD_decompress_usingDDict() :
+ *  Decompression using a digested Dictionary.
+ *  Recommended when same dictionary is used multiple times. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+                                              void* dst, size_t dstCapacity,
+                                        const void* src, size_t srcSize,
+                                        const ZSTD_DDict* ddict);
+
+
+/********************************
+ *  Dictionary helper functions
+ *******************************/
+
+/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
+ *  Provides the dictID stored within dictionary.
+ *  if @return == 0, the dictionary is not conformant with Zstandard specification.
+ *  It can still be loaded, but as a content-only dictionary. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
+ *  Provides the dictID of the dictionary loaded into `cdict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
+
+/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
+ *  Provides the dictID of the dictionary loaded into `ddict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
+ *  Provides the dictID required to decompressed the frame stored within `src`.
+ *  If @return == 0, the dictID could not be decoded.
+ *  This could for one of the following reasons :
+ *  - The frame does not require a dictionary to be decoded (most common case).
+ *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
+ *    Note : this use case also happens when using a non-conformant dictionary.
+ *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ *  - This is not a Zstandard frame.
+ *  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
+
+/*******************************************************************************
+ * Advanced dictionary and prefix API (Requires v1.4.0+)
+ *
+ * This API allows dictionaries to be used with ZSTD_compress2(),
+ * ZSTD_compressStream2(), and ZSTD_decompressDCtx().
+ * Dictionaries are sticky, they remain valid when same context is reused,
+ * they only reset when the context is reset
+ * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
+ * In contrast, Prefixes are single-use.
+ ******************************************************************************/
+
+
+/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
+ *  Create an internal CDict from `dict` buffer.
+ *  Decompression will have to use same dictionary.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
+ *           meaning "return to no-dictionary mode".
+ *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ *           until parameters are reset, a new dictionary is loaded, or the dictionary
+ *           is explicitly invalidated by loading a NULL dictionary.
+ *  Note 2 : Loading a dictionary involves building tables.
+ *           It's also a CPU consuming operation, with non-negligible impact on latency.
+ *           Tables are dependent on compression parameters, and for this reason,
+ *           compression parameters can no longer be changed after loading a dictionary.
+ *  Note 3 :`dict` content will be copied internally.
+ *           Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
+ *           In such a case, dictionary buffer must outlive its users.
+ *  Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
+ *           to precisely select how dictionary content must be interpreted.
+ *  Note 5 : This method does not benefit from LDM (long distance mode).
+ *           If you want to employ LDM on some large dictionary content,
+ *           prefer employing ZSTD_CCtx_refPrefix() described below.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
+ *  Reference a prepared dictionary, to be used for all future compressed frames.
+ *  Note that compression parameters are enforced from within CDict,
+ *  and supersede any compression parameter previously set within CCtx.
+ *  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+ *  The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
+ *  The dictionary will remain valid for future compressed frames using same CCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Referencing a NULL CDict means "return to no-dictionary mode".
+ *  Note 1 : Currently, only one dictionary can be managed.
+ *           Referencing a new dictionary effectively "discards" any previous one.
+ *  Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
+ *  Reference a prefix (single-usage dictionary) for next compressed frame.
+ *  A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
+ *  Decompression will need same prefix to properly regenerate data.
+ *  Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ *  but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ *  This method is compatible with LDM (long distance mode).
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
+ *  Note 1 : Prefix buffer is referenced. It **must** outlive compression.
+ *           Its content must remain unmodified during compression.
+ *  Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ *           ensure that the window size is large enough to contain the entire source.
+ *           See ZSTD_c_windowLog.
+ *  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ *           It's a CPU consuming operation, with non-negligible impact on latency.
+ *           If there is a need to use the same prefix multiple times, consider loadDictionary instead.
+ *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
+ *           Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
+                                 const void* prefix, size_t prefixSize);
+
+/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
+ *  Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ *  The dictionary remains valid for all future frames, until explicitly invalidated, or
+ *  a new dictionary is loaded.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ *            meaning "return to no-dictionary mode".
+ *  Note 1 : Loading a dictionary involves building tables,
+ *           which has a non-negligible impact on CPU usage and latency.
+ *           It's recommended to "load once, use many times", to amortize the cost
+ *  Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
+ *           Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
+ *  Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
+ *           how dictionary content is loaded and interpreted.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
+ *  Reference a prepared dictionary, to be used to decompress next frames.
+ *  The dictionary remains active for decompression of future frames using same DCtx.
+ *
+ *  If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
+ *  will store the DDict references in a table, and the DDict used for decompression
+ *  will be determined at decompression time, as per the dict ID in the frame.
+ *  The memory for the table is allocated on the first call to refDDict, and can be
+ *  freed with ZSTD_freeDCtx().
+ *
+ *  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ *  will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: referencing a NULL DDict means "return to no-dictionary mode".
+ *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
+ *  Reference a prefix (single-usage dictionary) to decompress next frame.
+ *  This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ *  and must use the same prefix as the one used during compression.
+ *  Prefix is **only used once**. Reference is discarded at end of frame.
+ *  End of frame is reached when ZSTD_decompressStream() returns 0.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ *  Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
+ *           Prefix buffer must remain unmodified up to the end of frame,
+ *           reached when ZSTD_decompressStream() returns 0.
+ *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
+ *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
+ *  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ *           A full dictionary is more costly, as it requires building tables.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
+                                 const void* prefix, size_t prefixSize);
+
+/* ===   Memory management   === */
+
+/*! ZSTD_sizeof_*() : Requires v1.4.0+
+ *  These functions give the _current_ memory usage of selected object.
+ *  Note that object memory usage can evolve (increase or decrease) over time. */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+#endif  /* ZSTD_H_235446 */
+
+
+/* **************************************************************************************
+ *   ADVANCED AND EXPERIMENTAL FUNCTIONS
+ ****************************************************************************************
+ * The definitions in the following section are considered experimental.
+ * They are provided for advanced scenarios.
+ * They should never be used with a dynamic library, as prototypes may change in the future.
+ * Use them only in association with static linking.
+ * ***************************************************************************************/
+
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
+#include <limits.h>   /* INT_MAX */
+
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZSTDLIB_STATIC_API
+#  if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#    define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
+#  elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#    define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
+#  else
+#    define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
+#  endif
+#endif
+
+/****************************************************************************************
+ *   experimental API (static linking only)
+ ****************************************************************************************
+ * The following symbols and constants
+ * are not planned to join "stable API" status in the near future.
+ * They can still change in future versions.
+ * Some of them are planned to remain in the static_only section indefinitely.
+ * Some of them might be removed in the future (especially when redundant with existing stable functions)
+ * ***************************************************************************************/
+
+#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1)   /* minimum input size required to query frame header size */
+#define ZSTD_FRAMEHEADERSIZE_MIN(format)    ((format) == ZSTD_f_zstd1 ? 6 : 2)
+#define ZSTD_FRAMEHEADERSIZE_MAX   18   /* can be useful for static allocation */
+#define ZSTD_SKIPPABLEHEADERSIZE    8
+
+/* compression parameter bounds */
+#define ZSTD_WINDOWLOG_MAX_32    30
+#define ZSTD_WINDOWLOG_MAX_64    31
+#define ZSTD_WINDOWLOG_MAX     ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MIN       10
+#define ZSTD_HASHLOG_MAX       ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
+#define ZSTD_HASHLOG_MIN          6
+#define ZSTD_CHAINLOG_MAX_32     29
+#define ZSTD_CHAINLOG_MAX_64     30
+#define ZSTD_CHAINLOG_MAX      ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
+#define ZSTD_CHAINLOG_MIN        ZSTD_HASHLOG_MIN
+#define ZSTD_SEARCHLOG_MAX      (ZSTD_WINDOWLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN        1
+#define ZSTD_MINMATCH_MAX         7   /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_MINMATCH_MIN         3   /* only for ZSTD_btopt+, faster strategies are limited to 4 */
+#define ZSTD_TARGETLENGTH_MAX    ZSTD_BLOCKSIZE_MAX
+#define ZSTD_TARGETLENGTH_MIN     0   /* note : comparing this constant to an unsigned results in a tautological test */
+#define ZSTD_STRATEGY_MIN        ZSTD_fast
+#define ZSTD_STRATEGY_MAX        ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
+
+
+#define ZSTD_OVERLAPLOG_MIN       0
+#define ZSTD_OVERLAPLOG_MAX       9
+
+#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27   /* by default, the streaming decoder will refuse any frame
+                                           * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
+                                           * to preserve host's memory from unreasonable requirements.
+                                           * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
+                                           * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
+
+
+/* LDM parameter bounds */
+#define ZSTD_LDM_HASHLOG_MIN      ZSTD_HASHLOG_MIN
+#define ZSTD_LDM_HASHLOG_MAX      ZSTD_HASHLOG_MAX
+#define ZSTD_LDM_MINMATCH_MIN        4
+#define ZSTD_LDM_MINMATCH_MAX     4096
+#define ZSTD_LDM_BUCKETSIZELOG_MIN   1
+#define ZSTD_LDM_BUCKETSIZELOG_MAX   8
+#define ZSTD_LDM_HASHRATELOG_MIN     0
+#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
+
+/* Advanced parameter bounds */
+#define ZSTD_TARGETCBLOCKSIZE_MIN   1340 /* suitable to fit into an ethernet / wifi / 4G transport frame */
+#define ZSTD_TARGETCBLOCKSIZE_MAX   ZSTD_BLOCKSIZE_MAX
+#define ZSTD_SRCSIZEHINT_MIN        0
+#define ZSTD_SRCSIZEHINT_MAX        INT_MAX
+
+
+/* ---  Advanced types  --- */
+
+typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
+
+typedef struct {
+    unsigned int offset;      /* The offset of the match. (NOT the same as the offset code)
+                               * If offset == 0 and matchLength == 0, this sequence represents the last
+                               * literals in the block of litLength size.
+                               */
+
+    unsigned int litLength;   /* Literal length of the sequence. */
+    unsigned int matchLength; /* Match length of the sequence. */
+
+                              /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
+                               * In this case, we will treat the sequence as a marker for a block boundary.
+                               */
+
+    unsigned int rep;         /* Represents which repeat offset is represented by the field 'offset'.
+                               * Ranges from [0, 3].
+                               *
+                               * Repeat offsets are essentially previous offsets from previous sequences sorted in
+                               * recency order. For more detail, see doc/zstd_compression_format.md
+                               *
+                               * If rep == 0, then 'offset' does not contain a repeat offset.
+                               * If rep > 0:
+                               *  If litLength != 0:
+                               *      rep == 1 --> offset == repeat_offset_1
+                               *      rep == 2 --> offset == repeat_offset_2
+                               *      rep == 3 --> offset == repeat_offset_3
+                               *  If litLength == 0:
+                               *      rep == 1 --> offset == repeat_offset_2
+                               *      rep == 2 --> offset == repeat_offset_3
+                               *      rep == 3 --> offset == repeat_offset_1 - 1
+                               *
+                               * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
+                               * 'rep', but repeat offsets do not necessarily need to be calculated from an external
+                               * sequence provider's perspective. For example, ZSTD_compressSequences() does not
+                               * use this 'rep' field at all (as of now).
+                               */
+} ZSTD_Sequence;
+
+typedef struct {
+    unsigned windowLog;       /**< largest match distance : larger == more compression, more memory needed during decompression */
+    unsigned chainLog;        /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+    unsigned hashLog;         /**< dispatch table : larger == faster, more memory */
+    unsigned searchLog;       /**< nb of searches : larger == more compression, slower */
+    unsigned minMatch;        /**< match length searched : larger == faster decompression, sometimes less compression */
+    unsigned targetLength;    /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+    ZSTD_strategy strategy;   /**< see ZSTD_strategy definition above */
+} ZSTD_compressionParameters;
+
+typedef struct {
+    int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+    int checksumFlag;    /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
+    int noDictIDFlag;    /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
+} ZSTD_frameParameters;
+
+typedef struct {
+    ZSTD_compressionParameters cParams;
+    ZSTD_frameParameters fParams;
+} ZSTD_parameters;
+
+typedef enum {
+    ZSTD_dct_auto = 0,       /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
+    ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
+    ZSTD_dct_fullDict = 2    /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
+} ZSTD_dictContentType_e;
+
+typedef enum {
+    ZSTD_dlm_byCopy = 0,  /**< Copy dictionary content internally */
+    ZSTD_dlm_byRef = 1    /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
+} ZSTD_dictLoadMethod_e;
+
+typedef enum {
+    ZSTD_f_zstd1 = 0,           /* zstd frame format, specified in zstd_compression_format.md (default) */
+    ZSTD_f_zstd1_magicless = 1  /* Variant of zstd frame format, without initial 4-bytes magic number.
+                                 * Useful to save 4 bytes per generated frame.
+                                 * Decoder cannot recognise automatically this format, requiring this instruction. */
+} ZSTD_format_e;
+
+typedef enum {
+    /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
+    ZSTD_d_validateChecksum = 0,
+    ZSTD_d_ignoreChecksum = 1
+} ZSTD_forceIgnoreChecksum_e;
+
+typedef enum {
+    /* Note: this enum controls ZSTD_d_refMultipleDDicts */
+    ZSTD_rmd_refSingleDDict = 0,
+    ZSTD_rmd_refMultipleDDicts = 1
+} ZSTD_refMultipleDDicts_e;
+
+typedef enum {
+    /* Note: this enum and the behavior it controls are effectively internal
+     * implementation details of the compressor. They are expected to continue
+     * to evolve and should be considered only in the context of extremely
+     * advanced performance tuning.
+     *
+     * Zstd currently supports the use of a CDict in three ways:
+     *
+     * - The contents of the CDict can be copied into the working context. This
+     *   means that the compression can search both the dictionary and input
+     *   while operating on a single set of internal tables. This makes
+     *   the compression faster per-byte of input. However, the initial copy of
+     *   the CDict's tables incurs a fixed cost at the beginning of the
+     *   compression. For small compressions (< 8 KB), that copy can dominate
+     *   the cost of the compression.
+     *
+     * - The CDict's tables can be used in-place. In this model, compression is
+     *   slower per input byte, because the compressor has to search two sets of
+     *   tables. However, this model incurs no start-up cost (as long as the
+     *   working context's tables can be reused). For small inputs, this can be
+     *   faster than copying the CDict's tables.
+     *
+     * - The CDict's tables are not used at all, and instead we use the working
+     *   context alone to reload the dictionary and use params based on the source
+     *   size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+     *   This method is effective when the dictionary sizes are very small relative
+     *   to the input size, and the input size is fairly large to begin with.
+     *
+     * Zstd has a simple internal heuristic that selects which strategy to use
+     * at the beginning of a compression. However, if experimentation shows that
+     * Zstd is making poor choices, it is possible to override that choice with
+     * this enum.
+     */
+    ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
+    ZSTD_dictForceAttach   = 1, /* Never copy the dictionary. */
+    ZSTD_dictForceCopy     = 2, /* Always copy the dictionary. */
+    ZSTD_dictForceLoad     = 3  /* Always reload the dictionary */
+} ZSTD_dictAttachPref_e;
+
+typedef enum {
+  ZSTD_lcm_auto = 0,          /**< Automatically determine the compression mode based on the compression level.
+                               *   Negative compression levels will be uncompressed, and positive compression
+                               *   levels will be compressed. */
+  ZSTD_lcm_huffman = 1,       /**< Always attempt Huffman compression. Uncompressed literals will still be
+                               *   emitted if Huffman compression is not profitable. */
+  ZSTD_lcm_uncompressed = 2   /**< Always emit uncompressed literals. */
+} ZSTD_literalCompressionMode_e;
+
+typedef enum {
+  /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
+   * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
+   * or ZSTD_ps_disable allow for a force enable/disable the feature.
+   */
+  ZSTD_ps_auto = 0,         /* Let the library automatically determine whether the feature shall be enabled */
+  ZSTD_ps_enable = 1,       /* Force-enable the feature */
+  ZSTD_ps_disable = 2       /* Do not use the feature */
+} ZSTD_paramSwitch_e;
+
+/***************************************
+*  Frame header and size functions
+***************************************/
+
+/*! ZSTD_findDecompressedSize() :
+ *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ *  `srcSize` must be the _exact_ size of this series
+ *       (i.e. there should be a frame boundary at `src + srcSize`)
+ *  @return : - decompressed size of all data in all successive frames
+ *            - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+ *            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ *   note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ *            When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ *            In which case, it's necessary to use streaming mode to decompress data.
+ *   note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ *   note 3 : decompressed size can be very large (64-bits value),
+ *            potentially larger than what local system can handle as a single memory segment.
+ *            In which case, it's necessary to use streaming mode to decompress data.
+ *   note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ *            Always ensure result fits within application's authorized limits.
+ *            Each application can set its own limits.
+ *   note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+ *            read each contained frame header.  This is fast as most of the data is skipped,
+ *            however it does mean that all frame data must be present and valid. */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_decompressBound() :
+ *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ *  `srcSize` must be the _exact_ size of this series
+ *       (i.e. there should be a frame boundary at `src + srcSize`)
+ *  @return : - upper-bound for the decompressed size of all data in all successive frames
+ *            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ *  note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame.
+ *  note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
+ *            in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
+ *  note 3  : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
+ *              upper-bound = # blocks * min(128 KB, Window_Size)
+ */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
+
+/*! ZSTD_frameHeaderSize() :
+ *  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
+ * @return : size of the Frame Header,
+ *           or an error code (if srcSize is too small) */
+ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+
+typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
+typedef struct {
+    unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
+    unsigned long long windowSize;       /* can be very large, up to <= frameContentSize */
+    unsigned blockSizeMax;
+    ZSTD_frameType_e frameType;          /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
+    unsigned headerSize;
+    unsigned dictID;
+    unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
+} ZSTD_frameHeader;
+
+/*! ZSTD_getFrameHeader() :
+ *  decode Frame Header, or requires larger `srcSize`.
+ * @return : 0, `zfhPtr` is correctly filled,
+ *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ *           or an error code, which can be tested using ZSTD_isError() */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
+/*! ZSTD_getFrameHeader_advanced() :
+ *  same as ZSTD_getFrameHeader(),
+ *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+
+/*! ZSTD_decompressionMargin() :
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ * and the input buffer must be at the end of the output buffer.
+ *
+ *  _______________________ Output Buffer ________________________
+ * |                                                              |
+ * |                                        ____ Input Buffer ____|
+ * |                                       |                      |
+ * v                                       v                      v
+ * |---------------------------------------|-----------|----------|
+ * ^                                                   ^          ^
+ * |___________________ Output_Size ___________________|_ Margin _|
+ *
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ * ZSTD_decompressDCtx().
+ * NOTE: This function supports multi-frame input.
+ *
+ * @param src The compressed frame(s)
+ * @param srcSize The size of the compressed frame(s)
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
+/*! ZSTD_DECOMPRESS_MARGIN() :
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ * the compressed frame, compute it from the original size and the blockSizeLog.
+ * See ZSTD_decompressionMargin() for details.
+ *
+ * WARNING: This macro does not support multi-frame input, the input must be a single
+ * zstd frame. If you need that support use the function, or implement it yourself.
+ *
+ * @param originalSize The original uncompressed size of the data.
+ * @param blockSize    The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+ *                     Unless you explicitly set the windowLog smaller than
+ *                     ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ */
+#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)(                                              \
+        ZSTD_FRAMEHEADERSIZE_MAX                                                              /* Frame header */ + \
+        4                                                                                         /* checksum */ + \
+        ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+        (blockSize)                                                                    /* One block of margin */   \
+    ))
+
+typedef enum {
+  ZSTD_sf_noBlockDelimiters = 0,         /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
+  ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
+} ZSTD_sequenceFormat_e;
+
+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ *  @return : upper-bound for the number of sequences that can be generated
+ *            from a buffer of srcSize bytes
+ *
+ *  note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
+/*! ZSTD_generateSequences() :
+ * WARNING: This function is meant for debugging and informational purposes ONLY!
+ * Its implementation is flawed, and it will be deleted in a future version.
+ * It is not guaranteed to succeed, as there are several cases where it will give
+ * up and fail. You should NOT use this function in production code.
+ *
+ * This function is deprecated, and will be removed in a future version.
+ *
+ * Generate sequences using ZSTD_compress2(), given a source buffer.
+ *
+ * @param zc The compression context to be used for ZSTD_compress2(). Set any
+ *           compression parameters you need on this context.
+ * @param outSeqs The output sequences buffer of size @p outSeqsSize
+ * @param outSeqsSize The size of the output sequences buffer.
+ *                    ZSTD_sequenceBound(srcSize) is an upper bound on the number
+ *                    of sequences that can be generated.
+ * @param src The source buffer to generate sequences from of size @p srcSize.
+ * @param srcSize The size of the source buffer.
+ *
+ * Each block will end with a dummy sequence
+ * with offset == 0, matchLength == 0, and litLength == length of last literals.
+ * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
+ * simply acts as a block delimiter.
+ *
+ * @returns The number of sequences generated, necessarily less than
+ *          ZSTD_sequenceBound(srcSize), or an error code that can be checked
+ *          with ZSTD_isError().
+ */
+ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
+ZSTDLIB_STATIC_API size_t
+ZSTD_generateSequences(ZSTD_CCtx* zc,
+                       ZSTD_Sequence* outSeqs, size_t outSeqsSize,
+                       const void* src, size_t srcSize);
+
+/*! ZSTD_mergeBlockDelimiters() :
+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
+ * by merging them into the literals of the next sequence.
+ *
+ * As such, the final generated result has no explicit representation of block boundaries,
+ * and the final last literals segment is not represented in the sequences.
+ *
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
+ * @return : number of sequences left after merging
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
+
+/*! ZSTD_compressSequences() :
+ * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
+ * @src contains the entire input (not just the literals).
+ * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
+ * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
+ * The entire source is compressed into a single frame.
+ *
+ * The compression behavior changes based on cctx params. In particular:
+ *    If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ *    no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
+ *    the block size derived from the cctx, and sequences may be split. This is the default setting.
+ *
+ *    If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ *    block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
+ *
+ *    If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
+ *    behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
+ *    specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
+ *
+ *    In addition to the two adjustable experimental params, there are other important cctx params.
+ *    - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
+ *    - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
+ *    - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
+ *      is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
+ *
+ * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
+ * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
+ *         and cannot emit an RLE block that disagrees with the repcode history
+ * @return : final compressed size, or a ZSTD error code.
+ */
+ZSTDLIB_STATIC_API size_t
+ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
+                        const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+                        const void* src, size_t srcSize);
+
+
+/*! ZSTD_writeSkippableFrame() :
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, if the source size is not representable
+ * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+                                            const void* src, size_t srcSize, unsigned magicVariant);
+
+/*! ZSTD_readSkippableFrame() :
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
+ * in the magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
+                                            const void* src, size_t srcSize);
+
+/*! ZSTD_isSkippableFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ */
+ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
+
+
+
+/***************************************
+*  Memory management
+***************************************/
+
+/*! ZSTD_estimate*() :
+ *  These functions make it possible to estimate memory usage
+ *  of a future {D,C}Ctx, before its creation.
+ *  This is useful in combination with ZSTD_initStatic(),
+ *  which makes it possible to employ a static buffer for ZSTD_CCtx* state.
+ *
+ *  ZSTD_estimateCCtxSize() will provide a memory budget large enough
+ *  to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2()
+ *  associated with any compression level up to max specified one.
+ *  The estimate will assume the input may be arbitrarily large,
+ *  which is the worst case.
+ *
+ *  Note that the size estimation is specific for one-shot compression,
+ *  it is not valid for streaming (see ZSTD_estimateCStreamSize*())
+ *  nor other potential ways of using a ZSTD_CCtx* state.
+ *
+ *  When srcSize can be bound by a known and rather "small" value,
+ *  this knowledge can be used to provide a tighter budget estimation
+ *  because the ZSTD_CCtx* state will need less memory for small inputs.
+ *  This tighter estimation can be provided by employing more advanced functions
+ *  ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
+ *  and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
+ *  Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
+ *
+ *  Note : only single-threaded compression is supported.
+ *  ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
+
+/*! ZSTD_estimateCStreamSize() :
+ *  ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression
+ *  using any compression level up to the max specified one.
+ *  It will also consider src size to be arbitrarily "large", which is a worst case scenario.
+ *  If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
+ *  ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  Note : CStream size estimation is only correct for single-threaded compression.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ *  Size estimates assume that no external sequence producer is registered.
+ *
+ *  ZSTD_DStream memory budget depends on frame's window Size.
+ *  This information can be passed manually, using ZSTD_estimateDStreamSize,
+ *  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
+ *  Any frame requesting a window size larger than max specified one will be rejected.
+ *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+ *         an internal ?Dict will be created, which additional size is not estimated here.
+ *         In this case, get total size by adding ZSTD_estimate?DictSize
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
+
+/*! ZSTD_estimate?DictSize() :
+ *  ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+ *  ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
+ *  Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
+
+/*! ZSTD_initStatic*() :
+ *  Initialize an object using a pre-allocated fixed-size buffer.
+ *  workspace: The memory area to emplace the object into.
+ *             Provided pointer *must be 8-bytes aligned*.
+ *             Buffer must outlive object.
+ *  workspaceSize: Use ZSTD_estimate*Size() to determine
+ *                 how large workspace must be to support target scenario.
+ * @return : pointer to object (same address as workspace, just different type),
+ *           or NULL if error (size too small, incorrect alignment, etc.)
+ *  Note : zstd will never resize nor malloc() when using a static buffer.
+ *         If the object requires more memory than available,
+ *         zstd will just error out (typically ZSTD_error_memory_allocation).
+ *  Note 2 : there is no corresponding "free" function.
+ *           Since workspace is allocated externally, it must be freed externally too.
+ *  Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
+ *           into its associated cParams.
+ *  Limitation 1 : currently not compatible with internal dictionary creation, triggered by
+ *                 ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
+ *  Limitation 2 : static cctx currently not compatible with multi-threading.
+ *  Limitation 3 : static dctx is incompatible with legacy support.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx*    ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticCCtx() */
+
+ZSTDLIB_STATIC_API ZSTD_DCtx*    ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticDCtx() */
+
+ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
+                                        void* workspace, size_t workspaceSize,
+                                        const void* dict, size_t dictSize,
+                                        ZSTD_dictLoadMethod_e dictLoadMethod,
+                                        ZSTD_dictContentType_e dictContentType,
+                                        ZSTD_compressionParameters cParams);
+
+ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
+                                        void* workspace, size_t workspaceSize,
+                                        const void* dict, size_t dictSize,
+                                        ZSTD_dictLoadMethod_e dictLoadMethod,
+                                        ZSTD_dictContentType_e dictContentType);
+
+
+/*! Custom memory allocation :
+ *  These prototypes make it possible to pass your own allocation/free functions.
+ *  ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
+ *  All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
+ */
+typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+
+#if defined(__clang__) && __clang_major__ >= 5
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
+#endif
+ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< this constant defers to stdlib's functions */
+#if defined(__clang__) && __clang_major__ >= 5
+#pragma clang diagnostic pop
+#endif
+
+ZSTDLIB_STATIC_API ZSTD_CCtx*    ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DCtx*    ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+                                                  ZSTD_dictLoadMethod_e dictLoadMethod,
+                                                  ZSTD_dictContentType_e dictContentType,
+                                                  ZSTD_compressionParameters cParams,
+                                                  ZSTD_customMem customMem);
+
+/*! Thread pool :
+ *  These prototypes make it possible to share a thread pool among multiple compression contexts.
+ *  This can limit resources for applications with multiple threads where each one uses
+ *  a threaded compression mode (via ZSTD_c_nbWorkers parameter).
+ *  ZSTD_createThreadPool creates a new thread pool with a given number of threads.
+ *  Note that the lifetime of such pool must exist while being used.
+ *  ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
+ *  to use an internal thread pool).
+ *  ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
+ */
+typedef struct POOL_ctx_s ZSTD_threadPool;
+ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
+ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);  /* accept NULL pointer */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
+
+
+/*
+ * This API is temporary and is expected to change or disappear in the future!
+ */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
+    const void* dict, size_t dictSize,
+    ZSTD_dictLoadMethod_e dictLoadMethod,
+    ZSTD_dictContentType_e dictContentType,
+    const ZSTD_CCtx_params* cctxParams,
+    ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
+    const void* dict, size_t dictSize,
+    ZSTD_dictLoadMethod_e dictLoadMethod,
+    ZSTD_dictContentType_e dictContentType,
+    ZSTD_customMem customMem);
+
+
+/***************************************
+*  Advanced compression functions
+***************************************/
+
+/*! ZSTD_createCDict_byReference() :
+ *  Create a digested dictionary for compression
+ *  Dictionary content is just referenced, not duplicated.
+ *  As a consequence, `dictBuffer` **must** outlive CDict,
+ *  and its content must remain unmodified throughout the lifetime of CDict.
+ *  note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
+/*! ZSTD_getCParams() :
+ * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+ * `estimatedSrcSize` value is optional, select 0 if not known */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_getParams() :
+ *  same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+ *  All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
+ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_checkCParams() :
+ *  Ensure param values remain within authorized range.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+
+/*! ZSTD_adjustCParams() :
+ *  optimize params for a given `srcSize` and `dictSize`.
+ * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
+ * `dictSize` must be `0` when there is no dictionary.
+ *  cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
+ *  This function never fails (wide contract) */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+
+/*! ZSTD_CCtx_setCParams() :
+ *  Set all parameters provided within @p cparams into the working @p cctx.
+ *  Note : if modifying parameters during compression (MT mode only),
+ *         note that changes to the .windowLog parameter will be ignored.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ *         On failure, no parameters are updated.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
+/*! ZSTD_CCtx_setFParams() :
+ *  Set all parameters provided within @p fparams into the working @p cctx.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
+
+/*! ZSTD_CCtx_setParams() :
+ *  Set all parameters provided within @p params into the working @p cctx.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
+
+/*! ZSTD_compress_advanced() :
+ *  Note : this function is now DEPRECATED.
+ *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
+ *  This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+                              void* dst, size_t dstCapacity,
+                        const void* src, size_t srcSize,
+                        const void* dict,size_t dictSize,
+                              ZSTD_parameters params);
+
+/*! ZSTD_compress_usingCDict_advanced() :
+ *  Note : this function is now DEPRECATED.
+ *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
+ *  This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+                                              void* dst, size_t dstCapacity,
+                                        const void* src, size_t srcSize,
+                                        const ZSTD_CDict* cdict,
+                                              ZSTD_frameParameters fParams);
+
+
+/*! ZSTD_CCtx_loadDictionary_byReference() :
+ *  Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
+ *  It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_loadDictionary_advanced() :
+ *  Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
+ *  how to load the dictionary (by copy ? by reference ?)
+ *  and how to interpret it (automatic ? force raw mode ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_CCtx_refPrefix_advanced() :
+ *  Same as ZSTD_CCtx_refPrefix(), but gives finer control over
+ *  how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/* ===   experimental parameters   === */
+/* these parameters can be used with ZSTD_setParameter()
+ * they are not guaranteed to remain supported in the future */
+
+ /* Enables rsyncable mode,
+  * which makes compressed files more rsync friendly
+  * by adding periodic synchronization points to the compressed data.
+  * The target average block size is ZSTD_c_jobSize / 2.
+  * It's possible to modify the job size to increase or decrease
+  * the granularity of the synchronization point.
+  * Once the jobSize is smaller than the window size,
+  * it will result in compression ratio degradation.
+  * NOTE 1: rsyncable mode only works when multithreading is enabled.
+  * NOTE 2: rsyncable performs poorly in combination with long range mode,
+  * since it will decrease the effectiveness of synchronization points,
+  * though mileage may vary.
+  * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
+  * If the selected compression level is already running significantly slower,
+  * the overall speed won't be significantly impacted.
+  */
+ #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
+
+/* Select a compression format.
+ * The value must be of type ZSTD_format_e.
+ * See ZSTD_format_e enum definition for details */
+#define ZSTD_c_format ZSTD_c_experimentalParam2
+
+/* Force back-reference distances to remain < windowSize,
+ * even when referencing into Dictionary content (default:0) */
+#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
+
+/* Controls whether the contents of a CDict
+ * are used in place, or copied into the working context.
+ * Accepts values from the ZSTD_dictAttachPref_e enum.
+ * See the comments on that enum for an explanation of the feature. */
+#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
+
+/* Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never compress literals.
+ * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
+ * may still be emitted if huffman is not beneficial to use.)
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * literals compression based on the compression parameters - specifically,
+ * negative compression levels do not use literal compression.
+ */
+#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
+
+/* User's best guess of source size.
+ * Hint is not valid when srcSizeHint == 0.
+ * There is no guarantee that hint is close to actual source size,
+ * but compression ratio may regress significantly if guess considerably underestimates */
+#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
+
+/* Controls whether the new and experimental "dedicated dictionary search
+ * structure" can be used. This feature is still rough around the edges, be
+ * prepared for surprising behavior!
+ *
+ * How to use it:
+ *
+ * When using a CDict, whether to use this feature or not is controlled at
+ * CDict creation, and it must be set in a CCtxParams set passed into that
+ * construction (via ZSTD_createCDict_advanced2()). A compression will then
+ * use the feature or not based on how the CDict was constructed; the value of
+ * this param, set in the CCtx, will have no effect.
+ *
+ * However, when a dictionary buffer is passed into a CCtx, such as via
+ * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
+ * whether the CDict that is created internally can use the feature or not.
+ *
+ * What it does:
+ *
+ * Normally, the internal data structures of the CDict are analogous to what
+ * would be stored in a CCtx after compressing the contents of a dictionary.
+ * To an approximation, a compression using a dictionary can then use those
+ * data structures to simply continue what is effectively a streaming
+ * compression where the simulated compression of the dictionary left off.
+ * Which is to say, the search structures in the CDict are normally the same
+ * format as in the CCtx.
+ *
+ * It is possible to do better, since the CDict is not like a CCtx: the search
+ * structures are written once during CDict creation, and then are only read
+ * after that, while the search structures in the CCtx are both read and
+ * written as the compression goes along. This means we can choose a search
+ * structure for the dictionary that is read-optimized.
+ *
+ * This feature enables the use of that different structure.
+ *
+ * Note that some of the members of the ZSTD_compressionParameters struct have
+ * different semantics and constraints in the dedicated search structure. It is
+ * highly recommended that you simply set a compression level in the CCtxParams
+ * you pass into the CDict creation call, and avoid messing with the cParams
+ * directly.
+ *
+ * Effects:
+ *
+ * This will only have any effect when the selected ZSTD_strategy
+ * implementation supports this feature. Currently, that's limited to
+ * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
+ *
+ * Note that this means that the CDict tables can no longer be copied into the
+ * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
+ * usable. The dictionary can only be attached or reloaded.
+ *
+ * In general, you should expect compression to be faster--sometimes very much
+ * so--and CDict creation to be slightly slower. Eventually, we will probably
+ * make this mode the default.
+ */
+#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
+
+/* ZSTD_c_stableInBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the compressor that input data presented with ZSTD_inBuffer
+ * will ALWAYS be the same between calls.
+ * Technically, the @src pointer must never be changed,
+ * and the @pos field can only be updated by zstd.
+ * However, it's possible to increase the @size field,
+ * allowing scenarios where more data can be appended after compressions starts.
+ * These conditions are checked by the compressor,
+ * and compression will fail if they are not respected.
+ * Also, data in the ZSTD_inBuffer within the range [src, src + pos)
+ * MUST not be modified during compression or it will result in data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an input window buffer,
+ * because the user guarantees it can reference the ZSTD_inBuffer until
+ * the frame is complete. But, it will still allocate an output buffer
+ * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
+ * avoid the memcpy() from the input buffer to the input window buffer.
+ *
+ * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, compression WILL fail if conditions are not respected.
+ *
+ * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
+ * not be modified during compression or it will result in data corruption.
+ * This is because zstd needs to reference data in the ZSTD_inBuffer to find
+ * matches. Normally zstd maintains its own window buffer for this purpose,
+ * but passing this flag tells zstd to rely on user provided buffer instead.
+ */
+#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
+
+/* ZSTD_c_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells he compressor that the ZSTD_outBuffer will not be resized between
+ * calls. Specifically: (out.size - out.pos) will never grow. This gives the
+ * compressor the freedom to say: If the compressed data doesn't fit in the
+ * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
+ * always decompress directly into the output buffer, instead of decompressing
+ * into an internal buffer and copying to the output buffer.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer. It will still allocate the
+ * input window buffer (see ZSTD_c_stableInBuffer).
+ *
+ * Zstd will check that (out.size - out.pos) never grows and return an error
+ * if it does. While not strictly necessary, this should prevent surprises.
+ */
+#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
+
+/* ZSTD_c_blockDelimiters
+ * Default is 0 == ZSTD_sf_noBlockDelimiters.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ *
+ * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
+ * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
+ * See the definition of ZSTD_Sequence for more specifics.
+ */
+#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
+
+/* ZSTD_c_validateSequences
+ * Default is 0 == disabled. Set to 1 to enable sequence validation.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
+ * during function execution.
+ *
+ * Without validation, providing a sequence that does not conform to the zstd spec will cause
+ * undefined behavior, and may produce a corrupted block.
+ *
+ * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
+ * specifics regarding offset/matchlength requirements) then the function will bail out and
+ * return an error.
+ *
+ */
+#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
+
+/* ZSTD_c_blockSplitterLevel
+ * note: this parameter only influences the first splitter stage,
+ *       which is active before producing the sequences.
+ *       ZSTD_c_splitAfterSequences controls the next splitter stage,
+ *       which is active after sequence production.
+ *       Note that both can be combined.
+ * Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included.
+ * 0 means "auto", which will select a value depending on current ZSTD_c_strategy.
+ * 1 means no splitting.
+ * Then, values from 2 to 6 are sorted in increasing cpu load order.
+ *
+ * Note that currently the first block is never split,
+ * to ensure expansion guarantees in presence of incompressible data.
+ */
+#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6
+#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20
+
+/* ZSTD_c_splitAfterSequences
+ * This is a stronger splitter algorithm,
+ * based on actual sequences previously produced by the selected parser.
+ * It's also slower, and as a consequence, mostly used for high compression levels.
+ * While the post-splitter does overlap with the pre-splitter,
+ * both can nonetheless be combined,
+ * notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX,
+ * resulting in higher compression ratio than just one of them.
+ *
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use block splitter.
+ * Set to ZSTD_ps_enable to always use block splitter.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * block splitting based on the compression parameters.
+ */
+#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13
+
+/* ZSTD_c_useRowMatchFinder
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use row-based matchfinder.
+ * Set to ZSTD_ps_enable to force usage of row-based matchfinder.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * the row-based matchfinder based on support for SIMD instructions and the window log.
+ * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
+ */
+#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
+
+/* ZSTD_c_deterministicRefPrefix
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Zstd produces different results for prefix compression when the prefix is
+ * directly adjacent to the data about to be compressed vs. when it isn't.
+ * This is because zstd detects that the two buffers are contiguous and it can
+ * use a more efficient match finding algorithm. However, this produces different
+ * results than when the two buffers are non-contiguous. This flag forces zstd
+ * to always load the prefix in non-contiguous mode, even if it happens to be
+ * adjacent to the data, to guarantee determinism.
+ *
+ * If you really care about determinism when using a dictionary or prefix,
+ * like when doing delta compression, you should select this option. It comes
+ * at a speed penalty of about ~2.5% if the dictionary and data happened to be
+ * contiguous, and is free if they weren't contiguous. We don't expect that
+ * intentionally making the dictionary and data contiguous will be worth the
+ * cost to memcpy() the data.
+ */
+#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
+
+/* ZSTD_c_prefetchCDictTables
+ * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
+ *
+ * In some situations, zstd uses CDict tables in-place rather than copying them
+ * into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
+ * In such situations, compression speed is seriously impacted when CDict tables are
+ * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
+ * when they are used in-place.
+ *
+ * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
+ * For sufficiently large inputs, zstd will by default memcpy() CDict tables
+ * into the working context, so there is no need to prefetch. This parameter is
+ * targeted at a middle range of input sizes, where a prefetch is cheap enough to be
+ * useful but memcpy() is too expensive. The exact range of input sizes where this
+ * makes sense is best determined by careful experimentation.
+ *
+ * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
+ * but in the future zstd may conditionally enable this feature via an auto-detection
+ * heuristic for cold CDicts.
+ * Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
+ */
+#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
+
+/* ZSTD_c_enableSeqProducerFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal sequence producer if an
+ * external sequence producer is registered and returns an error code. This fallback
+ * is block-by-block: the internal sequence producer will only be called for blocks
+ * where the external sequence producer returns an error code. Fallback parsing will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full Block-Level Sequence Producer API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
+
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() inaccurate). Only currently meant to be used for testing.
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
+/* ZSTD_c_searchForExternalRepcodes
+ * This parameter affects how zstd parses external sequences, such as sequences
+ * provided through the compressSequences() API or from an external block-level
+ * sequence producer.
+ *
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
+ * external sequences, even if those repcodes are not explicitly indicated in
+ * the "rep" field. Note that this is the only way to exploit repcode matches
+ * while using compressSequences() or an external sequence producer, since zstd
+ * currently ignores the "rep" field of external sequences.
+ *
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
+ * external sequences, regardless of whether the "rep" field has been set. This
+ * reduces sequence compression overhead by about 25% while sacrificing some
+ * compression ratio.
+ *
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
+ * based on compression level.
+ *
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
+ */
+#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
+
+
+/*! ZSTD_CCtx_getParameter() :
+ *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
+ *  and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+
+
+/*! ZSTD_CCtx_params :
+ *  Quick howto :
+ *  - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
+ *  - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
+ *                                     an existing ZSTD_CCtx_params structure.
+ *                                     This is similar to
+ *                                     ZSTD_CCtx_setParameter().
+ *  - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+ *                                    an existing CCtx.
+ *                                    These parameters will be applied to
+ *                                    all subsequent frames.
+ *  - ZSTD_compressStream2() : Do compression using the CCtx.
+ *  - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
+ *
+ *  This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+ *  for static allocation of CCtx for single-threaded compression.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  /* accept NULL pointer */
+
+/*! ZSTD_CCtxParams_reset() :
+ *  Reset params to default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+
+/*! ZSTD_CCtxParams_init() :
+ *  Initializes the compression parameters of cctxParams according to
+ *  compression level. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+
+/*! ZSTD_CCtxParams_init_advanced() :
+ *  Initializes the compression and frame parameters of cctxParams according to
+ *  params. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+
+/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
+ *  Similar to ZSTD_CCtx_setParameter.
+ *  Set one compression parameter, selected by enum ZSTD_cParameter.
+ *  Parameters must be applied to a ZSTD_CCtx using
+ *  ZSTD_CCtx_setParametersUsingCCtxParams().
+ * @result : a code representing success or failure (which can be tested with
+ *           ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtxParams_getParameter() :
+ * Similar to ZSTD_CCtx_getParameter.
+ * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+
+/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
+ *  Apply a set of ZSTD_CCtx_params to the compression context.
+ *  This can be done even after compression is started,
+ *    if nbWorkers==0, this will have no impact until a new compression is started.
+ *    if nbWorkers>=1, new parameters will be picked up at next job,
+ *       with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+        ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
+
+/*! ZSTD_compressStream2_simpleArgs() :
+ *  Same as ZSTD_compressStream2(),
+ *  but using only integral types as arguments.
+ *  This variant might be helpful for binders from dynamic languages
+ *  which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
+                            ZSTD_CCtx* cctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos,
+                            ZSTD_EndDirective endOp);
+
+
+/***************************************
+*  Advanced decompression functions
+***************************************/
+
+/*! ZSTD_isFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier.
+ *  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ *  Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ *  Note 3 : Skippable Frame Identifiers are considered valid. */
+ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
+
+/*! ZSTD_createDDict_byReference() :
+ *  Create a digested dictionary, ready to start decompression operation without startup delay.
+ *  Dictionary content is referenced, and therefore stays in dictBuffer.
+ *  It is important that dictBuffer outlives DDict,
+ *  it must remain read accessible throughout the lifetime of DDict */
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_byReference() :
+ *  Same as ZSTD_DCtx_loadDictionary(),
+ *  but references `dict` content instead of copying it into `dctx`.
+ *  This saves memory if `dict` remains around.,
+ *  However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_advanced() :
+ *  Same as ZSTD_DCtx_loadDictionary(),
+ *  but gives direct control over
+ *  how to load the dictionary (by copy ? by reference ?)
+ *  and how to interpret it (automatic ? force raw mode ? full mode only ?). */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_refPrefix_advanced() :
+ *  Same as ZSTD_DCtx_refPrefix(), but gives finer control over
+ *  how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_setMaxWindowSize() :
+ *  Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+ *  This protects a decoder context from reserving too much memory for itself (potential attack scenario).
+ *  This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ *  By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+
+/*! ZSTD_DCtx_getParameter() :
+ *  Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
+ *  and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
+
+/* ZSTD_d_format
+ * experimental parameter,
+ * allowing selection between ZSTD_format_e input compression formats
+ */
+#define ZSTD_d_format ZSTD_d_experimentalParam1
+/* ZSTD_d_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
+ * between calls, except for the modifications that zstd makes to pos (the
+ * caller must not modify pos). This is checked by the decompressor, and
+ * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
+ * MUST be large enough to fit the entire decompressed frame. This will be
+ * checked when the frame content size is known. The data in the ZSTD_outBuffer
+ * in the range [dst, dst + pos) MUST not be modified during decompression
+ * or you will get data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer, but it will still allocate
+ * an input buffer large enough to fit any compressed block. This will also
+ * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
+ * If you need to avoid the input buffer allocation use the buffer-less
+ * streaming API.
+ *
+ * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, decompression WILL fail if you violate the preconditions.
+ *
+ * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
+ * not be modified during decompression or you will get data corruption. This
+ * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
+ * matches. Normally zstd maintains its own buffer for this purpose, but passing
+ * this flag tells zstd to use the user provided buffer.
+ */
+#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
+
+/* ZSTD_d_forceIgnoreChecksum
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * Tells the decompressor to skip checksum validation during decompression, regardless
+ * of whether checksumming was specified during compression. This offers some
+ * slight performance benefits, and may be useful for debugging.
+ * Param has values of type ZSTD_forceIgnoreChecksum_e
+ */
+#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
+
+/* ZSTD_d_refMultipleDDicts
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * If enabled and dctx is allocated on the heap, then additional memory will be allocated
+ * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
+ * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
+ * store all references. At decompression time, the appropriate dictID is selected
+ * from the set of DDicts based on the dictID in the frame.
+ *
+ * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
+ *
+ * Param has values of byte ZSTD_refMultipleDDicts_e
+ *
+ * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
+ * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
+ * Memory is allocated as per ZSTD_DCtx::customMem.
+ *
+ * Although this function allocates memory for the table, the user is still responsible for
+ * memory management of the underlying ZSTD_DDict* themselves.
+ */
+#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
+
+/* ZSTD_d_disableHuffmanAssembly
+ * Set to 1 to disable the Huffman assembly implementation.
+ * The default value is 0, which allows zstd to use the Huffman assembly
+ * implementation if available.
+ *
+ * This parameter can be used to disable Huffman assembly at runtime.
+ * If you want to disable it at compile time you can define the macro
+ * ZSTD_DISABLE_ASM.
+ */
+#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
+
+/* ZSTD_d_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * Forces the decompressor to reject blocks whose content size is
+ * larger than the configured maxBlockSize. When maxBlockSize is
+ * larger than the windowSize, the windowSize is used instead.
+ * This saves memory on the decoder when you know all blocks are small.
+ *
+ * This option is typically used in conjunction with ZSTD_c_maxBlockSize.
+ *
+ * WARNING: This causes the decoder to reject otherwise valid frames
+ * that have block sizes larger than the configured maxBlockSize.
+ */
+#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6
+
+
+/*! ZSTD_DCtx_setFormat() :
+ *  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
+ *  Instruct the decoder context about what kind of data to decode next.
+ *  This instruction is mandatory to decode data without a fully-formed header,
+ *  such ZSTD_f_zstd1_magicless for example.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
+ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+
+/*! ZSTD_decompressStream_simpleArgs() :
+ *  Same as ZSTD_decompressStream(),
+ *  but using only integral types as arguments.
+ *  This can be helpful for binders from dynamic languages
+ *  which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
+                            ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos);
+
+
+/********************************************************************
+*  Advanced streaming functions
+*  Warning : most of these functions are now redundant with the Advanced API.
+*  Once Advanced API reaches "stable" status,
+*  redundant functions will be deprecated, and then at some point removed.
+********************************************************************/
+
+/*=====   Advanced Streaming compression functions  =====*/
+
+/*! ZSTD_initCStream_srcSize() :
+ * This function is DEPRECATED, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *
+ * pledgedSrcSize must be correct. If it is not known at init time, use
+ * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
+ * "0" also disables frame content size field. It may be enabled in the future.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+                         int compressionLevel,
+                         unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingDict() :
+ * This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * Creates of an internal CDict (incompatible with static CCtx), except if
+ * dict == NULL or dictSize < 8, in which case no dict is used.
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
+ * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+                     const void* dict, size_t dictSize,
+                           int compressionLevel);
+
+/*! ZSTD_initCStream_advanced() :
+ * This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setParams(zcs, params);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
+ * pledgedSrcSize must be correct.
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+                    const void* dict, size_t dictSize,
+                          ZSTD_parameters params,
+                          unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingCDict() :
+ * This function is DEPRECATED, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * note : cdict will just be referenced, and must outlive compression session
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+
+/*! ZSTD_initCStream_usingCDict_advanced() :
+ *   This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setFParams(zcs, fParams);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+                               const ZSTD_CDict* cdict,
+                                     ZSTD_frameParameters fParams,
+                                     unsigned long long pledgedSrcSize);
+
+/*! ZSTD_resetCStream() :
+ * This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
+ *       ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
+ *       explicitly specified.
+ *
+ *  start a new frame, using same parameters from previous frame.
+ *  This is typically useful to skip dictionary loading stage, since it will reuse it in-place.
+ *  Note that zcs must be init at least once before using ZSTD_resetCStream().
+ *  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
+ *  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
+ *  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ *  but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError())
+ *  This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+
+
+typedef struct {
+    unsigned long long ingested;   /* nb input bytes read and buffered */
+    unsigned long long consumed;   /* nb input bytes actually compressed */
+    unsigned long long produced;   /* nb of compressed bytes generated and buffered */
+    unsigned long long flushed;    /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
+    unsigned currentJobID;         /* MT only : latest started job nb */
+    unsigned nbActiveWorkers;      /* MT only : nb of workers actively compressing at probe time */
+} ZSTD_frameProgression;
+
+/* ZSTD_getFrameProgression() :
+ * tells how much data has been ingested (read from input)
+ * consumed (input actually compressed) and produced (output) for current frame.
+ * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
+ * Aggregates progression inside active worker threads.
+ */
+ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
+
+/*! ZSTD_toFlushNow() :
+ *  Tell how many bytes are ready to be flushed immediately.
+ *  Useful for multithreading scenarios (nbWorkers >= 1).
+ *  Probe the oldest active job, defined as oldest job not yet entirely flushed,
+ *  and check its output buffer.
+ * @return : amount of data stored in oldest job and ready to be flushed immediately.
+ *  if @return == 0, it means either :
+ *  + there is no active job (could be checked with ZSTD_frameProgression()), or
+ *  + oldest job is still actively compressing data,
+ *    but everything it has produced has also been flushed so far,
+ *    therefore flush speed is limited by production speed of oldest job
+ *    irrespective of the speed of concurrent (and newer) jobs.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
+
+
+/*=====   Advanced Streaming decompression functions  =====*/
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
+ *
+ * note: no dictionary will be used if dict == NULL or dictSize < 8
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_refDDict(zds, ddict);
+ *
+ * note : ddict is referenced, it must outlive decompression session
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *
+ * reuse decompression parameters from previous init; saves dictionary loading
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+
+
+/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
+ *
+ * *** OVERVIEW ***
+ * The Block-Level Sequence Producer API allows users to provide their own custom
+ * sequence producer which libzstd invokes to process each block. The produced list
+ * of sequences (literals and matches) is then post-processed by libzstd to produce
+ * valid compressed blocks.
+ *
+ * This block-level offload API is a more granular complement of the existing
+ * frame-level offload API compressSequences() (introduced in v1.5.1). It offers
+ * an easier migration story for applications already integrated with libzstd: the
+ * user application continues to invoke the same compression functions
+ * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
+ * from the specific advantages of the external sequence producer. For example,
+ * the sequence producer could be tuned to take advantage of known characteristics
+ * of the input, to offer better speed / ratio, or could leverage hardware
+ * acceleration not available within libzstd itself.
+ *
+ * See contrib/externalSequenceProducer for an example program employing the
+ * Block-Level Sequence Producer API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_sequenceProducer_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ *   - sequenceProducerState: a pointer to a user-managed state for the sequence
+ *     producer.
+ *
+ *   - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
+ *     outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
+ *     backing outSeqs is managed by the CCtx.
+ *
+ *   - src, srcSize: an input buffer for the sequence producer to parse.
+ *     srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ *   - dict, dictSize: a history buffer, which may be empty, which the sequence
+ *     producer may reference as it parses the src buffer. Currently, zstd will
+ *     always pass dictSize == 0 into external sequence producers, but this will
+ *     change in the future.
+ *
+ *   - compressionLevel: a signed integer representing the zstd compression level
+ *     set by the user for the current operation. The sequence producer may choose
+ *     to use this information to change its compression strategy and speed/ratio
+ *     tradeoff. Note: the compression level does not reflect zstd parameters set
+ *     through the advanced API.
+ *
+ *   - windowSize: a size_t representing the maximum allowed offset for external
+ *     sequences. Note that sequence offsets are sometimes allowed to exceed the
+ *     windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ *     for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ *   - The sum of matchLengths and literalLengths must equal srcSize.
+ *   - All sequences in the parse, except for the final sequence, must have
+ *     matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
+ *     matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
+ *   - All offsets must respect the windowSize parameter as specified in
+ *     doc/zstd_compression_format.md.
+ *   - If the final sequence has matchLength == 0, it must also have offset == 0.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal sequence producer or fail the compression operation. The user can
+ * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
+ * cParam. Fallback compression will follow any other cParam settings, such as
+ * compression level, the same as in a normal compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
+ * function by calling
+ *         ZSTD_registerSequenceProducer(cctx,
+ *                                       sequenceProducerState,
+ *                                       sequenceProducer)
+ * This setting will persist until the next parameter reset of the CCtx.
+ *
+ * The sequenceProducerState must be initialized by the user before calling
+ * ZSTD_registerSequenceProducer(). The user is responsible for destroying the
+ * sequenceProducerState.
+ *
+ * *** LIMITATIONS ***
+ * This API is compatible with all zstd compression APIs which respect advanced parameters.
+ * However, there are three limitations:
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
+ * external sequence producer.
+ *   - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
+ *     cases (see its documentation for details). Users must explicitly set
+ *     ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
+ *     sequence producer is registered.
+ *   - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ *     whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ *     check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
+ *     Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
+ *
+ * Second, history buffers are not currently supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external sequence producer (for now). This has two implications:
+ *   - Dictionaries are not currently supported. Compression will *not* fail if the user
+ *     references a dictionary, but the dictionary won't have any effect.
+ *   - Stream history is not currently supported. All advanced compression APIs, including
+ *     streaming APIs, work with external sequence producers, but each block is treated as
+ *     an independent chunk without history from previous blocks.
+ *
+ * Third, multi-threading within a single compression is not currently supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
+
+typedef size_t (*ZSTD_sequenceProducer_F) (
+  void* sequenceProducerState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+/*! ZSTD_registerSequenceProducer() :
+ * Instruct zstd to use a block-level external sequence producer function.
+ *
+ * The sequenceProducerState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * Sequence producer registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on compression
+ * APIs which respect advanced parameters, such as compress2() and compressStream2().
+ * Older compression APIs such as compressCCtx(), which predate the introduction of
+ * "advanced parameters", will ignore any external sequence producer setting.
+ *
+ * The sequence producer can be "cleared" by registering a NULL function pointer. This
+ * removes all limitations described above in the "LIMITATIONS" section of the API docs.
+ *
+ * The user is strongly encouraged to read the full API documentation (above) before
+ * calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerSequenceProducer(
+  ZSTD_CCtx* cctx,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F sequenceProducer
+);
+
+/*! ZSTD_CCtxParams_registerSequenceProducer() :
+ * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params.
+ * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(),
+ * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx().
+ *
+ * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx()
+ * is required, then this function is for you. Otherwise, you probably don't need it.
+ *
+ * See tests/zstreamtest.c for example usage. */
+ZSTDLIB_STATIC_API void
+ZSTD_CCtxParams_registerSequenceProducer(
+  ZSTD_CCtx_params* params,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F sequenceProducer
+);
+
+
+/*********************************************************************
+*  Buffer-less and synchronous inner streaming functions (DEPRECATED)
+*
+*  This API is deprecated, and will be removed in a future version.
+*  It allows streaming (de)compression with user allocated buffers.
+*  However, it is hard to use, and not as well tested as the rest of
+*  our API.
+*
+*  Please use the normal streaming API instead: ZSTD_compressStream2,
+*  and ZSTD_decompressStream.
+*  If there is functionality that you need, but it doesn't provide,
+*  please open an issue on our GitHub.
+********************************************************************* */
+
+/**
+  Buffer-less streaming compression (synchronous mode)
+
+  A ZSTD_CCtx object is required to track streaming operations.
+  Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
+  ZSTD_CCtx object can be reused multiple times within successive compression operations.
+
+  Start by initializing a context.
+  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
+
+  Then, consume your input using ZSTD_compressContinue().
+  There are some important considerations to keep in mind when using this advanced function :
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
+  - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
+  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+    Worst case evaluation is provided by ZSTD_compressBound().
+    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+  - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+    It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+  - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+    In which case, it will "discard" the relevant memory section from its history.
+
+  Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
+  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+  Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
+
+  `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again.
+*/
+
+/*=====   Buffer-less streaming compression functions  =====*/
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
+
+ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+/**
+  Buffer-less streaming decompression (synchronous mode)
+
+  A ZSTD_DCtx object is required to track streaming operations.
+  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+  A ZSTD_DCtx object can be reused multiple times.
+
+  First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
+  Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+  Data fragment must be large enough to ensure successful decoding.
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
+           errorCode, which can be tested using ZSTD_isError().
+
+  It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+  such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
+  Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
+  As a consequence, check that values remain within valid application range.
+  For example, do not allocate memory blindly, check that `windowSize` is within expectation.
+  Each application can set its own limits, depending on local restrictions.
+  For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
+
+  ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
+  ZSTD_decompressContinue() is very sensitive to contiguity,
+  if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
+  or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
+  There are multiple ways to guarantee this condition.
+
+  The most memory efficient way is to use a round buffer of sufficient size.
+  Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
+  which can return an error code if required value is too large for current system (in 32-bits mode).
+  In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
+  up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
+  which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
+  At which point, decoding can resume from the beginning of the buffer.
+  Note that already decoded data stored in the buffer should be flushed before being overwritten.
+
+  There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
+
+  Finally, if you control the compression process, you can also ignore all buffer size rules,
+  as long as the encoder and decoder progress in "lock-step",
+  aka use exactly the same buffer sizes, break contiguity at the same place, etc.
+
+  Once buffers are setup, start decompression, with ZSTD_decompressBegin().
+  If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
+
+  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
+  ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+  ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
+
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
+  It can also be an error code, which can be tested with ZSTD_isError().
+
+  A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+  Context can then be reset to start a new decompression.
+
+  Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
+  This information is not required to properly decode a frame.
+
+  == Special case : skippable frames ==
+
+  Skippable frames allow integration of user-defined data into a flow of concatenated frames.
+  Skippable frames will be ignored (skipped) by decompressor.
+  The format of skippable frames is as follows :
+  a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
+  b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
+  c) Frame Content - any content (User Data) of length equal to Frame Size
+  For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
+  For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
+*/
+
+/*=====   Buffer-less streaming decompression functions  =====*/
+
+ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* misc */
+ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+
+
+
+
+/* ========================================= */
+/**       Block level API (DEPRECATED)       */
+/* ========================================= */
+
+/*!
+
+    This API is deprecated in favor of the regular compression API.
+    You can get the frame header down to 2 bytes by setting:
+      - ZSTD_c_format = ZSTD_f_zstd1_magicless
+      - ZSTD_c_contentSizeFlag = 0
+      - ZSTD_c_checksumFlag = 0
+      - ZSTD_c_dictIDFlag = 0
+
+    This API is not as well tested as our normal API, so we recommend not using it.
+    We will be removing it in a future version. If the normal API doesn't provide
+    the functionality you need, please open a GitHub issue.
+
+    Block functions produce and decode raw zstd blocks, without frame metadata.
+    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+    But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
+
+    A few rules to respect :
+    - Compressing and decompressing require a context structure
+      + Use ZSTD_createCCtx() and ZSTD_createDCtx()
+    - It is necessary to init context before starting
+      + compression : any ZSTD_compressBegin*() variant, including with dictionary
+      + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+      + If input is larger than a block size, it's necessary to split input data into multiple blocks
+      + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
+        Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
+    - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
+      ===> In which case, nothing is produced into `dst` !
+      + User __must__ test for such outcome and deal directly with uncompressed data
+      + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
+        Doing so would mess up with statistics history, leading to potential data corruption.
+      + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
+      + In case of multiple successive blocks, should some of them be uncompressed,
+        decoder must be informed of their existence in order to follow proper history.
+        Use ZSTD_insertBlock() for such a case.
+*/
+
+/*=====   Raw zstd block functions  =====*/
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
+
+#endif   /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/thirdparty/zstd/zstd_errors.h b/thirdparty/zstd/zstd_errors.h
new file mode 100644
index 0000000..20e488f
--- /dev/null
+++ b/thirdparty/zstd/zstd_errors.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
+#ifndef ZSTDERRORLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
+#  else
+#    define ZSTDERRORLIB_VISIBLE
+#  endif
+#endif
+
+#ifndef ZSTDERRORLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDERRORLIB_HIDDEN
+#  endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
+#endif
+
+/*-*********************************************
+ *  Error codes list
+ *-*********************************************
+ *  Error codes _values_ are pinned down since v1.3.1 only.
+ *  Therefore, don't rely on values if you may link to any version < v1.3.1.
+ *
+ *  Only values < 100 are considered stable.
+ *
+ *  note 1 : this API shall be used with static linking only.
+ *           dynamic linking is not yet officially supported.
+ *  note 2 : Prefer relying on the enum than on its value whenever possible
+ *           This is the only supported way to use the error list < v1.3.1
+ *  note 3 : ZSTD_isError() is always correct, whatever the library version.
+ **********************************************/
+typedef enum {
+  ZSTD_error_no_error = 0,
+  ZSTD_error_GENERIC  = 1,
+  ZSTD_error_prefix_unknown                = 10,
+  ZSTD_error_version_unsupported           = 12,
+  ZSTD_error_frameParameter_unsupported    = 14,
+  ZSTD_error_frameParameter_windowTooLarge = 16,
+  ZSTD_error_corruption_detected = 20,
+  ZSTD_error_checksum_wrong      = 22,
+  ZSTD_error_literals_headerWrong = 24,
+  ZSTD_error_dictionary_corrupted      = 30,
+  ZSTD_error_dictionary_wrong          = 32,
+  ZSTD_error_dictionaryCreation_failed = 34,
+  ZSTD_error_parameter_unsupported   = 40,
+  ZSTD_error_parameter_combination_unsupported = 41,
+  ZSTD_error_parameter_outOfBound    = 42,
+  ZSTD_error_tableLog_tooLarge       = 44,
+  ZSTD_error_maxSymbolValue_tooLarge = 46,
+  ZSTD_error_maxSymbolValue_tooSmall = 48,
+  ZSTD_error_stabilityCondition_notRespected = 50,
+  ZSTD_error_stage_wrong       = 60,
+  ZSTD_error_init_missing      = 62,
+  ZSTD_error_memory_allocation = 64,
+  ZSTD_error_workSpace_tooSmall= 66,
+  ZSTD_error_dstSize_tooSmall = 70,
+  ZSTD_error_srcSize_wrong    = 72,
+  ZSTD_error_dstBuffer_null   = 74,
+  ZSTD_error_noForwardProgress_destFull = 80,
+  ZSTD_error_noForwardProgress_inputEmpty = 82,
+  /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
+  ZSTD_error_frameIndex_tooLarge = 100,
+  ZSTD_error_seekableIO          = 102,
+  ZSTD_error_dstBuffer_wrong     = 104,
+  ZSTD_error_srcBuffer_wrong     = 105,
+  ZSTD_error_sequenceProducer_failed = 106,
+  ZSTD_error_externalSequences_invalid = 107,
+  ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
+} ZSTD_ErrorCode;
+
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);   /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_ERRORS_H_398273423 */
diff --git a/thirdparty/zstd/zstddeclib.c b/thirdparty/zstd/zstddeclib.c
new file mode 100644
index 0000000..58db8e7
--- /dev/null
+++ b/thirdparty/zstd/zstddeclib.c
@@ -0,0 +1,22228 @@
+/**
+ * \file zstddeclib.c
+ * Single-file Zstandard decompressor.
+ *
+ * Generate using:
+ * \code
+ *	python combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
+ * \endcode
+ */
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+/*
+ * Settings to bake for the standalone decompressor.
+ *
+ * Note: It's important that none of these affects 'zstd.h' (only the
+ * implementation files we're amalgamating).
+ *
+ * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also
+ * defined in mem.h (breaking C99 compatibility).
+ *
+ * Note: the undefs for xxHash allow Zstd's implementation to coincide with
+ * standalone xxHash usage (with global defines).
+ *
+ * Note: if you enable ZSTD_LEGACY_SUPPORT the combine.py script will need
+ * re-running without the "-x legacy/zstd_legacy.h" option (it excludes the
+ * legacy support at the source level).
+ */
+#define DEBUGLEVEL 0
+#define MEM_MODULE
+#undef  XXH_NAMESPACE
+#define XXH_NAMESPACE ZSTD_
+#undef  XXH_PRIVATE_API
+#define XXH_PRIVATE_API
+#undef  XXH_INLINE_ALL
+#define XXH_INLINE_ALL
+#define ZSTD_LEGACY_SUPPORT 0
+#define ZSTD_STRIP_ERROR_STRINGS
+#define ZSTD_TRACE 0
+/* TODO: Can't amalgamate ASM function */
+#define ZSTD_DISABLE_ASM 1
+
+/* Include zstd_deps.h first with all the options we need enabled. */
+#define ZSTD_DEPS_NEED_MALLOC
+/**** start inlining common/zstd_deps.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides common libc dependencies that zstd requires.
+ * The purpose is to allow replacing this file with a custom implementation
+ * to compile zstd without libc support.
+ */
+
+/* Need:
+ * NULL
+ * INT_MAX
+ * UINT_MAX
+ * ZSTD_memcpy()
+ * ZSTD_memset()
+ * ZSTD_memmove()
+ */
+#ifndef ZSTD_DEPS_COMMON
+#define ZSTD_DEPS_COMMON
+
+/* Even though we use qsort_r only for the dictionary builder, the macro
+ * _GNU_SOURCE has to be declared *before* the inclusion of any standard
+ * header and the script 'combine.sh' combines the whole zstd source code
+ * in a single file.
+ */
+#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
+    defined(__CYGWIN__) || defined(__MSYS__)
+#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
+#define _GNU_SOURCE
+#endif
+#endif
+
+#include <limits.h>
+#include <stddef.h>
+#include <string.h>
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
+#else
+# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) memset((p),(v),(l))
+#endif
+
+#endif /* ZSTD_DEPS_COMMON */
+
+/* Need:
+ * ZSTD_malloc()
+ * ZSTD_free()
+ * ZSTD_calloc()
+ */
+#ifdef ZSTD_DEPS_NEED_MALLOC
+#ifndef ZSTD_DEPS_MALLOC
+#define ZSTD_DEPS_MALLOC
+
+#include <stdlib.h>
+
+#define ZSTD_malloc(s) malloc(s)
+#define ZSTD_calloc(n,s) calloc((n), (s))
+#define ZSTD_free(p) free((p))
+
+#endif /* ZSTD_DEPS_MALLOC */
+#endif /* ZSTD_DEPS_NEED_MALLOC */
+
+/*
+ * Provides 64-bit math support.
+ * Need:
+ * U64 ZSTD_div64(U64 dividend, U32 divisor)
+ */
+#ifdef ZSTD_DEPS_NEED_MATH64
+#ifndef ZSTD_DEPS_MATH64
+#define ZSTD_DEPS_MATH64
+
+#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
+
+#endif /* ZSTD_DEPS_MATH64 */
+#endif /* ZSTD_DEPS_NEED_MATH64 */
+
+/* Need:
+ * assert()
+ */
+#ifdef ZSTD_DEPS_NEED_ASSERT
+#ifndef ZSTD_DEPS_ASSERT
+#define ZSTD_DEPS_ASSERT
+
+#include <assert.h>
+
+#endif /* ZSTD_DEPS_ASSERT */
+#endif /* ZSTD_DEPS_NEED_ASSERT */
+
+/* Need:
+ * ZSTD_DEBUG_PRINT()
+ */
+#ifdef ZSTD_DEPS_NEED_IO
+#ifndef ZSTD_DEPS_IO
+#define ZSTD_DEPS_IO
+
+#include <stdio.h>
+#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
+
+#endif /* ZSTD_DEPS_IO */
+#endif /* ZSTD_DEPS_NEED_IO */
+
+/* Only requested when <stdint.h> is known to be present.
+ * Need:
+ * intptr_t
+ */
+#ifdef ZSTD_DEPS_NEED_STDINT
+#ifndef ZSTD_DEPS_STDINT
+#define ZSTD_DEPS_STDINT
+
+#include <stdint.h>
+
+#endif /* ZSTD_DEPS_STDINT */
+#endif /* ZSTD_DEPS_NEED_STDINT */
+/**** ended inlining common/zstd_deps.h ****/
+
+/**** start inlining common/debug.c ****/
+/* ******************************************************************
+ * debug
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/*
+ * This module only hosts one global variable
+ * which can be used to dynamically influence the verbosity of traces,
+ * such as DEBUGLOG and RAWLOG
+ */
+
+/**** start inlining debug.h ****/
+/* ******************************************************************
+ * debug
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/*
+ * The purpose of this header is to enable debug functions.
+ * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
+ * and DEBUG_STATIC_ASSERT() for compile-time.
+ *
+ * By default, DEBUGLEVEL==0, which means run-time debug is disabled.
+ *
+ * Level 1 enables assert() only.
+ * Starting level 2, traces can be generated and pushed to stderr.
+ * The higher the level, the more verbose the traces.
+ *
+ * It's possible to dynamically adjust level using variable g_debug_level,
+ * which is only declared if DEBUGLEVEL>=2,
+ * and is a global variable, not multi-thread protected (use with care)
+ */
+
+#ifndef DEBUG_H_12987983217
+#define DEBUG_H_12987983217
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* static assert is triggered at compile time, leaving no runtime artefact.
+ * static assert only works with compile-time constants.
+ * Also, this variant can only be used inside a function. */
+#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
+
+
+/* DEBUGLEVEL is expected to be defined externally,
+ * typically through compiler command line.
+ * Value must be a number. */
+#ifndef DEBUGLEVEL
+#  define DEBUGLEVEL 0
+#endif
+
+
+/* recommended values for DEBUGLEVEL :
+ * 0 : release mode, no debug, all run-time checks disabled
+ * 1 : enables assert() only, no display
+ * 2 : reserved, for currently active debug path
+ * 3 : events once per object lifetime (CCtx, CDict, etc.)
+ * 4 : events once per frame
+ * 5 : events once per block
+ * 6 : events once per sequence (verbose)
+ * 7+: events at every position (*very* verbose)
+ *
+ * It's generally inconvenient to output traces > 5.
+ * In which case, it's possible to selectively trigger high verbosity levels
+ * by modifying g_debug_level.
+ */
+
+#if (DEBUGLEVEL>=1)
+#  define ZSTD_DEPS_NEED_ASSERT
+/**** skipping file: zstd_deps.h ****/
+#else
+#  ifndef assert   /* assert may be already defined, due to prior #include <assert.h> */
+#    define assert(condition) ((void)0)   /* disable assert (default) */
+#  endif
+#endif
+
+#if (DEBUGLEVEL>=2)
+#  define ZSTD_DEPS_NEED_IO
+/**** skipping file: zstd_deps.h ****/
+extern int g_debuglevel; /* the variable is only declared,
+                            it actually lives in debug.c,
+                            and is shared by the whole process.
+                            It's not thread-safe.
+                            It's useful when enabling very verbose levels
+                            on selective conditions (such as position in src) */
+
+#  define RAWLOG(l, ...)                   \
+    do {                                   \
+        if (l<=g_debuglevel) {             \
+            ZSTD_DEBUG_PRINT(__VA_ARGS__); \
+        }                                  \
+    } while (0)
+
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+#define LINE_AS_STRING TOSTRING(__LINE__)
+
+#  define DEBUGLOG(l, ...)                               \
+    do {                                                 \
+        if (l<=g_debuglevel) {                           \
+            ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \
+            ZSTD_DEBUG_PRINT(" \n");                     \
+        }                                                \
+    } while (0)
+#else
+#  define RAWLOG(l, ...)   do { } while (0)    /* disabled */
+#  define DEBUGLOG(l, ...) do { } while (0)    /* disabled */
+#endif
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* DEBUG_H_12987983217 */
+/**** ended inlining debug.h ****/
+
+#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2)
+/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a
+ * translation unit is empty. So remove this from Linux kernel builds, but
+ * otherwise just leave it in.
+ */
+int g_debuglevel = DEBUGLEVEL;
+#endif
+/**** ended inlining common/debug.c ****/
+/**** start inlining common/entropy_common.c ****/
+/* ******************************************************************
+ * Common functions of New Generation Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ *  You can contact the author at :
+ *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *  - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* *************************************
+*  Dependencies
+***************************************/
+/**** start inlining mem.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MEM_H_MODULE
+#define MEM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-****************************************
+*  Dependencies
+******************************************/
+#include <stddef.h>  /* size_t, ptrdiff_t */
+/**** start inlining compiler.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPILER_H
+#define ZSTD_COMPILER_H
+
+#include <stddef.h>
+
+/**** start inlining portability_macros.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_PORTABILITY_MACROS_H
+#define ZSTD_PORTABILITY_MACROS_H
+
+/**
+ * This header file contains macro definitions to support portability.
+ * This header is shared between C and ASM code, so it MUST only
+ * contain macro definitions. It MUST not contain any C code.
+ *
+ * This header ONLY defines macros to detect platforms/feature support.
+ *
+ */
+
+
+/* compat. with non-clang compilers */
+#ifndef __has_attribute
+  #define __has_attribute(x) 0
+#endif
+
+/* compat. with non-clang compilers */
+#ifndef __has_builtin
+#  define __has_builtin(x) 0
+#endif
+
+/* compat. with non-clang compilers */
+#ifndef __has_feature
+#  define __has_feature(x) 0
+#endif
+
+/* detects whether we are being compiled under msan */
+#ifndef ZSTD_MEMORY_SANITIZER
+#  if __has_feature(memory_sanitizer)
+#    define ZSTD_MEMORY_SANITIZER 1
+#  else
+#    define ZSTD_MEMORY_SANITIZER 0
+#  endif
+#endif
+
+/* detects whether we are being compiled under asan */
+#ifndef ZSTD_ADDRESS_SANITIZER
+#  if __has_feature(address_sanitizer)
+#    define ZSTD_ADDRESS_SANITIZER 1
+#  elif defined(__SANITIZE_ADDRESS__)
+#    define ZSTD_ADDRESS_SANITIZER 1
+#  else
+#    define ZSTD_ADDRESS_SANITIZER 0
+#  endif
+#endif
+
+/* detects whether we are being compiled under dfsan */
+#ifndef ZSTD_DATAFLOW_SANITIZER
+# if __has_feature(dataflow_sanitizer)
+#  define ZSTD_DATAFLOW_SANITIZER 1
+# else
+#  define ZSTD_DATAFLOW_SANITIZER 0
+# endif
+#endif
+
+/* Mark the internal assembly functions as hidden  */
+#ifdef __ELF__
+# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
+#elif defined(__APPLE__)
+# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func
+#else
+# define ZSTD_HIDE_ASM_FUNCTION(func)
+#endif
+
+/* Enable runtime BMI2 dispatch based on the CPU.
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
+ */
+#ifndef DYNAMIC_BMI2
+  #if ((defined(__clang__) && __has_attribute(__target__)) \
+      || (defined(__GNUC__) \
+          && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
+      && (defined(__x86_64__) || defined(_M_X64)) \
+      && !defined(__BMI2__)
+  #  define DYNAMIC_BMI2 1
+  #else
+  #  define DYNAMIC_BMI2 0
+  #endif
+#endif
+
+/**
+ * Only enable assembly for GNU C compatible compilers,
+ * because other platforms may not support GAS assembly syntax.
+ *
+ * Only enable assembly for Linux / MacOS, other platforms may
+ * work, but they haven't been tested. This could likely be
+ * extended to BSD systems.
+ *
+ * Disable assembly when MSAN is enabled, because MSAN requires
+ * 100% of code to be instrumented to work.
+ */
+#if defined(__GNUC__)
+#  if defined(__linux__) || defined(__linux) || defined(__APPLE__)
+#    if ZSTD_MEMORY_SANITIZER
+#      define ZSTD_ASM_SUPPORTED 0
+#    elif ZSTD_DATAFLOW_SANITIZER
+#      define ZSTD_ASM_SUPPORTED 0
+#    else
+#      define ZSTD_ASM_SUPPORTED 1
+#    endif
+#  else
+#    define ZSTD_ASM_SUPPORTED 0
+#  endif
+#else
+#  define ZSTD_ASM_SUPPORTED 0
+#endif
+
+/**
+ * Determines whether we should enable assembly for x86-64
+ * with BMI2.
+ *
+ * Enable if all of the following conditions hold:
+ * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
+ * - Assembly is supported
+ * - We are compiling for x86-64 and either:
+ *   - DYNAMIC_BMI2 is enabled
+ *   - BMI2 is supported at compile time
+ */
+#if !defined(ZSTD_DISABLE_ASM) &&                                 \
+    ZSTD_ASM_SUPPORTED &&                                         \
+    defined(__x86_64__) &&                                        \
+    (DYNAMIC_BMI2 || defined(__BMI2__))
+# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
+#else
+# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
+#endif
+
+/*
+ * For x86 ELF targets, add .note.gnu.property section for Intel CET in
+ * assembly sources when CET is enabled.
+ *
+ * Additionally, any function that may be called indirectly must begin
+ * with ZSTD_CET_ENDBRANCH.
+ */
+#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
+    && defined(__has_include)
+# if __has_include(<cet.h>)
+#  include <cet.h>
+#  define ZSTD_CET_ENDBRANCH _CET_ENDBR
+# endif
+#endif
+
+#ifndef ZSTD_CET_ENDBRANCH
+# define ZSTD_CET_ENDBRANCH
+#endif
+
+#endif /* ZSTD_PORTABILITY_MACROS_H */
+/**** ended inlining portability_macros.h ****/
+
+/*-*******************************************************
+*  Compiler specifics
+*********************************************************/
+/* force inlining */
+
+#if !defined(ZSTD_NO_INLINE)
+#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#  define INLINE_KEYWORD inline
+#else
+#  define INLINE_KEYWORD
+#endif
+
+#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
+#  define FORCE_INLINE_ATTR __attribute__((always_inline))
+#elif defined(_MSC_VER)
+#  define FORCE_INLINE_ATTR __forceinline
+#else
+#  define FORCE_INLINE_ATTR
+#endif
+
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
+#endif
+
+/**
+  On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).
+  This explicitly marks such functions as __cdecl so that the code will still compile
+  if a CC other than __cdecl has been made the default.
+*/
+#if  defined(_MSC_VER)
+#  define WIN_CDECL __cdecl
+#else
+#  define WIN_CDECL
+#endif
+
+/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
+#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
+#  define UNUSED_ATTR __attribute__((unused))
+#else
+#  define UNUSED_ATTR
+#endif
+
+/**
+ * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
+ * parameters. They must be inlined for the compiler to eliminate the constant
+ * branches.
+ */
+#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR
+/**
+ * HINT_INLINE is used to help the compiler generate better code. It is *not*
+ * used for "templates", so it can be tweaked based on the compilers
+ * performance.
+ *
+ * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
+ * always_inline attribute.
+ *
+ * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
+ * attribute.
+ */
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
+#  define HINT_INLINE static INLINE_KEYWORD
+#else
+#  define HINT_INLINE FORCE_INLINE_TEMPLATE
+#endif
+
+/* "soft" inline :
+ * The compiler is free to select if it's a good idea to inline or not.
+ * The main objective is to silence compiler warnings
+ * when a defined function in included but not used.
+ *
+ * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit.
+ * Updating the prefix is probably preferable, but requires a fairly large codemod,
+ * since this name is used everywhere.
+ */
+#ifndef MEM_STATIC  /* already defined in Linux Kernel mem.h */
+#if defined(__GNUC__)
+#  define MEM_STATIC static __inline UNUSED_ATTR
+#elif defined(__IAR_SYSTEMS_ICC__)
+#  define MEM_STATIC static inline UNUSED_ATTR
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#  define MEM_STATIC static inline
+#elif defined(_MSC_VER)
+#  define MEM_STATIC static __inline
+#else
+#  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+#endif
+
+/* force no inlining */
+#ifdef _MSC_VER
+#  define FORCE_NOINLINE static __declspec(noinline)
+#else
+#  if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
+#    define FORCE_NOINLINE static __attribute__((__noinline__))
+#  else
+#    define FORCE_NOINLINE static
+#  endif
+#endif
+
+
+/* target attribute */
+#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
+#  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
+#else
+#  define TARGET_ATTRIBUTE(target)
+#endif
+
+/* Target attribute for BMI2 dynamic dispatch.
+ * Enable lzcnt, bmi, and bmi2.
+ * We test for bmi1 & bmi2. lzcnt is included in bmi1.
+ */
+#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
+
+/* prefetch
+ * can be disabled, by declaring NO_PREFETCH build macro */
+#if defined(NO_PREFETCH)
+#  define PREFETCH_L1(ptr)  do { (void)(ptr); } while (0)  /* disabled */
+#  define PREFETCH_L2(ptr)  do { (void)(ptr); } while (0)  /* disabled */
+#else
+#  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC)  /* _mm_prefetch() is not defined outside of x86/x64 */
+#    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+#    define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#    define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+#  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+#    define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#    define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+#  elif defined(__aarch64__)
+#    define PREFETCH_L1(ptr)  do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0)
+#    define PREFETCH_L2(ptr)  do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0)
+#  else
+#    define PREFETCH_L1(ptr) do { (void)(ptr); } while (0)  /* disabled */
+#    define PREFETCH_L2(ptr) do { (void)(ptr); } while (0)  /* disabled */
+#  endif
+#endif  /* NO_PREFETCH */
+
+#define CACHELINE_SIZE 64
+
+#define PREFETCH_AREA(p, s)                              \
+    do {                                                 \
+        const char* const _ptr = (const char*)(p);       \
+        size_t const _size = (size_t)(s);                \
+        size_t _pos;                                     \
+        for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
+            PREFETCH_L2(_ptr + _pos);                    \
+        }                                                \
+    } while (0)
+
+/* vectorization
+ * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
+ * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */
+#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__)
+#  if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
+#    define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+#  else
+#    define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")
+#  endif
+#else
+#  define DONT_VECTORIZE
+#endif
+
+/* Tell the compiler that a branch is likely or unlikely.
+ * Only use these macros if it causes the compiler to generate better code.
+ * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc
+ * and clang, please do.
+ */
+#if defined(__GNUC__)
+#define LIKELY(x) (__builtin_expect((x), 1))
+#define UNLIKELY(x) (__builtin_expect((x), 0))
+#else
+#define LIKELY(x) (x)
+#define UNLIKELY(x) (x)
+#endif
+
+#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)))
+#  define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0)
+#else
+#  define ZSTD_UNREACHABLE do { assert(0); } while (0)
+#endif
+
+/* disable warnings */
+#ifdef _MSC_VER    /* Visual Studio */
+#  include <intrin.h>                    /* For Visual 2005 */
+#  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
+#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
+#  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
+#endif
+
+/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
+#ifndef STATIC_BMI2
+#  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
+#    ifdef __AVX2__  //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
+#       define STATIC_BMI2 1
+#    endif
+#  elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__)
+#    define STATIC_BMI2 1
+#  endif
+#endif
+
+#ifndef STATIC_BMI2
+    #define STATIC_BMI2 0
+#endif
+
+/* compile time determination of SIMD support */
+#if !defined(ZSTD_NO_INTRINSICS)
+#  if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
+#    define ZSTD_ARCH_X86_SSE2
+#  endif
+#  if defined(__ARM_NEON) || defined(_M_ARM64)
+#    define ZSTD_ARCH_ARM_NEON
+#  endif
+#
+#  if defined(ZSTD_ARCH_X86_SSE2)
+#    include <emmintrin.h>
+#  elif defined(ZSTD_ARCH_ARM_NEON)
+#    include <arm_neon.h>
+#  endif
+#endif
+
+/* C-language Attributes are added in C23. */
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
+# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
+#else
+# define ZSTD_HAS_C_ATTRIBUTE(x) 0
+#endif
+
+/* Only use C++ attributes in C++. Some compilers report support for C++
+ * attributes when compiling with C.
+ */
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute.
+ * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough
+ * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough
+ * - Else: __attribute__((__fallthrough__))
+ */
+#ifndef ZSTD_FALLTHROUGH
+# if ZSTD_HAS_C_ATTRIBUTE(fallthrough)
+#  define ZSTD_FALLTHROUGH [[fallthrough]]
+# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough)
+#  define ZSTD_FALLTHROUGH [[fallthrough]]
+# elif __has_attribute(__fallthrough__)
+/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon
+ * gcc complains about: a label can only be part of a statement and a declaration is not a statement.
+ */
+#  define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__))
+# else
+#  define ZSTD_FALLTHROUGH
+# endif
+#endif
+
+/*-**************************************************************
+*  Alignment check
+*****************************************************************/
+
+/* @return 1 if @u is a 2^n value, 0 otherwise
+ * useful to check a value is valid for alignment restrictions */
+MEM_STATIC int ZSTD_isPower2(size_t u) {
+    return (u & (u-1)) == 0;
+}
+
+/* this test was initially positioned in mem.h,
+ * but this file is removed (or replaced) for linux kernel
+ * so it's now hosted in compiler.h,
+ * which remains valid for both user & kernel spaces.
+ */
+
+#ifndef ZSTD_ALIGNOF
+# if defined(__GNUC__) || defined(_MSC_VER)
+/* covers gcc, clang & MSVC */
+/* note : this section must come first, before C11,
+ * due to a limitation in the kernel source generator */
+#  define ZSTD_ALIGNOF(T) __alignof(T)
+
+# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
+/* C11 support */
+#  include <stdalign.h>
+#  define ZSTD_ALIGNOF(T) alignof(T)
+
+# else
+/* No known support for alignof() - imperfect backup */
+#  define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T))
+
+# endif
+#endif /* ZSTD_ALIGNOF */
+
+/*-**************************************************************
+*  Sanitizer
+*****************************************************************/
+
+/**
+ * Zstd relies on pointer overflow in its decompressor.
+ * We add this attribute to functions that rely on pointer overflow.
+ */
+#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+#  if __has_attribute(no_sanitize)
+#    if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8
+       /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */
+#      define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow")))
+#    else
+       /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */
+#      define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow")))
+#    endif
+#  else
+#    define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+#  endif
+#endif
+
+/**
+ * Helper function to perform a wrapped pointer difference without triggering
+ * UBSAN.
+ *
+ * @returns lhs - rhs with wrapping
+ */
+MEM_STATIC
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs)
+{
+    return lhs - rhs;
+}
+
+/**
+ * Helper function to perform a wrapped pointer add without triggering UBSAN.
+ *
+ * @return ptr + add with wrapping
+ */
+MEM_STATIC
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add)
+{
+    return ptr + add;
+}
+
+/**
+ * Helper function to perform a wrapped pointer subtraction without triggering
+ * UBSAN.
+ *
+ * @return ptr - sub with wrapping
+ */
+MEM_STATIC
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub)
+{
+    return ptr - sub;
+}
+
+/**
+ * Helper function to add to a pointer that works around C's undefined behavior
+ * of adding 0 to NULL.
+ *
+ * @returns `ptr + add` except it defines `NULL + 0 == NULL`.
+ */
+MEM_STATIC
+unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add)
+{
+    return add > 0 ? ptr + add : ptr;
+}
+
+/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
+ * abundance of caution, disable our custom poisoning on mingw. */
+#ifdef __MINGW32__
+#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE
+#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1
+#endif
+#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE
+#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1
+#endif
+#endif
+
+#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE)
+/* Not all platforms that support msan provide sanitizers/msan_interface.h.
+ * We therefore declare the functions we need ourselves, rather than trying to
+ * include the header file... */
+#include <stddef.h>  /* size_t */
+#define ZSTD_DEPS_NEED_STDINT
+/**** skipping file: zstd_deps.h ****/
+
+/* Make memory region fully initialized (without changing its contents). */
+void __msan_unpoison(const volatile void *a, size_t size);
+
+/* Make memory region fully uninitialized (without changing its contents).
+   This is a legacy interface that does not update origin information. Use
+   __msan_allocated_memory() instead. */
+void __msan_poison(const volatile void *a, size_t size);
+
+/* Returns the offset of the first (at least partially) poisoned byte in the
+   memory range, or -1 if the whole range is good. */
+intptr_t __msan_test_shadow(const volatile void *x, size_t size);
+
+/* Print shadow and origin for the memory range to stderr in a human-readable
+   format. */
+void __msan_print_shadow(const volatile void *x, size_t size);
+#endif
+
+#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE)
+/* Not all platforms that support asan provide sanitizers/asan_interface.h.
+ * We therefore declare the functions we need ourselves, rather than trying to
+ * include the header file... */
+#include <stddef.h>  /* size_t */
+
+/**
+ * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
+ *
+ * This memory must be previously allocated by your program. Instrumented
+ * code is forbidden from accessing addresses in this region until it is
+ * unpoisoned. This function is not guaranteed to poison the entire region -
+ * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
+ * alignment restrictions.
+ *
+ * \note This function is not thread-safe because no two threads can poison or
+ * unpoison memory in the same memory region simultaneously.
+ *
+ * \param addr Start of memory region.
+ * \param size Size of memory region. */
+void __asan_poison_memory_region(void const volatile *addr, size_t size);
+
+/**
+ * Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
+ *
+ * This memory must be previously allocated by your program. Accessing
+ * addresses in this region is allowed until this region is poisoned again.
+ * This function could unpoison a super-region of <c>[addr, addr+size)</c> due
+ * to ASan alignment restrictions.
+ *
+ * \note This function is not thread-safe because no two threads can
+ * poison or unpoison memory in the same memory region simultaneously.
+ *
+ * \param addr Start of memory region.
+ * \param size Size of memory region. */
+void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
+#endif
+
+#endif /* ZSTD_COMPILER_H */
+/**** ended inlining compiler.h ****/
+/**** skipping file: debug.h ****/
+/**** skipping file: zstd_deps.h ****/
+
+
+/*-****************************************
+*  Compiler specifics
+******************************************/
+#if defined(_MSC_VER)   /* Visual Studio */
+#   include <stdlib.h>  /* _byteswap_ulong */
+#   include <intrin.h>  /* _byteswap_* */
+#elif defined(__ICCARM__)
+#   include <intrinsics.h>
+#endif
+
+/*-**************************************************************
+*  Basic Types
+*****************************************************************/
+#if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#  if defined(_AIX)
+#    include <inttypes.h>
+#  else
+#    include <stdint.h> /* intptr_t */
+#  endif
+  typedef   uint8_t BYTE;
+  typedef   uint8_t U8;
+  typedef    int8_t S8;
+  typedef  uint16_t U16;
+  typedef   int16_t S16;
+  typedef  uint32_t U32;
+  typedef   int32_t S32;
+  typedef  uint64_t U64;
+  typedef   int64_t S64;
+#else
+# include <limits.h>
+#if CHAR_BIT != 8
+#  error "this implementation requires char to be exactly 8-bit type"
+#endif
+  typedef unsigned char      BYTE;
+  typedef unsigned char      U8;
+  typedef   signed char      S8;
+#if USHRT_MAX != 65535
+#  error "this implementation requires short to be exactly 16-bit type"
+#endif
+  typedef unsigned short      U16;
+  typedef   signed short      S16;
+#if UINT_MAX != 4294967295
+#  error "this implementation requires int to be exactly 32-bit type"
+#endif
+  typedef unsigned int        U32;
+  typedef   signed int        S32;
+/* note : there are no limits defined for long long type in C90.
+ * limits exist in C99, however, in such case, <stdint.h> is preferred */
+  typedef unsigned long long  U64;
+  typedef   signed long long  S64;
+#endif
+
+
+/*-**************************************************************
+*  Memory I/O API
+*****************************************************************/
+/*=== Static platform detection ===*/
+MEM_STATIC unsigned MEM_32bits(void);
+MEM_STATIC unsigned MEM_64bits(void);
+MEM_STATIC unsigned MEM_isLittleEndian(void);
+
+/*=== Native unaligned read/write ===*/
+MEM_STATIC U16 MEM_read16(const void* memPtr);
+MEM_STATIC U32 MEM_read32(const void* memPtr);
+MEM_STATIC U64 MEM_read64(const void* memPtr);
+MEM_STATIC size_t MEM_readST(const void* memPtr);
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value);
+MEM_STATIC void MEM_write32(void* memPtr, U32 value);
+MEM_STATIC void MEM_write64(void* memPtr, U64 value);
+
+/*=== Little endian unaligned read/write ===*/
+MEM_STATIC U16 MEM_readLE16(const void* memPtr);
+MEM_STATIC U32 MEM_readLE24(const void* memPtr);
+MEM_STATIC U32 MEM_readLE32(const void* memPtr);
+MEM_STATIC U64 MEM_readLE64(const void* memPtr);
+MEM_STATIC size_t MEM_readLEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val);
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val);
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val);
+
+/*=== Big endian unaligned read/write ===*/
+MEM_STATIC U32 MEM_readBE32(const void* memPtr);
+MEM_STATIC U64 MEM_readBE64(const void* memPtr);
+MEM_STATIC size_t MEM_readBEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val);
+
+/*=== Byteswap ===*/
+MEM_STATIC U32 MEM_swap32(U32 in);
+MEM_STATIC U64 MEM_swap64(U64 in);
+MEM_STATIC size_t MEM_swapST(size_t in);
+
+
+/*-**************************************************************
+*  Memory I/O Implementation
+*****************************************************************/
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
+ * Method 2 : direct access. This method is portable but violate C standard.
+ *            It can generate buggy code on targets depending on alignment.
+ * Default  : method 1 if supported, else method 0
+ */
+#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  ifdef __GNUC__
+#    define MEM_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
+MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
+
+MEM_STATIC unsigned MEM_isLittleEndian(void)
+{
+#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+    return 1;
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+    return 0;
+#elif defined(__clang__) && __LITTLE_ENDIAN__
+    return 1;
+#elif defined(__clang__) && __BIG_ENDIAN__
+    return 0;
+#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
+    return 1;
+#elif defined(__DMC__) && defined(_M_IX86)
+    return 1;
+#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__
+    return 1;
+#else
+    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
+    return one.c[0];
+#endif
+}
+
+#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
+
+/* violates C standard, by lying on structure alignment.
+Only use if no other choice to achieve best performance on target platform */
+MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
+MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
+MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
+MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
+
+#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
+
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
+typedef __attribute__((aligned(1))) size_t unalignArch;
+
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
+MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
+
+#else
+
+/* default method, safe and standard.
+   can sometimes prove slower */
+
+MEM_STATIC U16 MEM_read16(const void* memPtr)
+{
+    U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U32 MEM_read32(const void* memPtr)
+{
+    U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U64 MEM_read64(const void* memPtr)
+{
+    U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC size_t MEM_readST(const void* memPtr)
+{
+    size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value)
+{
+    ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write32(void* memPtr, U32 value)
+{
+    ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write64(void* memPtr, U64 value)
+{
+    ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+#endif /* MEM_FORCE_MEMORY_ACCESS */
+
+MEM_STATIC U32 MEM_swap32_fallback(U32 in)
+{
+    return  ((in << 24) & 0xff000000 ) |
+            ((in <<  8) & 0x00ff0000 ) |
+            ((in >>  8) & 0x0000ff00 ) |
+            ((in >> 24) & 0x000000ff );
+}
+
+MEM_STATIC U32 MEM_swap32(U32 in)
+{
+#if defined(_MSC_VER)     /* Visual Studio */
+    return _byteswap_ulong(in);
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+  || (defined(__clang__) && __has_builtin(__builtin_bswap32))
+    return __builtin_bswap32(in);
+#elif defined(__ICCARM__)
+    return __REV(in);
+#else
+    return MEM_swap32_fallback(in);
+#endif
+}
+
+MEM_STATIC U64 MEM_swap64_fallback(U64 in)
+{
+     return  ((in << 56) & 0xff00000000000000ULL) |
+            ((in << 40) & 0x00ff000000000000ULL) |
+            ((in << 24) & 0x0000ff0000000000ULL) |
+            ((in << 8)  & 0x000000ff00000000ULL) |
+            ((in >> 8)  & 0x00000000ff000000ULL) |
+            ((in >> 24) & 0x0000000000ff0000ULL) |
+            ((in >> 40) & 0x000000000000ff00ULL) |
+            ((in >> 56) & 0x00000000000000ffULL);
+}
+
+MEM_STATIC U64 MEM_swap64(U64 in)
+{
+#if defined(_MSC_VER)     /* Visual Studio */
+    return _byteswap_uint64(in);
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+  || (defined(__clang__) && __has_builtin(__builtin_bswap64))
+    return __builtin_bswap64(in);
+#else
+    return MEM_swap64_fallback(in);
+#endif
+}
+
+MEM_STATIC size_t MEM_swapST(size_t in)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_swap32((U32)in);
+    else
+        return (size_t)MEM_swap64((U64)in);
+}
+
+/*=== Little endian r/w ===*/
+
+MEM_STATIC U16 MEM_readLE16(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read16(memPtr);
+    else {
+        const BYTE* p = (const BYTE*)memPtr;
+        return (U16)(p[0] + (p[1]<<8));
+    }
+}
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
+{
+    if (MEM_isLittleEndian()) {
+        MEM_write16(memPtr, val);
+    } else {
+        BYTE* p = (BYTE*)memPtr;
+        p[0] = (BYTE)val;
+        p[1] = (BYTE)(val>>8);
+    }
+}
+
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+    return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
+}
+
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
+{
+    MEM_writeLE16(memPtr, (U16)val);
+    ((BYTE*)memPtr)[2] = (BYTE)(val>>16);
+}
+
+MEM_STATIC U32 MEM_readLE32(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read32(memPtr);
+    else
+        return MEM_swap32(MEM_read32(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
+{
+    if (MEM_isLittleEndian())
+        MEM_write32(memPtr, val32);
+    else
+        MEM_write32(memPtr, MEM_swap32(val32));
+}
+
+MEM_STATIC U64 MEM_readLE64(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read64(memPtr);
+    else
+        return MEM_swap64(MEM_read64(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
+{
+    if (MEM_isLittleEndian())
+        MEM_write64(memPtr, val64);
+    else
+        MEM_write64(memPtr, MEM_swap64(val64));
+}
+
+MEM_STATIC size_t MEM_readLEST(const void* memPtr)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_readLE32(memPtr);
+    else
+        return (size_t)MEM_readLE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
+{
+    if (MEM_32bits())
+        MEM_writeLE32(memPtr, (U32)val);
+    else
+        MEM_writeLE64(memPtr, (U64)val);
+}
+
+/*=== Big endian r/w ===*/
+
+MEM_STATIC U32 MEM_readBE32(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_swap32(MEM_read32(memPtr));
+    else
+        return MEM_read32(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
+{
+    if (MEM_isLittleEndian())
+        MEM_write32(memPtr, MEM_swap32(val32));
+    else
+        MEM_write32(memPtr, val32);
+}
+
+MEM_STATIC U64 MEM_readBE64(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_swap64(MEM_read64(memPtr));
+    else
+        return MEM_read64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
+{
+    if (MEM_isLittleEndian())
+        MEM_write64(memPtr, MEM_swap64(val64));
+    else
+        MEM_write64(memPtr, val64);
+}
+
+MEM_STATIC size_t MEM_readBEST(const void* memPtr)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_readBE32(memPtr);
+    else
+        return (size_t)MEM_readBE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
+{
+    if (MEM_32bits())
+        MEM_writeBE32(memPtr, (U32)val);
+    else
+        MEM_writeBE64(memPtr, (U64)val);
+}
+
+/* code only tested on 32 and 64 bits systems */
+MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* MEM_H_MODULE */
+/**** ended inlining mem.h ****/
+/**** start inlining error_private.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* Note : this module is expected to remain private, do not expose it */
+
+#ifndef ERROR_H_MODULE
+#define ERROR_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************************
+*  Dependencies
+******************************************/
+/**** start inlining ../zstd_errors.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
+#ifndef ZSTDERRORLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
+#  else
+#    define ZSTDERRORLIB_VISIBLE
+#  endif
+#endif
+
+#ifndef ZSTDERRORLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDERRORLIB_HIDDEN
+#  endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
+#endif
+
+/*-*********************************************
+ *  Error codes list
+ *-*********************************************
+ *  Error codes _values_ are pinned down since v1.3.1 only.
+ *  Therefore, don't rely on values if you may link to any version < v1.3.1.
+ *
+ *  Only values < 100 are considered stable.
+ *
+ *  note 1 : this API shall be used with static linking only.
+ *           dynamic linking is not yet officially supported.
+ *  note 2 : Prefer relying on the enum than on its value whenever possible
+ *           This is the only supported way to use the error list < v1.3.1
+ *  note 3 : ZSTD_isError() is always correct, whatever the library version.
+ **********************************************/
+typedef enum {
+  ZSTD_error_no_error = 0,
+  ZSTD_error_GENERIC  = 1,
+  ZSTD_error_prefix_unknown                = 10,
+  ZSTD_error_version_unsupported           = 12,
+  ZSTD_error_frameParameter_unsupported    = 14,
+  ZSTD_error_frameParameter_windowTooLarge = 16,
+  ZSTD_error_corruption_detected = 20,
+  ZSTD_error_checksum_wrong      = 22,
+  ZSTD_error_literals_headerWrong = 24,
+  ZSTD_error_dictionary_corrupted      = 30,
+  ZSTD_error_dictionary_wrong          = 32,
+  ZSTD_error_dictionaryCreation_failed = 34,
+  ZSTD_error_parameter_unsupported   = 40,
+  ZSTD_error_parameter_combination_unsupported = 41,
+  ZSTD_error_parameter_outOfBound    = 42,
+  ZSTD_error_tableLog_tooLarge       = 44,
+  ZSTD_error_maxSymbolValue_tooLarge = 46,
+  ZSTD_error_maxSymbolValue_tooSmall = 48,
+  ZSTD_error_stabilityCondition_notRespected = 50,
+  ZSTD_error_stage_wrong       = 60,
+  ZSTD_error_init_missing      = 62,
+  ZSTD_error_memory_allocation = 64,
+  ZSTD_error_workSpace_tooSmall= 66,
+  ZSTD_error_dstSize_tooSmall = 70,
+  ZSTD_error_srcSize_wrong    = 72,
+  ZSTD_error_dstBuffer_null   = 74,
+  ZSTD_error_noForwardProgress_destFull = 80,
+  ZSTD_error_noForwardProgress_inputEmpty = 82,
+  /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
+  ZSTD_error_frameIndex_tooLarge = 100,
+  ZSTD_error_seekableIO          = 102,
+  ZSTD_error_dstBuffer_wrong     = 104,
+  ZSTD_error_srcBuffer_wrong     = 105,
+  ZSTD_error_sequenceProducer_failed = 106,
+  ZSTD_error_externalSequences_invalid = 107,
+  ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
+} ZSTD_ErrorCode;
+
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);   /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_ERRORS_H_398273423 */
+/**** ended inlining ../zstd_errors.h ****/
+/**** skipping file: compiler.h ****/
+/**** skipping file: debug.h ****/
+/**** skipping file: zstd_deps.h ****/
+
+
+/* ****************************************
+*  Compiler-specific
+******************************************/
+#if defined(__GNUC__)
+#  define ERR_STATIC static __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#  define ERR_STATIC static inline
+#elif defined(_MSC_VER)
+#  define ERR_STATIC static __inline
+#else
+#  define ERR_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+
+/*-****************************************
+*  Customization (error_public.h)
+******************************************/
+typedef ZSTD_ErrorCode ERR_enum;
+#define PREFIX(name) ZSTD_error_##name
+
+
+/*-****************************************
+*  Error codes handling
+******************************************/
+#undef ERROR   /* already defined on Visual Studio */
+#define ERROR(name) ZSTD_ERROR(name)
+#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
+
+ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
+
+ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
+
+/* check and forward error code */
+#define CHECK_V_F(e, f)     \
+    size_t const e = f;     \
+    do {                    \
+        if (ERR_isError(e)) \
+            return e;       \
+    } while (0)
+#define CHECK_F(f)   do { CHECK_V_F(_var_err__, f); } while (0)
+
+
+/*-****************************************
+*  Error Strings
+******************************************/
+
+const char* ERR_getErrorString(ERR_enum code);   /* error_private.c */
+
+ERR_STATIC const char* ERR_getErrorName(size_t code)
+{
+    return ERR_getErrorString(ERR_getErrorCode(code));
+}
+
+/**
+ * Ignore: this is an internal helper.
+ *
+ * This is a helper function to help force C99-correctness during compilation.
+ * Under strict compilation modes, variadic macro arguments can't be empty.
+ * However, variadic function arguments can be. Using a function therefore lets
+ * us statically check that at least one (string) argument was passed,
+ * independent of the compilation flags.
+ */
+static INLINE_KEYWORD UNUSED_ATTR
+void _force_has_format_string(const char *format, ...) {
+  (void)format;
+}
+
+/**
+ * Ignore: this is an internal helper.
+ *
+ * We want to force this function invocation to be syntactically correct, but
+ * we don't want to force runtime evaluation of its arguments.
+ */
+#define _FORCE_HAS_FORMAT_STRING(...)              \
+    do {                                           \
+        if (0) {                                   \
+            _force_has_format_string(__VA_ARGS__); \
+        }                                          \
+    } while (0)
+
+#define ERR_QUOTE(str) #str
+
+/**
+ * Return the specified error if the condition evaluates to true.
+ *
+ * In debug modes, prints additional information.
+ * In order to do that (particularly, printing the conditional that failed),
+ * this can't just wrap RETURN_ERROR().
+ */
+#define RETURN_ERROR_IF(cond, err, ...)                                        \
+    do {                                                                       \
+        if (cond) {                                                            \
+            RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s",          \
+                  __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
+            _FORCE_HAS_FORMAT_STRING(__VA_ARGS__);                             \
+            RAWLOG(3, ": " __VA_ARGS__);                                       \
+            RAWLOG(3, "\n");                                                   \
+            return ERROR(err);                                                 \
+        }                                                                      \
+    } while (0)
+
+/**
+ * Unconditionally return the specified error.
+ *
+ * In debug modes, prints additional information.
+ */
+#define RETURN_ERROR(err, ...)                                               \
+    do {                                                                     \
+        RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
+              __FILE__, __LINE__, ERR_QUOTE(ERROR(err)));                    \
+        _FORCE_HAS_FORMAT_STRING(__VA_ARGS__);                               \
+        RAWLOG(3, ": " __VA_ARGS__);                                         \
+        RAWLOG(3, "\n");                                                     \
+        return ERROR(err);                                                   \
+    } while(0)
+
+/**
+ * If the provided expression evaluates to an error code, returns that error code.
+ *
+ * In debug modes, prints additional information.
+ */
+#define FORWARD_IF_ERROR(err, ...)                                                 \
+    do {                                                                           \
+        size_t const err_code = (err);                                             \
+        if (ERR_isError(err_code)) {                                               \
+            RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s",                 \
+                  __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
+            _FORCE_HAS_FORMAT_STRING(__VA_ARGS__);                                 \
+            RAWLOG(3, ": " __VA_ARGS__);                                           \
+            RAWLOG(3, "\n");                                                       \
+            return err_code;                                                       \
+        }                                                                          \
+    } while(0)
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ERROR_H_MODULE */
+/**** ended inlining error_private.h ****/
+#define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
+/**** start inlining fse.h ****/
+/* ******************************************************************
+ * FSE : Finite State Entropy codec
+ * Public Prototypes declaration
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef FSE_H
+#define FSE_H
+
+
+/*-*****************************************
+*  Dependencies
+******************************************/
+/**** skipping file: zstd_deps.h ****/
+
+
+/*-*****************************************
+*  FSE_PUBLIC_API : control library symbols visibility
+******************************************/
+#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
+#  define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
+#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
+#  define FSE_PUBLIC_API __declspec(dllexport)
+#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
+#  define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define FSE_PUBLIC_API
+#endif
+
+/*------   Version   ------*/
+#define FSE_VERSION_MAJOR    0
+#define FSE_VERSION_MINOR    9
+#define FSE_VERSION_RELEASE  0
+
+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
+#define FSE_QUOTE(str) #str
+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
+
+#define FSE_VERSION_NUMBER  (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
+FSE_PUBLIC_API unsigned FSE_versionNumber(void);   /**< library version number; to be used when checking dll version */
+
+
+/*-*****************************************
+*  Tool functions
+******************************************/
+FSE_PUBLIC_API size_t FSE_compressBound(size_t size);       /* maximum compressed size */
+
+/* Error Management */
+FSE_PUBLIC_API unsigned    FSE_isError(size_t code);        /* tells if a return value is an error code */
+FSE_PUBLIC_API const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
+
+
+/*-*****************************************
+*  FSE detailed API
+******************************************/
+/*!
+FSE_compress() does the following:
+1. count symbol occurrence from source[] into table count[] (see hist.h)
+2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
+3. save normalized counters to memory buffer using writeNCount()
+4. build encoding table 'CTable' from normalized counters
+5. encode the data stream using encoding table 'CTable'
+
+FSE_decompress() does the following:
+1. read normalized counters with readNCount()
+2. build decoding table 'DTable' from normalized counters
+3. decode the data stream using decoding table 'DTable'
+
+The following API allows targeting specific sub-functions for advanced tasks.
+For example, it's possible to compress several blocks using the same 'CTable',
+or to save and provide normalized distribution using external method.
+*/
+
+/* *** COMPRESSION *** */
+
+/*! FSE_optimalTableLog():
+    dynamically downsize 'tableLog' when conditions are met.
+    It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
+    @return : recommended tableLog (necessarily <= 'maxTableLog') */
+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+
+/*! FSE_normalizeCount():
+    normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
+    'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
+    useLowProbCount is a boolean parameter which trades off compressed size for
+    faster header decoding. When it is set to 1, the compressed data will be slightly
+    smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
+    faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
+    is a good default, since header deserialization makes a big speed difference.
+    Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
+    @return : tableLog,
+              or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
+                    const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount);
+
+/*! FSE_NCountWriteBound():
+    Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
+    Typically useful for allocation purpose. */
+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
+
+/*! FSE_writeNCount():
+    Compactly save 'normalizedCounter' into 'buffer'.
+    @return : size of the compressed table,
+              or an errorCode, which can be tested using FSE_isError(). */
+FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
+                                 const short* normalizedCounter,
+                                 unsigned maxSymbolValue, unsigned tableLog);
+
+/*! Constructor and Destructor of FSE_CTable.
+    Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
+typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
+
+/*! FSE_buildCTable():
+    Builds `ct`, which must be already allocated, using FSE_createCTable().
+    @return : 0, or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+
+/*! FSE_compress_usingCTable():
+    Compress `src` using `ct` into `dst` which must be already allocated.
+    @return : size of compressed data (<= `dstCapacity`),
+              or 0 if compressed data could not fit into `dst`,
+              or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
+
+/*!
+Tutorial :
+----------
+The first step is to count all symbols. FSE_count() does this job very fast.
+Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
+'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
+maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
+FSE_count() will return the number of occurrence of the most frequent symbol.
+This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+The next step is to normalize the frequencies.
+FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
+It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
+You can use 'tableLog'==0 to mean "use default tableLog value".
+If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
+which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
+
+The result of FSE_normalizeCount() will be saved into a table,
+called 'normalizedCounter', which is a table of signed short.
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
+The return value is tableLog if everything proceeded as expected.
+It is 0 if there is a single symbol within distribution.
+If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
+'buffer' must be already allocated.
+For guaranteed success, buffer size must be at least FSE_headerBound().
+The result of the function is the number of bytes written into 'buffer'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
+
+'normalizedCounter' can then be used to create the compression table 'CTable'.
+The space required by 'CTable' must be already allocated, using FSE_createCTable().
+You can then use FSE_buildCTable() to fill 'CTable'.
+If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
+
+'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
+Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
+The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
+If it returns '0', compressed data could not fit into 'dst'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+*/
+
+
+/* *** DECOMPRESSION *** */
+
+/*! FSE_readNCount():
+    Read compactly saved 'normalizedCounter' from 'rBuffer'.
+    @return : size read from 'rBuffer',
+              or an errorCode, which can be tested using FSE_isError().
+              maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
+FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
+                           unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
+                           const void* rBuffer, size_t rBuffSize);
+
+/*! FSE_readNCount_bmi2():
+ * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
+ */
+FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
+                           unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
+                           const void* rBuffer, size_t rBuffSize, int bmi2);
+
+typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be more restrictive than void* */
+
+/*!
+Tutorial :
+----------
+(Note : these functions only decompress FSE-compressed blocks.
+ If block is uncompressed, use memcpy() instead
+ If block is a single repeated byte, use memset() instead )
+
+The first step is to obtain the normalized frequencies of symbols.
+This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
+In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
+or size the table to handle worst case situations (typically 256).
+FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
+The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
+Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
+This is performed by the function FSE_buildDTable().
+The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
+`cSrcSize` must be strictly correct, otherwise decompression will fail.
+FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
+If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
+*/
+
+#endif  /* FSE_H */
+
+
+#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
+#define FSE_H_FSE_STATIC_LINKING_ONLY
+
+/* *** Dependency *** */
+/**** start inlining bitstream.h ****/
+/* ******************************************************************
+ * bitstream
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+#ifndef BITSTREAM_H_MODULE
+#define BITSTREAM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+/*
+*  This API consists of small unitary functions, which must be inlined for best performance.
+*  Since link-time-optimization is not available for all compilers,
+*  these functions are defined into a .h to be included.
+*/
+
+/*-****************************************
+*  Dependencies
+******************************************/
+/**** skipping file: mem.h ****/
+/**** skipping file: compiler.h ****/
+/**** skipping file: debug.h ****/
+/**** skipping file: error_private.h ****/
+/**** start inlining bits.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_BITS_H
+#define ZSTD_BITS_H
+
+/**** skipping file: mem.h ****/
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
+{
+    assert(val != 0);
+    {
+        static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
+                                                30, 22, 20, 15, 25, 17, 4, 8,
+                                                31, 27, 13, 23, 21, 19, 16, 7,
+                                                26, 12, 18, 6, 11, 5, 10, 9};
+        return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
+    }
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
+{
+    assert(val != 0);
+#   if defined(_MSC_VER)
+#       if STATIC_BMI2 == 1
+            return (unsigned)_tzcnt_u32(val);
+#       else
+            if (val != 0) {
+                unsigned long r;
+                _BitScanForward(&r, val);
+                return (unsigned)r;
+            } else {
+                /* Should not reach this code path */
+                __assume(0);
+            }
+#       endif
+#   elif defined(__GNUC__) && (__GNUC__ >= 4)
+        return (unsigned)__builtin_ctz(val);
+#   elif defined(__ICCARM__)
+        return (unsigned)__builtin_ctz(val);
+#   else
+        return ZSTD_countTrailingZeros32_fallback(val);
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
+    assert(val != 0);
+    {
+        static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
+                                            11, 14, 16, 18, 22, 25, 3, 30,
+                                            8, 12, 20, 28, 15, 17, 24, 7,
+                                            19, 27, 23, 6, 26, 5, 4, 31};
+        val |= val >> 1;
+        val |= val >> 2;
+        val |= val >> 4;
+        val |= val >> 8;
+        val |= val >> 16;
+        return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27];
+    }
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
+{
+    assert(val != 0);
+#   if defined(_MSC_VER)
+#       if STATIC_BMI2 == 1
+            return (unsigned)_lzcnt_u32(val);
+#       else
+            if (val != 0) {
+                unsigned long r;
+                _BitScanReverse(&r, val);
+                return (unsigned)(31 - r);
+            } else {
+                /* Should not reach this code path */
+                __assume(0);
+            }
+#       endif
+#   elif defined(__GNUC__) && (__GNUC__ >= 4)
+        return (unsigned)__builtin_clz(val);
+#   elif defined(__ICCARM__)
+        return (unsigned)__builtin_clz(val);
+#   else
+        return ZSTD_countLeadingZeros32_fallback(val);
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
+{
+    assert(val != 0);
+#   if defined(_MSC_VER) && defined(_WIN64)
+#       if STATIC_BMI2 == 1
+            return (unsigned)_tzcnt_u64(val);
+#       else
+            if (val != 0) {
+                unsigned long r;
+                _BitScanForward64(&r, val);
+                return (unsigned)r;
+            } else {
+                /* Should not reach this code path */
+                __assume(0);
+            }
+#       endif
+#   elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
+        return (unsigned)__builtin_ctzll(val);
+#   elif defined(__ICCARM__)
+        return (unsigned)__builtin_ctzll(val);
+#   else
+        {
+            U32 mostSignificantWord = (U32)(val >> 32);
+            U32 leastSignificantWord = (U32)val;
+            if (leastSignificantWord == 0) {
+                return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
+            } else {
+                return ZSTD_countTrailingZeros32(leastSignificantWord);
+            }
+        }
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
+{
+    assert(val != 0);
+#   if defined(_MSC_VER) && defined(_WIN64)
+#       if STATIC_BMI2 == 1
+            return (unsigned)_lzcnt_u64(val);
+#       else
+            if (val != 0) {
+                unsigned long r;
+                _BitScanReverse64(&r, val);
+                return (unsigned)(63 - r);
+            } else {
+                /* Should not reach this code path */
+                __assume(0);
+            }
+#       endif
+#   elif defined(__GNUC__) && (__GNUC__ >= 4)
+        return (unsigned)(__builtin_clzll(val));
+#   elif defined(__ICCARM__)
+        return (unsigned)(__builtin_clzll(val));
+#   else
+        {
+            U32 mostSignificantWord = (U32)(val >> 32);
+            U32 leastSignificantWord = (U32)val;
+            if (mostSignificantWord == 0) {
+                return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
+            } else {
+                return ZSTD_countLeadingZeros32(mostSignificantWord);
+            }
+        }
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
+{
+    if (MEM_isLittleEndian()) {
+        if (MEM_64bits()) {
+            return ZSTD_countTrailingZeros64((U64)val) >> 3;
+        } else {
+            return ZSTD_countTrailingZeros32((U32)val) >> 3;
+        }
+    } else {  /* Big Endian CPU */
+        if (MEM_64bits()) {
+            return ZSTD_countLeadingZeros64((U64)val) >> 3;
+        } else {
+            return ZSTD_countLeadingZeros32((U32)val) >> 3;
+        }
+    }
+}
+
+MEM_STATIC unsigned ZSTD_highbit32(U32 val)   /* compress, dictBuilder, decodeCorpus */
+{
+    assert(val != 0);
+    return 31 - ZSTD_countLeadingZeros32(val);
+}
+
+/* ZSTD_rotateRight_*():
+ * Rotates a bitfield to the right by "count" bits.
+ * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
+ */
+MEM_STATIC
+U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
+    assert(count < 64);
+    count &= 0x3F; /* for fickle pattern recognition */
+    return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
+}
+
+MEM_STATIC
+U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
+    assert(count < 32);
+    count &= 0x1F; /* for fickle pattern recognition */
+    return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
+}
+
+MEM_STATIC
+U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
+    assert(count < 16);
+    count &= 0x0F; /* for fickle pattern recognition */
+    return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
+}
+
+#endif /* ZSTD_BITS_H */
+/**** ended inlining bits.h ****/
+
+
+/*=========================================
+*  Target specific
+=========================================*/
+#ifndef ZSTD_NO_INTRINSICS
+#  if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__)
+#    include <immintrin.h>   /* support for bextr (experimental)/bzhi */
+#  elif defined(__ICCARM__)
+#    include <intrinsics.h>
+#  endif
+#endif
+
+#define STREAM_ACCUMULATOR_MIN_32  25
+#define STREAM_ACCUMULATOR_MIN_64  57
+#define STREAM_ACCUMULATOR_MIN    ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
+
+
+/*-******************************************
+*  bitStream encoding API (write forward)
+********************************************/
+/* bitStream can mix input from multiple sources.
+ * A critical property of these streams is that they encode and decode in **reverse** direction.
+ * So the first bit sequence you add will be the last to be read, like a LIFO stack.
+ */
+typedef struct {
+    size_t bitContainer;
+    unsigned bitPos;
+    char*  startPtr;
+    char*  ptr;
+    char*  endPtr;
+} BIT_CStream_t;
+
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
+MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+MEM_STATIC void   BIT_flushBits(BIT_CStream_t* bitC);
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
+
+/* Start with initCStream, providing the size of buffer to write into.
+*  bitStream will never write outside of this buffer.
+*  `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
+*
+*  bits are first added to a local register.
+*  Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
+*  Writing data into memory is an explicit operation, performed by the flushBits function.
+*  Hence keep track how many bits are potentially stored into local register to avoid register overflow.
+*  After a flushBits, a maximum of 7 bits might still be stored into local register.
+*
+*  Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
+*
+*  Last operation is to close the bitStream.
+*  The function returns the final size of CStream in bytes.
+*  If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
+*/
+
+
+/*-********************************************
+*  bitStream decoding API (read backward)
+**********************************************/
+typedef size_t BitContainerType;
+typedef struct {
+    BitContainerType bitContainer;
+    unsigned bitsConsumed;
+    const char* ptr;
+    const char* start;
+    const char* limitPtr;
+} BIT_DStream_t;
+
+typedef enum { BIT_DStream_unfinished = 0,  /* fully refilled */
+               BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */
+               BIT_DStream_completed = 2,   /* bitstream entirely consumed, bit-exact */
+               BIT_DStream_overflow = 3     /* user requested more bits than present in bitstream */
+    } BIT_DStream_status;  /* result of BIT_reloadDStream() */
+
+MEM_STATIC size_t   BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
+MEM_STATIC size_t   BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
+
+
+/* Start by invoking BIT_initDStream().
+*  A chunk of the bitStream is then stored into a local register.
+*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType).
+*  You can then retrieve bitFields stored into the local register, **in reverse order**.
+*  Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
+*  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
+*  Otherwise, it can be less than that, so proceed accordingly.
+*  Checking if DStream has reached its end can be performed with BIT_endOfDStream().
+*/
+
+
+/*-****************************************
+*  unsafe API
+******************************************/
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
+
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
+/* unsafe version; does not check buffer overflow */
+
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
+/* faster, but works only if nbBits >= 1 */
+
+/*=====    Local Constants   =====*/
+static const unsigned BIT_mask[] = {
+    0,          1,         3,         7,         0xF,       0x1F,
+    0x3F,       0x7F,      0xFF,      0x1FF,     0x3FF,     0x7FF,
+    0xFFF,      0x1FFF,    0x3FFF,    0x7FFF,    0xFFFF,    0x1FFFF,
+    0x3FFFF,    0x7FFFF,   0xFFFFF,   0x1FFFFF,  0x3FFFFF,  0x7FFFFF,
+    0xFFFFFF,   0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
+    0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
+#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
+
+/*-**************************************************************
+*  bitStream encoding
+****************************************************************/
+/*! BIT_initCStream() :
+ *  `dstCapacity` must be > sizeof(size_t)
+ *  @return : 0 if success,
+ *            otherwise an error code (can be tested using ERR_isError()) */
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
+                                  void* startPtr, size_t dstCapacity)
+{
+    bitC->bitContainer = 0;
+    bitC->bitPos = 0;
+    bitC->startPtr = (char*)startPtr;
+    bitC->ptr = bitC->startPtr;
+    bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
+    if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
+    return 0;
+}
+
+FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
+{
+#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
+    return  _bzhi_u64(bitContainer, nbBits);
+#else
+    assert(nbBits < BIT_MASK_SIZE);
+    return bitContainer & BIT_mask[nbBits];
+#endif
+}
+
+/*! BIT_addBits() :
+ *  can add up to 31 bits into `bitC`.
+ *  Note : does not check for register overflow ! */
+MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
+                            size_t value, unsigned nbBits)
+{
+    DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
+    assert(nbBits < BIT_MASK_SIZE);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+    bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos;
+    bitC->bitPos += nbBits;
+}
+
+/*! BIT_addBitsFast() :
+ *  works only if `value` is _clean_,
+ *  meaning all high bits above nbBits are 0 */
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
+                                size_t value, unsigned nbBits)
+{
+    assert((value>>nbBits) == 0);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+    bitC->bitContainer |= value << bitC->bitPos;
+    bitC->bitPos += nbBits;
+}
+
+/*! BIT_flushBitsFast() :
+ *  assumption : bitContainer has not overflowed
+ *  unsafe version; does not check buffer overflow */
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
+{
+    size_t const nbBytes = bitC->bitPos >> 3;
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+    assert(bitC->ptr <= bitC->endPtr);
+    MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+    bitC->ptr += nbBytes;
+    bitC->bitPos &= 7;
+    bitC->bitContainer >>= nbBytes*8;
+}
+
+/*! BIT_flushBits() :
+ *  assumption : bitContainer has not overflowed
+ *  safe version; check for buffer overflow, and prevents it.
+ *  note : does not signal buffer overflow.
+ *  overflow will be revealed later on using BIT_closeCStream() */
+MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
+{
+    size_t const nbBytes = bitC->bitPos >> 3;
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+    assert(bitC->ptr <= bitC->endPtr);
+    MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+    bitC->ptr += nbBytes;
+    if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
+    bitC->bitPos &= 7;
+    bitC->bitContainer >>= nbBytes*8;
+}
+
+/*! BIT_closeCStream() :
+ *  @return : size of CStream, in bytes,
+ *            or 0 if it could not fit into dstBuffer */
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
+{
+    BIT_addBitsFast(bitC, 1, 1);   /* endMark */
+    BIT_flushBits(bitC);
+    if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
+    return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
+}
+
+
+/*-********************************************************
+*  bitStream decoding
+**********************************************************/
+/*! BIT_initDStream() :
+ *  Initialize a BIT_DStream_t.
+ * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
+ * `srcSize` must be the *exact* size of the bitStream, in bytes.
+ * @return : size of stream (== srcSize), or an errorCode if a problem is detected
+ */
+MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
+{
+    if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
+
+    bitD->start = (const char*)srcBuffer;
+    bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
+
+    if (srcSize >=  sizeof(bitD->bitContainer)) {  /* normal case */
+        bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
+        bitD->bitContainer = MEM_readLEST(bitD->ptr);
+        { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+          bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;  /* ensures bitsConsumed is always set */
+          if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
+    } else {
+        bitD->ptr   = bitD->start;
+        bitD->bitContainer = *(const BYTE*)(bitD->start);
+        switch(srcSize)
+        {
+        case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+                ZSTD_FALLTHROUGH;
+
+        case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+                ZSTD_FALLTHROUGH;
+
+        case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+                ZSTD_FALLTHROUGH;
+
+        case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24;
+                ZSTD_FALLTHROUGH;
+
+        case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16;
+                ZSTD_FALLTHROUGH;
+
+        case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) <<  8;
+                ZSTD_FALLTHROUGH;
+
+        default: break;
+        }
+        {   BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+            bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
+            if (lastByte == 0) return ERROR(corruption_detected);  /* endMark not present */
+        }
+        bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
+    }
+
+    return srcSize;
+}
+
+FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
+{
+    return bitContainer >> start;
+}
+
+FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
+{
+    U32 const regMask = sizeof(bitContainer)*8 - 1;
+    /* if start > regMask, bitstream is corrupted, and result is undefined */
+    assert(nbBits < BIT_MASK_SIZE);
+    /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
+     * than accessing memory. When bmi2 instruction is not present, we consider
+     * such cpus old (pre-Haswell, 2013) and their performance is not of that
+     * importance.
+     */
+#if defined(__x86_64__) || defined(_M_X86)
+    return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
+#else
+    return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
+#endif
+}
+
+/*! BIT_lookBits() :
+ *  Provides next n bits from local register.
+ *  local register is not modified.
+ *  On 32-bits, maxNbBits==24.
+ *  On 64-bits, maxNbBits==56.
+ * @return : value extracted */
+FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t*  bitD, U32 nbBits)
+{
+    /* arbitrate between double-shift and shift+mask */
+#if 1
+    /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
+     * bitstream is likely corrupted, and result is undefined */
+    return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
+#else
+    /* this code path is slower on my os-x laptop */
+    U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+    return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
+#endif
+}
+
+/*! BIT_lookBitsFast() :
+ *  unsafe version; only works if nbBits >= 1 */
+MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
+{
+    U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+    assert(nbBits >= 1);
+    return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
+}
+
+FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
+{
+    bitD->bitsConsumed += nbBits;
+}
+
+/*! BIT_readBits() :
+ *  Read (consume) next n bits from local register and update.
+ *  Pay attention to not read more than nbBits contained into local register.
+ * @return : extracted value. */
+FORCE_INLINE_TEMPLATE size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
+{
+    size_t const value = BIT_lookBits(bitD, nbBits);
+    BIT_skipBits(bitD, nbBits);
+    return value;
+}
+
+/*! BIT_readBitsFast() :
+ *  unsafe version; only works if nbBits >= 1 */
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
+{
+    size_t const value = BIT_lookBitsFast(bitD, nbBits);
+    assert(nbBits >= 1);
+    BIT_skipBits(bitD, nbBits);
+    return value;
+}
+
+/*! BIT_reloadDStream_internal() :
+ *  Simple variant of BIT_reloadDStream(), with two conditions:
+ *  1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
+ *  2. look window is valid after shifted down : bitD->ptr >= bitD->start
+ */
+MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD)
+{
+    assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
+    bitD->ptr -= bitD->bitsConsumed >> 3;
+    assert(bitD->ptr >= bitD->start);
+    bitD->bitsConsumed &= 7;
+    bitD->bitContainer = MEM_readLEST(bitD->ptr);
+    return BIT_DStream_unfinished;
+}
+
+/*! BIT_reloadDStreamFast() :
+ *  Similar to BIT_reloadDStream(), but with two differences:
+ *  1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
+ *  2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
+ *     point you must use BIT_reloadDStream() to reload.
+ */
+MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
+{
+    if (UNLIKELY(bitD->ptr < bitD->limitPtr))
+        return BIT_DStream_overflow;
+    return BIT_reloadDStream_internal(bitD);
+}
+
+/*! BIT_reloadDStream() :
+ *  Refill `bitD` from buffer previously set in BIT_initDStream() .
+ *  This function is safe, it guarantees it will not never beyond src buffer.
+ * @return : status of `BIT_DStream_t` internal register.
+ *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
+FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
+{
+    /* note : once in overflow mode, a bitstream remains in this mode until it's reset */
+    if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) {
+        static const BitContainerType zeroFilled = 0;
+        bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */
+        /* overflow detected, erroneous scenario or end of stream: no update */
+        return BIT_DStream_overflow;
+    }
+
+    assert(bitD->ptr >= bitD->start);
+
+    if (bitD->ptr >= bitD->limitPtr) {
+        return BIT_reloadDStream_internal(bitD);
+    }
+    if (bitD->ptr == bitD->start) {
+        /* reached end of bitStream => no update */
+        if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
+        return BIT_DStream_completed;
+    }
+    /* start < ptr < limitPtr => cautious update */
+    {   U32 nbBytes = bitD->bitsConsumed >> 3;
+        BIT_DStream_status result = BIT_DStream_unfinished;
+        if (bitD->ptr - nbBytes < bitD->start) {
+            nbBytes = (U32)(bitD->ptr - bitD->start);  /* ptr > start */
+            result = BIT_DStream_endOfBuffer;
+        }
+        bitD->ptr -= nbBytes;
+        bitD->bitsConsumed -= nbBytes*8;
+        bitD->bitContainer = MEM_readLEST(bitD->ptr);   /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
+        return result;
+    }
+}
+
+/*! BIT_endOfDStream() :
+ * @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
+ */
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
+{
+    return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* BITSTREAM_H_MODULE */
+/**** ended inlining bitstream.h ****/
+
+
+/* *****************************************
+*  Static allocation
+*******************************************/
+/* FSE buffer bounds */
+#define FSE_NCOUNTBOUND 512
+#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)
+#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size))   /* Macro version, useful for static allocation */
+
+/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
+#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue)   (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2))
+#define FSE_DTABLE_SIZE_U32(maxTableLog)                   (1 + (1<<(maxTableLog)))
+
+/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
+#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue)   (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
+#define FSE_DTABLE_SIZE(maxTableLog)                   (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
+
+
+/* *****************************************
+ *  FSE advanced API
+ ***************************************** */
+
+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
+/**< same as FSE_optimalTableLog(), which used `minus==2` */
+
+size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
+/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
+ * See FSE_buildCTable_wksp() for breakdown of workspace usage.
+ */
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */)
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
+size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+
+#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
+#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned))
+FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
+
+#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
+#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
+size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
+ * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
+
+typedef enum {
+   FSE_repeat_none,  /**< Cannot use the previous table */
+   FSE_repeat_check, /**< Can use the previous table but it must be checked */
+   FSE_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
+ } FSE_repeat;
+
+/* *****************************************
+*  FSE symbol compression API
+*******************************************/
+/*!
+   This API consists of small unitary functions, which highly benefit from being inlined.
+   Hence their body are included in next section.
+*/
+typedef struct {
+    ptrdiff_t   value;
+    const void* stateTable;
+    const void* symbolTT;
+    unsigned    stateLog;
+} FSE_CState_t;
+
+static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
+
+static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
+
+static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
+
+/**<
+These functions are inner components of FSE_compress_usingCTable().
+They allow the creation of custom streams, mixing multiple tables and bit sources.
+
+A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
+So the first symbol you will encode is the last you will decode, like a LIFO stack.
+
+You will need a few variables to track your CStream. They are :
+
+FSE_CTable    ct;         // Provided by FSE_buildCTable()
+BIT_CStream_t bitStream;  // bitStream tracking structure
+FSE_CState_t  state;      // State tracking structure (can have several)
+
+
+The first thing to do is to init bitStream and state.
+    size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
+    FSE_initCState(&state, ct);
+
+Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
+You can then encode your input data, byte after byte.
+FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
+Remember decoding will be done in reverse direction.
+    FSE_encodeByte(&bitStream, &state, symbol);
+
+At any time, you can also add any bit sequence.
+Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
+    BIT_addBits(&bitStream, bitField, nbBits);
+
+The above methods don't commit data to memory, they just store it into local register, for speed.
+Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+Writing data to memory is a manual operation, performed by the flushBits function.
+    BIT_flushBits(&bitStream);
+
+Your last FSE encoding operation shall be to flush your last state value(s).
+    FSE_flushState(&bitStream, &state);
+
+Finally, you must close the bitStream.
+The function returns the size of CStream in bytes.
+If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
+If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
+    size_t size = BIT_closeCStream(&bitStream);
+*/
+
+
+/* *****************************************
+*  FSE symbol decompression API
+*******************************************/
+typedef struct {
+    size_t      state;
+    const void* table;   /* precise table may vary, depending on U16 */
+} FSE_DState_t;
+
+
+static void     FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
+
+static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
+
+static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
+
+/**<
+Let's now decompose FSE_decompress_usingDTable() into its unitary components.
+You will decode FSE-encoded symbols from the bitStream,
+and also any other bitFields you put in, **in reverse order**.
+
+You will need a few variables to track your bitStream. They are :
+
+BIT_DStream_t DStream;    // Stream context
+FSE_DState_t  DState;     // State context. Multiple ones are possible
+FSE_DTable*   DTablePtr;  // Decoding table, provided by FSE_buildDTable()
+
+The first thing to do is to init the bitStream.
+    errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
+
+You should then retrieve your initial state(s)
+(in reverse flushing order if you have several ones) :
+    errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
+
+You can then decode your data, symbol after symbol.
+For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
+Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
+    unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
+
+You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
+Note : maximum allowed nbBits is 25, for 32-bits compatibility
+    size_t bitField = BIT_readBits(&DStream, nbBits);
+
+All above operations only read from local register (which size depends on size_t).
+Refueling the register from memory is manually performed by the reload method.
+    endSignal = FSE_reloadDStream(&DStream);
+
+BIT_reloadDStream() result tells if there is still some more data to read from DStream.
+BIT_DStream_unfinished : there is still some data left into the DStream.
+BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
+BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
+BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
+
+When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
+to properly detect the exact end of stream.
+After each decoded symbol, check if DStream is fully consumed using this simple test :
+    BIT_reloadDStream(&DStream) >= BIT_DStream_completed
+
+When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
+Checking if DStream has reached its end is performed by :
+    BIT_endOfDStream(&DStream);
+Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
+    FSE_endOfDState(&DState);
+*/
+
+
+/* *****************************************
+*  FSE unsafe API
+*******************************************/
+static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
+/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
+
+
+/* *****************************************
+*  Implementation of inlined functions
+*******************************************/
+typedef struct {
+    int deltaFindState;
+    U32 deltaNbBits;
+} FSE_symbolCompressionTransform; /* total 8 bytes */
+
+MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
+{
+    const void* ptr = ct;
+    const U16* u16ptr = (const U16*) ptr;
+    const U32 tableLog = MEM_read16(ptr);
+    statePtr->value = (ptrdiff_t)1<<tableLog;
+    statePtr->stateTable = u16ptr+2;
+    statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
+    statePtr->stateLog = tableLog;
+}
+
+
+/*! FSE_initCState2() :
+*   Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
+*   uses the smallest state value possible, saving the cost of this symbol */
+MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
+{
+    FSE_initCState(statePtr, ct);
+    {   const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
+        const U16* stateTable = (const U16*)(statePtr->stateTable);
+        U32 nbBitsOut  = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
+        statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
+        statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+    }
+}
+
+MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
+{
+    FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
+    const U16* const stateTable = (const U16*)(statePtr->stateTable);
+    U32 const nbBitsOut  = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
+    BIT_addBits(bitC,  (size_t)statePtr->value, nbBitsOut);
+    statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+}
+
+MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
+{
+    BIT_addBits(bitC, (size_t)statePtr->value, statePtr->stateLog);
+    BIT_flushBits(bitC);
+}
+
+
+/* FSE_getMaxNbBits() :
+ * Approximate maximum cost of a symbol, in bits.
+ * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
+MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
+{
+    const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
+    return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
+}
+
+/* FSE_bitCost() :
+ * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
+MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
+{
+    const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
+    U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
+    U32 const threshold = (minNbBits+1) << 16;
+    assert(tableLog < 16);
+    assert(accuracyLog < 31-tableLog);  /* ensure enough room for renormalization double shift */
+    {   U32 const tableSize = 1 << tableLog;
+        U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
+        U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog;   /* linear interpolation (very approximate) */
+        U32 const bitMultiplier = 1 << accuracyLog;
+        assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
+        assert(normalizedDeltaFromThreshold <= bitMultiplier);
+        return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
+    }
+}
+
+
+/* ======    Decompression    ====== */
+
+typedef struct {
+    U16 tableLog;
+    U16 fastMode;
+} FSE_DTableHeader;   /* sizeof U32 */
+
+typedef struct
+{
+    unsigned short newState;
+    unsigned char  symbol;
+    unsigned char  nbBits;
+} FSE_decode_t;   /* size == U32 */
+
+MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
+{
+    const void* ptr = dt;
+    const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
+    DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
+    BIT_reloadDStream(bitD);
+    DStatePtr->table = dt + 1;
+}
+
+MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
+{
+    FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+    return DInfo.symbol;
+}
+
+MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+    FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+    U32 const nbBits = DInfo.nbBits;
+    size_t const lowBits = BIT_readBits(bitD, nbBits);
+    DStatePtr->state = DInfo.newState + lowBits;
+}
+
+MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+    FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+    U32 const nbBits = DInfo.nbBits;
+    BYTE const symbol = DInfo.symbol;
+    size_t const lowBits = BIT_readBits(bitD, nbBits);
+
+    DStatePtr->state = DInfo.newState + lowBits;
+    return symbol;
+}
+
+/*! FSE_decodeSymbolFast() :
+    unsafe, only works if no symbol has a probability > 50% */
+MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+    FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+    U32 const nbBits = DInfo.nbBits;
+    BYTE const symbol = DInfo.symbol;
+    size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
+
+    DStatePtr->state = DInfo.newState + lowBits;
+    return symbol;
+}
+
+MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
+{
+    return DStatePtr->state == 0;
+}
+
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/* **************************************************************
+*  Tuning parameters
+****************************************************************/
+/*!MEMORY_USAGE :
+*  Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+*  Increasing memory usage improves compression ratio
+*  Reduced memory usage can improve speed, due to cache effect
+*  Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
+#ifndef FSE_MAX_MEMORY_USAGE
+#  define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+#  define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
+#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE)
+#  error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE"
+#endif
+
+/*!FSE_MAX_SYMBOL_VALUE :
+*  Maximum symbol value authorized.
+*  Required for proper stack allocation */
+#ifndef FSE_MAX_SYMBOL_VALUE
+#  define FSE_MAX_SYMBOL_VALUE 255
+#endif
+
+/* **************************************************************
+*  template functions type & suffix
+****************************************************************/
+#define FSE_FUNCTION_TYPE BYTE
+#define FSE_FUNCTION_EXTENSION
+#define FSE_DECODE_TYPE FSE_decode_t
+
+
+#endif   /* !FSE_COMMONDEFS_ONLY */
+
+
+/* ***************************************************************
+*  Constants
+*****************************************************************/
+#define FSE_MAX_TABLELOG  (FSE_MAX_MEMORY_USAGE-2)
+#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
+#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
+#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
+#define FSE_MIN_TABLELOG 5
+
+#define FSE_TABLELOG_ABSOLUTE_MAX 15
+#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
+#  error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
+#endif
+
+#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
+
+
+#endif /* FSE_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
+/**** ended inlining fse.h ****/
+/**** start inlining huf.h ****/
+/* ******************************************************************
+ * huff0 huffman codec,
+ * part of Finite State Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef HUF_H_298734234
+#define HUF_H_298734234
+
+/* *** Dependencies *** */
+/**** skipping file: zstd_deps.h ****/
+/**** skipping file: mem.h ****/
+#define FSE_STATIC_LINKING_ONLY
+/**** skipping file: fse.h ****/
+
+
+/* ***   Tool functions *** */
+#define HUF_BLOCKSIZE_MAX (128 * 1024)   /**< maximum input size for a single block compressed with HUF_compress */
+size_t HUF_compressBound(size_t size);   /**< maximum compressed size (worst case) */
+
+/* Error Management */
+unsigned    HUF_isError(size_t code);       /**< tells if a return value is an error code */
+const char* HUF_getErrorName(size_t code);  /**< provides error code string (useful for debugging) */
+
+
+#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
+#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
+
+/* *** Constants *** */
+#define HUF_TABLELOG_MAX      12      /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
+#define HUF_TABLELOG_DEFAULT  11      /* default tableLog value when none specified */
+#define HUF_SYMBOLVALUE_MAX  255
+
+#define HUF_TABLELOG_ABSOLUTEMAX  12  /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
+#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
+#  error "HUF_TABLELOG_MAX is too large !"
+#endif
+
+
+/* ****************************************
+*  Static allocation
+******************************************/
+/* HUF buffer bounds */
+#define HUF_CTABLEBOUND 129
+#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8)   /* only true when incompressible is pre-filtered with fast heuristic */
+#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size))   /* Macro version, useful for static allocation */
+
+/* static allocation of HUF's Compression Table */
+/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
+typedef size_t HUF_CElt;   /* consider it an incomplete type */
+#define HUF_CTABLE_SIZE_ST(maxSymbolValue)   ((maxSymbolValue)+2)   /* Use tables of size_t, for proper alignment */
+#define HUF_CTABLE_SIZE(maxSymbolValue)       (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
+#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
+    HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
+
+/* static allocation of HUF's DTable */
+typedef U32 HUF_DTable;
+#define HUF_DTABLE_SIZE(maxTableLog)   (1 + (1<<(maxTableLog)))
+#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
+        HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
+#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
+        HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
+
+
+/* ****************************************
+*  Advanced decompression functions
+******************************************/
+
+/**
+ * Huffman flags bitset.
+ * For all flags, 0 is the default value.
+ */
+typedef enum {
+    /**
+     * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
+     * Otherwise: Ignored.
+     */
+    HUF_flags_bmi2 = (1 << 0),
+    /**
+     * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
+     * If unset: Use heuristic to find the table depth.
+     */
+    HUF_flags_optimalDepth = (1 << 1),
+    /**
+     * If set: If the previous table can encode the input, always reuse the previous table.
+     * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
+     */
+    HUF_flags_preferRepeat = (1 << 2),
+    /**
+     * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
+     * If unset: Always histogram the entire input.
+     */
+    HUF_flags_suspectUncompressible = (1 << 3),
+    /**
+     * If set: Don't use assembly implementations
+     * If unset: Allow using assembly implementations
+     */
+    HUF_flags_disableAsm = (1 << 4),
+    /**
+     * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
+     * If unset: Use the fast decoding loop when possible.
+     */
+    HUF_flags_disableFast = (1 << 5)
+} HUF_flags_e;
+
+
+/* ****************************************
+ *  HUF detailed API
+ * ****************************************/
+#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
+
+/*! HUF_compress() does the following:
+ *  1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
+ *  2. (optional) refine tableLog using HUF_optimalTableLog()
+ *  3. build Huffman table from count using HUF_buildCTable()
+ *  4. save Huffman table to memory buffer using HUF_writeCTable()
+ *  5. encode the data stream using HUF_compress4X_usingCTable()
+ *
+ *  The following API allows targeting specific sub-functions for advanced tasks.
+ *  For example, it's possible to compress several blocks using the same 'CTable',
+ *  or to save and regenerate 'CTable' using external methods.
+ */
+unsigned HUF_minTableLog(unsigned symbolCardinality);
+unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
+ size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
+size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
+size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
+int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
+
+typedef enum {
+   HUF_repeat_none,  /**< Cannot use the previous table */
+   HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
+   HUF_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
+ } HUF_repeat;
+
+/** HUF_compress4X_repeat() :
+ *  Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ *  If it uses hufTable it does not modify hufTable or repeat.
+ *  If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ *  If preferRepeat then the old table will always be used if valid.
+ *  If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
+                       const void* src, size_t srcSize,
+                       unsigned maxSymbolValue, unsigned tableLog,
+                       void* workSpace, size_t wkspSize,    /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
+
+/** HUF_buildCTable_wksp() :
+ *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
+ */
+#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
+#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
+size_t HUF_buildCTable_wksp (HUF_CElt* tree,
+                       const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
+                             void* workSpace, size_t wkspSize);
+
+/*! HUF_readStats() :
+ *  Read compact Huffman tree, saved by HUF_writeCTable().
+ * `huffWeight` is destination buffer.
+ * @return : size read from `src` , or an error Code .
+ *  Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
+size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
+                     U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
+                     const void* src, size_t srcSize);
+
+/*! HUF_readStats_wksp() :
+ * Same as HUF_readStats() but takes an external workspace which must be
+ * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
+ */
+#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
+#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
+size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
+                          U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
+                          const void* src, size_t srcSize,
+                          void* workspace, size_t wkspSize,
+                          int flags);
+
+/** HUF_readCTable() :
+ *  Loading a CTable saved with HUF_writeCTable() */
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
+
+/** HUF_getNbBitsFromCTable() :
+ *  Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
+ *  Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0
+ *  Note 2 : is not inlined, as HUF_CElt definition is private
+ */
+U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
+
+typedef struct {
+    BYTE tableLog;
+    BYTE maxSymbolValue;
+    BYTE unused[sizeof(size_t) - 2];
+} HUF_CTableHeader;
+
+/** HUF_readCTableHeader() :
+ *  @returns The header from the CTable specifying the tableLog and the maxSymbolValue.
+ */
+HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable);
+
+/*
+ * HUF_decompress() does the following:
+ * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
+ * 2. build Huffman table from save, using HUF_readDTableX?()
+ * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
+ */
+
+/** HUF_selectDecoder() :
+ *  Tells which decoder is likely to decode faster,
+ *  based on a set of pre-computed metrics.
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
+ *  Assumption : 0 < dstSize <= 128 KB */
+U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
+
+/**
+ *  The minimum workspace size for the `workSpace` used in
+ *  HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
+ *
+ *  The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
+ *  HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
+ *  Buffer overflow errors may potentially occur if code modifications result in
+ *  a required workspace size greater than that specified in the following
+ *  macro.
+ */
+#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
+#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
+
+
+/* ====================== */
+/* single stream variants */
+/* ====================== */
+
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
+/** HUF_compress1X_repeat() :
+ *  Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ *  If it uses hufTable it does not modify hufTable or repeat.
+ *  If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ *  If preferRepeat then the old table will always be used if valid.
+ *  If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
+                       const void* src, size_t srcSize,
+                       unsigned maxSymbolValue, unsigned tableLog,
+                       void* workSpace, size_t wkspSize,   /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X1
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);   /**< double-symbols decoder */
+#endif
+
+/* BMI2 variants.
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
+ */
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+#ifndef HUF_FORCE_DECOMPRESS_X1
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+
+#endif   /* HUF_H_298734234 */
+
+#if defined (__cplusplus)
+}
+#endif
+/**** ended inlining huf.h ****/
+/**** skipping file: bits.h ****/
+
+
+/*===   Version   ===*/
+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
+
+
+/*===   Error Management   ===*/
+unsigned FSE_isError(size_t code) { return ERR_isError(code); }
+const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+unsigned HUF_isError(size_t code) { return ERR_isError(code); }
+const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+
+/*-**************************************************************
+*  FSE NCount encoding-decoding
+****************************************************************/
+FORCE_INLINE_TEMPLATE
+size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+                           const void* headerBuffer, size_t hbSize)
+{
+    const BYTE* const istart = (const BYTE*) headerBuffer;
+    const BYTE* const iend = istart + hbSize;
+    const BYTE* ip = istart;
+    int nbBits;
+    int remaining;
+    int threshold;
+    U32 bitStream;
+    int bitCount;
+    unsigned charnum = 0;
+    unsigned const maxSV1 = *maxSVPtr + 1;
+    int previous0 = 0;
+
+    if (hbSize < 8) {
+        /* This function only works when hbSize >= 8 */
+        char buffer[8] = {0};
+        ZSTD_memcpy(buffer, headerBuffer, hbSize);
+        {   size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
+                                                    buffer, sizeof(buffer));
+            if (FSE_isError(countSize)) return countSize;
+            if (countSize > hbSize) return ERROR(corruption_detected);
+            return countSize;
+    }   }
+    assert(hbSize >= 8);
+
+    /* init */
+    ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0]));   /* all symbols not present in NCount have a frequency of 0 */
+    bitStream = MEM_readLE32(ip);
+    nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG;   /* extract tableLog */
+    if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
+    bitStream >>= 4;
+    bitCount = 4;
+    *tableLogPtr = nbBits;
+    remaining = (1<<nbBits)+1;
+    threshold = 1<<nbBits;
+    nbBits++;
+
+    for (;;) {
+        if (previous0) {
+            /* Count the number of repeats. Each time the
+             * 2-bit repeat code is 0b11 there is another
+             * repeat.
+             * Avoid UB by setting the high bit to 1.
+             */
+            int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
+            while (repeats >= 12) {
+                charnum += 3 * 12;
+                if (LIKELY(ip <= iend-7)) {
+                    ip += 3;
+                } else {
+                    bitCount -= (int)(8 * (iend - 7 - ip));
+                    bitCount &= 31;
+                    ip = iend - 4;
+                }
+                bitStream = MEM_readLE32(ip) >> bitCount;
+                repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
+            }
+            charnum += 3 * repeats;
+            bitStream >>= 2 * repeats;
+            bitCount += 2 * repeats;
+
+            /* Add the final repeat which isn't 0b11. */
+            assert((bitStream & 3) < 3);
+            charnum += bitStream & 3;
+            bitCount += 2;
+
+            /* This is an error, but break and return an error
+             * at the end, because returning out of a loop makes
+             * it harder for the compiler to optimize.
+             */
+            if (charnum >= maxSV1) break;
+
+            /* We don't need to set the normalized count to 0
+             * because we already memset the whole buffer to 0.
+             */
+
+            if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
+                assert((bitCount >> 3) <= 3); /* For first condition to work */
+                ip += bitCount>>3;
+                bitCount &= 7;
+            } else {
+                bitCount -= (int)(8 * (iend - 4 - ip));
+                bitCount &= 31;
+                ip = iend - 4;
+            }
+            bitStream = MEM_readLE32(ip) >> bitCount;
+        }
+        {
+            int const max = (2*threshold-1) - remaining;
+            int count;
+
+            if ((bitStream & (threshold-1)) < (U32)max) {
+                count = bitStream & (threshold-1);
+                bitCount += nbBits-1;
+            } else {
+                count = bitStream & (2*threshold-1);
+                if (count >= threshold) count -= max;
+                bitCount += nbBits;
+            }
+
+            count--;   /* extra accuracy */
+            /* When it matters (small blocks), this is a
+             * predictable branch, because we don't use -1.
+             */
+            if (count >= 0) {
+                remaining -= count;
+            } else {
+                assert(count == -1);
+                remaining += count;
+            }
+            normalizedCounter[charnum++] = (short)count;
+            previous0 = !count;
+
+            assert(threshold > 1);
+            if (remaining < threshold) {
+                /* This branch can be folded into the
+                 * threshold update condition because we
+                 * know that threshold > 1.
+                 */
+                if (remaining <= 1) break;
+                nbBits = ZSTD_highbit32(remaining) + 1;
+                threshold = 1 << (nbBits - 1);
+            }
+            if (charnum >= maxSV1) break;
+
+            if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
+                ip += bitCount>>3;
+                bitCount &= 7;
+            } else {
+                bitCount -= (int)(8 * (iend - 4 - ip));
+                bitCount &= 31;
+                ip = iend - 4;
+            }
+            bitStream = MEM_readLE32(ip) >> bitCount;
+    }   }
+    if (remaining != 1) return ERROR(corruption_detected);
+    /* Only possible when there are too many zeros. */
+    if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall);
+    if (bitCount > 32) return ERROR(corruption_detected);
+    *maxSVPtr = charnum-1;
+
+    ip += (bitCount+7)>>3;
+    return ip-istart;
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t FSE_readNCount_body_default(
+        short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+        const void* headerBuffer, size_t hbSize)
+{
+    return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
+        short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+        const void* headerBuffer, size_t hbSize)
+{
+    return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+#endif
+
+size_t FSE_readNCount_bmi2(
+        short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+        const void* headerBuffer, size_t hbSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+    if (bmi2) {
+        return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+    }
+#endif
+    (void)bmi2;
+    return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+
+size_t FSE_readNCount(
+        short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+        const void* headerBuffer, size_t hbSize)
+{
+    return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0);
+}
+
+
+/*! HUF_readStats() :
+    Read compact Huffman tree, saved by HUF_writeCTable().
+    `huffWeight` is destination buffer.
+    `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
+    @return : size read from `src` , or an error Code .
+    Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
+*/
+size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+                     U32* nbSymbolsPtr, U32* tableLogPtr,
+                     const void* src, size_t srcSize)
+{
+    U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+    return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+                   U32* nbSymbolsPtr, U32* tableLogPtr,
+                   const void* src, size_t srcSize,
+                   void* workSpace, size_t wkspSize,
+                   int bmi2)
+{
+    U32 weightTotal;
+    const BYTE* ip = (const BYTE*) src;
+    size_t iSize;
+    size_t oSize;
+
+    if (!srcSize) return ERROR(srcSize_wrong);
+    iSize = ip[0];
+    /* ZSTD_memset(huffWeight, 0, hwSize);   *//* is not necessary, even though some analyzer complain ... */
+
+    if (iSize >= 128) {  /* special header */
+        oSize = iSize - 127;
+        iSize = ((oSize+1)/2);
+        if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
+        if (oSize >= hwSize) return ERROR(corruption_detected);
+        ip += 1;
+        {   U32 n;
+            for (n=0; n<oSize; n+=2) {
+                huffWeight[n]   = ip[n/2] >> 4;
+                huffWeight[n+1] = ip[n/2] & 15;
+    }   }   }
+    else  {   /* header compressed with FSE (normal case) */
+        if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
+        /* max (hwSize-1) values decoded, as last one is implied */
+        oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2);
+        if (FSE_isError(oSize)) return oSize;
+    }
+
+    /* collect weight stats */
+    ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
+    weightTotal = 0;
+    {   U32 n; for (n=0; n<oSize; n++) {
+            if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
+            rankStats[huffWeight[n]]++;
+            weightTotal += (1 << huffWeight[n]) >> 1;
+    }   }
+    if (weightTotal == 0) return ERROR(corruption_detected);
+
+    /* get last non-null symbol weight (implied, total must be 2^n) */
+    {   U32 const tableLog = ZSTD_highbit32(weightTotal) + 1;
+        if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
+        *tableLogPtr = tableLog;
+        /* determine last weight */
+        {   U32 const total = 1 << tableLog;
+            U32 const rest = total - weightTotal;
+            U32 const verif = 1 << ZSTD_highbit32(rest);
+            U32 const lastWeight = ZSTD_highbit32(rest) + 1;
+            if (verif != rest) return ERROR(corruption_detected);    /* last value must be a clean power of 2 */
+            huffWeight[oSize] = (BYTE)lastWeight;
+            rankStats[lastWeight]++;
+    }   }
+
+    /* check tree construction validity */
+    if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected);   /* by construction : at least 2 elts of rank 1, must be even */
+
+    /* results */
+    *nbSymbolsPtr = (U32)(oSize+1);
+    return iSize+1;
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+                     U32* nbSymbolsPtr, U32* tableLogPtr,
+                     const void* src, size_t srcSize,
+                     void* workSpace, size_t wkspSize)
+{
+    return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
+}
+
+#if DYNAMIC_BMI2
+static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+                     U32* nbSymbolsPtr, U32* tableLogPtr,
+                     const void* src, size_t srcSize,
+                     void* workSpace, size_t wkspSize)
+{
+    return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
+}
+#endif
+
+size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+                     U32* nbSymbolsPtr, U32* tableLogPtr,
+                     const void* src, size_t srcSize,
+                     void* workSpace, size_t wkspSize,
+                     int flags)
+{
+#if DYNAMIC_BMI2
+    if (flags & HUF_flags_bmi2) {
+        return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
+    }
+#endif
+    (void)flags;
+    return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
+}
+/**** ended inlining common/entropy_common.c ****/
+/**** start inlining common/error_private.c ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* The purpose of this file is to have a single list of error strings embedded in binary */
+
+/**** skipping file: error_private.h ****/
+
+const char* ERR_getErrorString(ERR_enum code)
+{
+#ifdef ZSTD_STRIP_ERROR_STRINGS
+    (void)code;
+    return "Error strings stripped";
+#else
+    static const char* const notErrorCode = "Unspecified error code";
+    switch( code )
+    {
+    case PREFIX(no_error): return "No error detected";
+    case PREFIX(GENERIC):  return "Error (generic)";
+    case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+    case PREFIX(version_unsupported): return "Version not supported";
+    case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
+    case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
+    case PREFIX(corruption_detected): return "Data corruption detected";
+    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
+    case PREFIX(parameter_unsupported): return "Unsupported parameter";
+    case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
+    case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
+    case PREFIX(init_missing): return "Context should be init first";
+    case PREFIX(memory_allocation): return "Allocation error : not enough memory";
+    case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
+    case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
+    case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
+    case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
+    case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
+    case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
+    case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
+    case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+    case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
+    case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+    case PREFIX(srcSize_wrong): return "Src size is incorrect";
+    case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
+    case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
+    case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
+        /* following error codes are not stable and may be removed or changed in a future version */
+    case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
+    case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
+    case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
+    case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
+    case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
+    case PREFIX(externalSequences_invalid): return "External sequences are not valid";
+    case PREFIX(maxCode):
+    default: return notErrorCode;
+    }
+#endif
+}
+/**** ended inlining common/error_private.c ****/
+/**** start inlining common/fse_decompress.c ****/
+/* ******************************************************************
+ * FSE : Finite State Entropy decoder
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ *  You can contact the author at :
+ *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *  - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/* **************************************************************
+*  Includes
+****************************************************************/
+/**** skipping file: debug.h ****/
+/**** skipping file: bitstream.h ****/
+/**** skipping file: compiler.h ****/
+#define FSE_STATIC_LINKING_ONLY
+/**** skipping file: fse.h ****/
+/**** skipping file: error_private.h ****/
+/**** skipping file: zstd_deps.h ****/
+/**** skipping file: bits.h ****/
+
+
+/* **************************************************************
+*  Error Management
+****************************************************************/
+#define FSE_isError ERR_isError
+#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)   /* use only *after* variable declarations */
+
+
+/* **************************************************************
+*  Templates
+****************************************************************/
+/*
+  designed to be included
+  for type-specific functions (template emulation in C)
+  Objective is to write these functions only once, for improved maintenance
+*/
+
+/* safety checks */
+#ifndef FSE_FUNCTION_EXTENSION
+#  error "FSE_FUNCTION_EXTENSION must be defined"
+#endif
+#ifndef FSE_FUNCTION_TYPE
+#  error "FSE_FUNCTION_TYPE must be defined"
+#endif
+
+/* Function names */
+#define FSE_CAT(X,Y) X##Y
+#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
+#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
+
+static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
+{
+    void* const tdPtr = dt+1;   /* because *dt is unsigned, 32-bits aligned on 32-bits */
+    FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
+    U16* symbolNext = (U16*)workSpace;
+    BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1);
+
+    U32 const maxSV1 = maxSymbolValue + 1;
+    U32 const tableSize = 1 << tableLog;
+    U32 highThreshold = tableSize-1;
+
+    /* Sanity Checks */
+    if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge);
+    if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
+    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
+
+    /* Init, lay down lowprob symbols */
+    {   FSE_DTableHeader DTableH;
+        DTableH.tableLog = (U16)tableLog;
+        DTableH.fastMode = 1;
+        {   S16 const largeLimit= (S16)(1 << (tableLog-1));
+            U32 s;
+            for (s=0; s<maxSV1; s++) {
+                if (normalizedCounter[s]==-1) {
+                    tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
+                    symbolNext[s] = 1;
+                } else {
+                    if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
+                    symbolNext[s] = (U16)normalizedCounter[s];
+        }   }   }
+        ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
+    }
+
+    /* Spread symbols */
+    if (highThreshold == tableSize - 1) {
+        size_t const tableMask = tableSize-1;
+        size_t const step = FSE_TABLESTEP(tableSize);
+        /* First lay down the symbols in order.
+         * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
+         * misses since small blocks generally have small table logs, so nearly
+         * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
+         * our buffer to handle the over-write.
+         */
+        {   U64 const add = 0x0101010101010101ull;
+            size_t pos = 0;
+            U64 sv = 0;
+            U32 s;
+            for (s=0; s<maxSV1; ++s, sv += add) {
+                int i;
+                int const n = normalizedCounter[s];
+                MEM_write64(spread + pos, sv);
+                for (i = 8; i < n; i += 8) {
+                    MEM_write64(spread + pos + i, sv);
+                }
+                pos += (size_t)n;
+        }   }
+        /* Now we spread those positions across the table.
+         * The benefit of doing it in two stages is that we avoid the
+         * variable size inner loop, which caused lots of branch misses.
+         * Now we can run through all the positions without any branch misses.
+         * We unroll the loop twice, since that is what empirically worked best.
+         */
+        {
+            size_t position = 0;
+            size_t s;
+            size_t const unroll = 2;
+            assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+            for (s = 0; s < (size_t)tableSize; s += unroll) {
+                size_t u;
+                for (u = 0; u < unroll; ++u) {
+                    size_t const uPosition = (position + (u * step)) & tableMask;
+                    tableDecode[uPosition].symbol = spread[s + u];
+                }
+                position = (position + (unroll * step)) & tableMask;
+            }
+            assert(position == 0);
+        }
+    } else {
+        U32 const tableMask = tableSize-1;
+        U32 const step = FSE_TABLESTEP(tableSize);
+        U32 s, position = 0;
+        for (s=0; s<maxSV1; s++) {
+            int i;
+            for (i=0; i<normalizedCounter[s]; i++) {
+                tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
+                position = (position + step) & tableMask;
+                while (position > highThreshold) position = (position + step) & tableMask;   /* lowprob area */
+        }   }
+        if (position!=0) return ERROR(GENERIC);   /* position must reach all cells once, otherwise normalizedCounter is incorrect */
+    }
+
+    /* Build Decoding table */
+    {   U32 u;
+        for (u=0; u<tableSize; u++) {
+            FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
+            U32 const nextState = symbolNext[symbol]++;
+            tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
+            tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
+    }   }
+
+    return 0;
+}
+
+size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
+{
+    return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
+}
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/*-*******************************************************
+*  Decompression (Byte symbols)
+*********************************************************/
+
+FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
+          void* dst, size_t maxDstSize,
+    const void* cSrc, size_t cSrcSize,
+    const FSE_DTable* dt, const unsigned fast)
+{
+    BYTE* const ostart = (BYTE*) dst;
+    BYTE* op = ostart;
+    BYTE* const omax = op + maxDstSize;
+    BYTE* const olimit = omax-3;
+
+    BIT_DStream_t bitD;
+    FSE_DState_t state1;
+    FSE_DState_t state2;
+
+    /* Init */
+    CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
+
+    FSE_initDState(&state1, &bitD, dt);
+    FSE_initDState(&state2, &bitD, dt);
+
+    RETURN_ERROR_IF(BIT_reloadDStream(&bitD)==BIT_DStream_overflow, corruption_detected, "");
+
+#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
+
+    /* 4 symbols per loop */
+    for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
+        op[0] = FSE_GETSYMBOL(&state1);
+
+        if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8)    /* This test must be static */
+            BIT_reloadDStream(&bitD);
+
+        op[1] = FSE_GETSYMBOL(&state2);
+
+        if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8)    /* This test must be static */
+            { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
+
+        op[2] = FSE_GETSYMBOL(&state1);
+
+        if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8)    /* This test must be static */
+            BIT_reloadDStream(&bitD);
+
+        op[3] = FSE_GETSYMBOL(&state2);
+    }
+
+    /* tail */
+    /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
+    while (1) {
+        if (op>(omax-2)) return ERROR(dstSize_tooSmall);
+        *op++ = FSE_GETSYMBOL(&state1);
+        if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
+            *op++ = FSE_GETSYMBOL(&state2);
+            break;
+        }
+
+        if (op>(omax-2)) return ERROR(dstSize_tooSmall);
+        *op++ = FSE_GETSYMBOL(&state2);
+        if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
+            *op++ = FSE_GETSYMBOL(&state1);
+            break;
+    }   }
+
+    assert(op >= ostart);
+    return (size_t)(op-ostart);
+}
+
+typedef struct {
+    short ncount[FSE_MAX_SYMBOL_VALUE + 1];
+} FSE_DecompressWksp;
+
+
+FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
+        void* dst, size_t dstCapacity,
+        const void* cSrc, size_t cSrcSize,
+        unsigned maxLog, void* workSpace, size_t wkspSize,
+        int bmi2)
+{
+    const BYTE* const istart = (const BYTE*)cSrc;
+    const BYTE* ip = istart;
+    unsigned tableLog;
+    unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
+    FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
+    size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable);
+    FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos;
+
+    FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
+    if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
+
+    /* correct offset to dtable depends on this property */
+    FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0);
+
+    /* normal FSE decoding mode */
+    {   size_t const NCountLength =
+            FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
+        if (FSE_isError(NCountLength)) return NCountLength;
+        if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
+        assert(NCountLength <= cSrcSize);
+        ip += NCountLength;
+        cSrcSize -= NCountLength;
+    }
+
+    if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
+    assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize);
+    workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
+    wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
+
+    CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
+
+    {
+        const void* ptr = dtable;
+        const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
+        const U32 fastMode = DTableH->fastMode;
+
+        /* select fast mode (static) */
+        if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 1);
+        return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 0);
+    }
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
+{
+    return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
+{
+    return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
+}
+#endif
+
+size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+    if (bmi2) {
+        return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
+    }
+#endif
+    (void)bmi2;
+    return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
+}
+
+#endif   /* FSE_COMMONDEFS_ONLY */
+/**** ended inlining common/fse_decompress.c ****/
+/**** start inlining common/zstd_common.c ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+
+/*-*************************************
+*  Dependencies
+***************************************/
+#define ZSTD_DEPS_NEED_MALLOC
+/**** skipping file: error_private.h ****/
+/**** start inlining zstd_internal.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_CCOMMON_H_MODULE
+#define ZSTD_CCOMMON_H_MODULE
+
+/* this module contains definitions which must be identical
+ * across compression, decompression and dictBuilder.
+ * It also contains a few functions useful to at least 2 of them
+ * and which benefit from being inlined */
+
+/*-*************************************
+*  Dependencies
+***************************************/
+/**** skipping file: compiler.h ****/
+/**** start inlining cpu.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMMON_CPU_H
+#define ZSTD_COMMON_CPU_H
+
+/**
+ * Implementation taken from folly/CpuId.h
+ * https://github.com/facebook/folly/blob/master/folly/CpuId.h
+ */
+
+/**** skipping file: mem.h ****/
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+typedef struct {
+    U32 f1c;
+    U32 f1d;
+    U32 f7b;
+    U32 f7c;
+} ZSTD_cpuid_t;
+
+MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
+    U32 f1c = 0;
+    U32 f1d = 0;
+    U32 f7b = 0;
+    U32 f7c = 0;
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16
+    int reg[4];
+    __cpuid((int*)reg, 0);
+    {
+        int const n = reg[0];
+        if (n >= 1) {
+            __cpuid((int*)reg, 1);
+            f1c = (U32)reg[2];
+            f1d = (U32)reg[3];
+        }
+        if (n >= 7) {
+            __cpuidex((int*)reg, 7, 0);
+            f7b = (U32)reg[1];
+            f7c = (U32)reg[2];
+        }
+    }
+#else
+    /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in
+     * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs
+     * to due to being a reserved register. So in that case, do the `cpuid`
+     * ourselves. Clang supports inline assembly anyway.
+     */
+    U32 n;
+    __asm__(
+        "pushq %%rbx\n\t"
+        "cpuid\n\t"
+        "popq %%rbx\n\t"
+        : "=a"(n)
+        : "a"(0)
+        : "rcx", "rdx");
+    if (n >= 1) {
+      U32 f1a;
+      __asm__(
+          "pushq %%rbx\n\t"
+          "cpuid\n\t"
+          "popq %%rbx\n\t"
+          : "=a"(f1a), "=c"(f1c), "=d"(f1d)
+          : "a"(1)
+          :);
+    }
+    if (n >= 7) {
+      __asm__(
+          "pushq %%rbx\n\t"
+          "cpuid\n\t"
+          "movq %%rbx, %%rax\n\t"
+          "popq %%rbx"
+          : "=a"(f7b), "=c"(f7c)
+          : "a"(7), "c"(0)
+          : "rdx");
+    }
+#endif
+#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
+    /* The following block like the normal cpuid branch below, but gcc
+     * reserves ebx for use of its pic register so we must specially
+     * handle the save and restore to avoid clobbering the register
+     */
+    U32 n;
+    __asm__(
+        "pushl %%ebx\n\t"
+        "cpuid\n\t"
+        "popl %%ebx\n\t"
+        : "=a"(n)
+        : "a"(0)
+        : "ecx", "edx");
+    if (n >= 1) {
+      U32 f1a;
+      __asm__(
+          "pushl %%ebx\n\t"
+          "cpuid\n\t"
+          "popl %%ebx\n\t"
+          : "=a"(f1a), "=c"(f1c), "=d"(f1d)
+          : "a"(1));
+    }
+    if (n >= 7) {
+      __asm__(
+          "pushl %%ebx\n\t"
+          "cpuid\n\t"
+          "movl %%ebx, %%eax\n\t"
+          "popl %%ebx"
+          : "=a"(f7b), "=c"(f7c)
+          : "a"(7), "c"(0)
+          : "edx");
+    }
+#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
+    U32 n;
+    __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
+    if (n >= 1) {
+      U32 f1a;
+      __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
+    }
+    if (n >= 7) {
+      U32 f7a;
+      __asm__("cpuid"
+              : "=a"(f7a), "=b"(f7b), "=c"(f7c)
+              : "a"(7), "c"(0)
+              : "edx");
+    }
+#endif
+    {
+        ZSTD_cpuid_t cpuid;
+        cpuid.f1c = f1c;
+        cpuid.f1d = f1d;
+        cpuid.f7b = f7b;
+        cpuid.f7c = f7c;
+        return cpuid;
+    }
+}
+
+#define X(name, r, bit)                                                        \
+  MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) {                 \
+    return ((cpuid.r) & (1U << bit)) != 0;                                     \
+  }
+
+/* cpuid(1): Processor Info and Feature Bits. */
+#define C(name, bit) X(name, f1c, bit)
+  C(sse3, 0)
+  C(pclmuldq, 1)
+  C(dtes64, 2)
+  C(monitor, 3)
+  C(dscpl, 4)
+  C(vmx, 5)
+  C(smx, 6)
+  C(eist, 7)
+  C(tm2, 8)
+  C(ssse3, 9)
+  C(cnxtid, 10)
+  C(fma, 12)
+  C(cx16, 13)
+  C(xtpr, 14)
+  C(pdcm, 15)
+  C(pcid, 17)
+  C(dca, 18)
+  C(sse41, 19)
+  C(sse42, 20)
+  C(x2apic, 21)
+  C(movbe, 22)
+  C(popcnt, 23)
+  C(tscdeadline, 24)
+  C(aes, 25)
+  C(xsave, 26)
+  C(osxsave, 27)
+  C(avx, 28)
+  C(f16c, 29)
+  C(rdrand, 30)
+#undef C
+#define D(name, bit) X(name, f1d, bit)
+  D(fpu, 0)
+  D(vme, 1)
+  D(de, 2)
+  D(pse, 3)
+  D(tsc, 4)
+  D(msr, 5)
+  D(pae, 6)
+  D(mce, 7)
+  D(cx8, 8)
+  D(apic, 9)
+  D(sep, 11)
+  D(mtrr, 12)
+  D(pge, 13)
+  D(mca, 14)
+  D(cmov, 15)
+  D(pat, 16)
+  D(pse36, 17)
+  D(psn, 18)
+  D(clfsh, 19)
+  D(ds, 21)
+  D(acpi, 22)
+  D(mmx, 23)
+  D(fxsr, 24)
+  D(sse, 25)
+  D(sse2, 26)
+  D(ss, 27)
+  D(htt, 28)
+  D(tm, 29)
+  D(pbe, 31)
+#undef D
+
+/* cpuid(7): Extended Features. */
+#define B(name, bit) X(name, f7b, bit)
+  B(bmi1, 3)
+  B(hle, 4)
+  B(avx2, 5)
+  B(smep, 7)
+  B(bmi2, 8)
+  B(erms, 9)
+  B(invpcid, 10)
+  B(rtm, 11)
+  B(mpx, 14)
+  B(avx512f, 16)
+  B(avx512dq, 17)
+  B(rdseed, 18)
+  B(adx, 19)
+  B(smap, 20)
+  B(avx512ifma, 21)
+  B(pcommit, 22)
+  B(clflushopt, 23)
+  B(clwb, 24)
+  B(avx512pf, 26)
+  B(avx512er, 27)
+  B(avx512cd, 28)
+  B(sha, 29)
+  B(avx512bw, 30)
+  B(avx512vl, 31)
+#undef B
+#define C(name, bit) X(name, f7c, bit)
+  C(prefetchwt1, 0)
+  C(avx512vbmi, 1)
+#undef C
+
+#undef X
+
+#endif /* ZSTD_COMMON_CPU_H */
+/**** ended inlining cpu.h ****/
+/**** skipping file: mem.h ****/
+/**** skipping file: debug.h ****/
+/**** skipping file: error_private.h ****/
+#define ZSTD_STATIC_LINKING_ONLY
+/**** start inlining ../zstd.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef ZSTD_H_235446
+#define ZSTD_H_235446
+
+/* ======   Dependencies   ======*/
+#include <stddef.h>   /* size_t */
+
+
+/* =====   ZSTDLIB_API : control library symbols visibility   ===== */
+#ifndef ZSTDLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDLIB_VISIBILITY
+#    define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
+#  else
+#    define ZSTDLIB_VISIBLE
+#  endif
+#endif
+
+#ifndef ZSTDLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDLIB_HIDDEN
+#  endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDLIB_API ZSTDLIB_VISIBLE
+#endif
+
+/* Deprecation warnings :
+ * Should these warnings be a problem, it is generally possible to disable them,
+ * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
+ * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
+ */
+#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
+#  define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
+#else
+#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#    define ZSTD_DEPRECATED(message) [[deprecated(message)]]
+#  elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__)
+#    define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif defined(__GNUC__) && (__GNUC__ >= 3)
+#    define ZSTD_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
+#    define ZSTD_DEPRECATED(message)
+#  endif
+#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
+
+
+/*******************************************************************************
+  Introduction
+
+  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
+  real-time compression scenarios at zlib-level and better compression ratios.
+  The zstd compression library provides in-memory compression and decompression
+  functions.
+
+  The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
+  which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
+  caution, as they require more memory. The library also offers negative
+  compression levels, which extend the range of speed vs. ratio preferences.
+  The lower the level, the faster the speed (at the cost of compression).
+
+  Compression can be done in:
+    - a single step (described as Simple API)
+    - a single step, reusing a context (described as Explicit context)
+    - unbounded multiple steps (described as Streaming compression)
+
+  The compression ratio achievable on small data can be highly improved using
+  a dictionary. Dictionary compression can be performed in:
+    - a single step (described as Simple dictionary API)
+    - a single step, reusing a dictionary (described as Bulk-processing
+      dictionary API)
+
+  Advanced experimental functions can be accessed using
+  `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
+
+  Advanced experimental APIs should never be used with a dynamically-linked
+  library. They are not "stable"; their definitions or signatures may change in
+  the future. Only static linking is allowed.
+*******************************************************************************/
+
+/*------   Version   ------*/
+#define ZSTD_VERSION_MAJOR    1
+#define ZSTD_VERSION_MINOR    5
+#define ZSTD_VERSION_RELEASE  7
+#define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+
+/*! ZSTD_versionNumber() :
+ *  Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
+ZSTDLIB_API unsigned ZSTD_versionNumber(void);
+
+#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
+#define ZSTD_QUOTE(str) #str
+#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
+#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
+
+/*! ZSTD_versionString() :
+ *  Return runtime library version, like "1.4.5". Requires v1.3.0+. */
+ZSTDLIB_API const char* ZSTD_versionString(void);
+
+/* *************************************
+ *  Default constant
+ ***************************************/
+#ifndef ZSTD_CLEVEL_DEFAULT
+#  define ZSTD_CLEVEL_DEFAULT 3
+#endif
+
+/* *************************************
+ *  Constants
+ ***************************************/
+
+/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
+#define ZSTD_MAGICNUMBER            0xFD2FB528    /* valid since v0.8.0 */
+#define ZSTD_MAGIC_DICTIONARY       0xEC30A437    /* valid since v0.7.0 */
+#define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50    /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
+#define ZSTD_MAGIC_SKIPPABLE_MASK   0xFFFFFFF0
+
+#define ZSTD_BLOCKSIZELOG_MAX  17
+#define ZSTD_BLOCKSIZE_MAX     (1<<ZSTD_BLOCKSIZELOG_MAX)
+
+
+/***************************************
+*  Simple Core API
+***************************************/
+/*! ZSTD_compress() :
+ *  Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
+ *  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
+ *        enough space to successfully compress the data.
+ *  @return : compressed size written into `dst` (<= `dstCapacity),
+ *            or an error code if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                                  int compressionLevel);
+
+/*! ZSTD_decompress() :
+ * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
+ *  Multiple compressed frames can be decompressed at once with this method.
+ *  The result will be the concatenation of all decompressed frames, back to back.
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
+ *  First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
+ *  If maximum upper bound isn't known, prefer using streaming mode to decompress data.
+ * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ *           or an errorCode if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+                              const void* src, size_t compressedSize);
+
+
+/*======  Decompression helper functions  ======*/
+
+/*! ZSTD_getFrameContentSize() : requires v1.3.0+
+ * `src` should point to the start of a ZSTD encoded frame.
+ * `srcSize` must be at least as large as the frame header.
+ *           hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
+ * @return : - decompressed size of `src` frame content, if known
+ *           - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ *           - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
+ *  note 1 : a 0 return value means the frame is valid but "empty".
+ *  note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode).
+ *           When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ *           In which case, it's necessary to use streaming mode to decompress data.
+ *           Optionally, application can rely on some implicit limit,
+ *           as ZSTD_decompress() only needs an upper bound of decompressed size.
+ *           (For example, data could be necessarily cut into blocks <= 16 KB).
+ *  note 3 : decompressed size is always present when compression is completed using single-pass functions,
+ *           such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
+ *  note 4 : decompressed size can be very large (64-bits value),
+ *           potentially larger than what local system can handle as a single memory segment.
+ *           In which case, it's necessary to use streaming mode to decompress data.
+ *  note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ *           Always ensure return value fits within application's authorized limits.
+ *           Each application can set its own limits.
+ *  note 6 : This function replaces ZSTD_getDecompressedSize() */
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
+ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+
+/*! ZSTD_getDecompressedSize() :
+ *  NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
+ *  Both functions work the same way, but ZSTD_getDecompressedSize() blends
+ *  "empty", "unknown" and "error" results to the same return value (0),
+ *  while ZSTD_getFrameContentSize() gives them separate return values.
+ * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
+ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
+ * `src` should point to the start of a ZSTD frame or skippable frame.
+ * `srcSize` must be >= first frame size
+ * @return : the compressed size of the first frame starting at `src`,
+ *           suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ *        or an error code if input is invalid */
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
+
+/*======  Compression helper functions  ======*/
+
+/*! ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()`, or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize is too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+
+
+/*======  Error helper functions  ======*/
+/**** skipping file: zstd_errors.h ****/
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
+ZSTDLIB_API unsigned     ZSTD_isError(size_t result);      /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */
+ZSTDLIB_API const char*  ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */
+ZSTDLIB_API int          ZSTD_minCLevel(void);             /*!< minimum negative compression level allowed, requires v1.4.0+ */
+ZSTDLIB_API int          ZSTD_maxCLevel(void);             /*!< maximum compression level available */
+ZSTDLIB_API int          ZSTD_defaultCLevel(void);         /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
+
+
+/***************************************
+*  Explicit context
+***************************************/
+/*= Compression context
+ *  When compressing many times,
+ *  it is recommended to allocate a compression context just once,
+ *  and reuse it for each successive compression operation.
+ *  This will make the workload easier for system's memory.
+ *  Note : re-using context is just a speed / resource optimization.
+ *         It doesn't change the compression ratio, which remains identical.
+ *  Note 2: For parallel execution in multi-threaded environments,
+ *         use one different context per thread .
+ */
+typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
+ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  /* compatible with NULL pointer */
+
+/*! ZSTD_compressCCtx() :
+ *  Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
+ *  Important : in order to mirror `ZSTD_compress()` behavior,
+ *  this function compresses at the requested compression level,
+ *  __ignoring any other advanced parameter__ .
+ *  If any advanced parameter was set using the advanced API,
+ *  they will all be reset. Only @compressionLevel remains.
+ */
+ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
+                                     void* dst, size_t dstCapacity,
+                               const void* src, size_t srcSize,
+                                     int compressionLevel);
+
+/*= Decompression context
+ *  When decompressing many times,
+ *  it is recommended to allocate a context only once,
+ *  and reuse it for each successive compression operation.
+ *  This will make workload friendlier for system's memory.
+ *  Use one context per thread for parallel execution. */
+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
+ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  /* accept NULL pointer */
+
+/*! ZSTD_decompressDCtx() :
+ *  Same as ZSTD_decompress(),
+ *  requires an allocated ZSTD_DCtx.
+ *  Compatible with sticky parameters (see below).
+ */
+ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
+                                       void* dst, size_t dstCapacity,
+                                 const void* src, size_t srcSize);
+
+
+/*********************************************
+*  Advanced compression API (Requires v1.4.0+)
+**********************************************/
+
+/* API design :
+ *   Parameters are pushed one by one into an existing context,
+ *   using ZSTD_CCtx_set*() functions.
+ *   Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
+ *   "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
+ *   __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ .
+ *
+ *   It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
+ *
+ *   This API supersedes all other "advanced" API entry points in the experimental section.
+ *   In the future, we expect to remove API entry points from experimental which are redundant with this API.
+ */
+
+
+/* Compression strategies, listed from fastest to strongest */
+typedef enum { ZSTD_fast=1,
+               ZSTD_dfast=2,
+               ZSTD_greedy=3,
+               ZSTD_lazy=4,
+               ZSTD_lazy2=5,
+               ZSTD_btlazy2=6,
+               ZSTD_btopt=7,
+               ZSTD_btultra=8,
+               ZSTD_btultra2=9
+               /* note : new strategies _might_ be added in the future.
+                         Only the order (from fast to strong) is guaranteed */
+} ZSTD_strategy;
+
+typedef enum {
+
+    /* compression parameters
+     * Note: When compressing with a ZSTD_CDict these parameters are superseded
+     * by the parameters used to construct the ZSTD_CDict.
+     * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
+    ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
+                              * Note that exact compression parameters are dynamically determined,
+                              * depending on both compression level and srcSize (when known).
+                              * Default level is ZSTD_CLEVEL_DEFAULT==3.
+                              * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
+                              * Note 1 : it's possible to pass a negative compression level.
+                              * Note 2 : setting a level does not automatically set all other compression parameters
+                              *   to default. Setting this will however eventually dynamically impact the compression
+                              *   parameters which have not been manually set. The manually set
+                              *   ones will 'stick'. */
+    /* Advanced compression parameters :
+     * It's possible to pin down compression parameters to some specific values.
+     * In which case, these values are no longer dynamically selected by the compressor */
+    ZSTD_c_windowLog=101,    /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * This will set a memory budget for streaming decompression,
+                              * with larger values requiring more memory
+                              * and typically compressing more.
+                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+                              * Special: value 0 means "use default windowLog".
+                              * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
+                              *       requires explicitly allowing such size at streaming decompression stage. */
+    ZSTD_c_hashLog=102,      /* Size of the initial probe table, as a power of 2.
+                              * Resulting memory usage is (1 << (hashLog+2)).
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+                              * Larger tables improve compression ratio of strategies <= dFast,
+                              * and improve speed of strategies > dFast.
+                              * Special: value 0 means "use default hashLog". */
+    ZSTD_c_chainLog=103,     /* Size of the multi-probe search table, as a power of 2.
+                              * Resulting memory usage is (1 << (chainLog+2)).
+                              * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
+                              * Larger tables result in better and slower compression.
+                              * This parameter is useless for "fast" strategy.
+                              * It's still useful when using "dfast" strategy,
+                              * in which case it defines a secondary probe table.
+                              * Special: value 0 means "use default chainLog". */
+    ZSTD_c_searchLog=104,    /* Number of search attempts, as a power of 2.
+                              * More attempts result in better and slower compression.
+                              * This parameter is useless for "fast" and "dFast" strategies.
+                              * Special: value 0 means "use default searchLog". */
+    ZSTD_c_minMatch=105,     /* Minimum size of searched matches.
+                              * Note that Zstandard can still find matches of smaller size,
+                              * it just tweaks its search algorithm to look for this size and larger.
+                              * Larger values increase compression and decompression speed, but decrease ratio.
+                              * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
+                              * Note that currently, for all strategies < btopt, effective minimum is 4.
+                              *                    , for all strategies > fast, effective maximum is 6.
+                              * Special: value 0 means "use default minMatchLength". */
+    ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
+                              * For strategies btopt, btultra & btultra2:
+                              *     Length of Match considered "good enough" to stop search.
+                              *     Larger values make compression stronger, and slower.
+                              * For strategy fast:
+                              *     Distance between match sampling.
+                              *     Larger values make compression faster, and weaker.
+                              * Special: value 0 means "use default targetLength". */
+    ZSTD_c_strategy=107,     /* See ZSTD_strategy enum definition.
+                              * The higher the value of selected strategy, the more complex it is,
+                              * resulting in stronger and slower compression.
+                              * Special: value 0 means "use default strategy". */
+
+    ZSTD_c_targetCBlockSize=130, /* v1.5.6+
+                                  * Attempts to fit compressed block size into approximately targetCBlockSize.
+                                  * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
+                                  * Note that it's not a guarantee, just a convergence target (default:0).
+                                  * No target when targetCBlockSize == 0.
+                                  * This is helpful in low bandwidth streaming environments to improve end-to-end latency,
+                                  * when a client can make use of partial documents (a prominent example being Chrome).
+                                  * Note: this parameter is stable since v1.5.6.
+                                  * It was present as an experimental parameter in earlier versions,
+                                  * but it's not recommended using it with earlier library versions
+                                  * due to massive performance regressions.
+                                  */
+    /* LDM mode parameters */
+    ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
+                                     * This parameter is designed to improve compression ratio
+                                     * for large inputs, by finding large matches at long distance.
+                                     * It increases memory usage and window size.
+                                     * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
+                                     * except when expressly set to a different value.
+                                     * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
+                                     * compression strategy >= ZSTD_btopt (== compression level 16+) */
+    ZSTD_c_ldmHashLog=161,   /* Size of the table for long distance matching, as a power of 2.
+                              * Larger values increase memory usage and compression ratio,
+                              * but decrease compression speed.
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+                              * default: windowlog - 7.
+                              * Special: value 0 means "automatically determine hashlog". */
+    ZSTD_c_ldmMinMatch=162,  /* Minimum match size for long distance matcher.
+                              * Larger/too small values usually decrease compression ratio.
+                              * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
+                              * Special: value 0 means "use default value" (default: 64). */
+    ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
+                              * Larger values improve collision resolution but decrease compression speed.
+                              * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
+                              * Special: value 0 means "use default value" (default: 3). */
+    ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
+                              * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
+                              * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
+                              * Larger values improve compression speed.
+                              * Deviating far from default value will likely result in a compression ratio decrease.
+                              * Special: value 0 means "automatically determine hashRateLog". */
+
+    /* frame parameters */
+    ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+                              * Content size must be known at the beginning of compression.
+                              * This is automatically the case when using ZSTD_compress2(),
+                              * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
+    ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
+    ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
+
+    /* multi-threading parameters */
+    /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+     * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
+     * In a situation where it's unknown if the linked library supports multi-threading or not,
+     * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
+     */
+    ZSTD_c_nbWorkers=400,    /* Select how many threads will be spawned to compress in parallel.
+                              * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
+                              * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
+                              * while compression is performed in parallel, within worker thread(s).
+                              * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
+                              *  in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
+                              * More workers improve speed, but also increase memory usage.
+                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
+                              * compression is performed inside Caller's thread, and all invocations are blocking */
+    ZSTD_c_jobSize=401,      /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
+                              * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
+                              * 0 means default, which is dynamically determined based on compression parameters.
+                              * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
+                              * The minimum size is automatically and transparently enforced. */
+    ZSTD_c_overlapLog=402,   /* Control the overlap size, as a fraction of window size.
+                              * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
+                              * It helps preserve compression ratio, while each job is compressed in parallel.
+                              * This value is enforced only when nbWorkers >= 1.
+                              * Larger values increase compression ratio, but decrease speed.
+                              * Possible values range from 0 to 9 :
+                              * - 0 means "default" : value will be determined by the library, depending on strategy
+                              * - 1 means "no overlap"
+                              * - 9 means "full overlap", using a full window size.
+                              * Each intermediate rank increases/decreases load size by a factor 2 :
+                              * 9: full window;  8: w/2;  7: w/4;  6: w/8;  5:w/16;  4: w/32;  3:w/64;  2:w/128;  1:no overlap;  0:default
+                              * default value varies between 6 and 9, depending on strategy */
+
+    /* note : additional experimental parameters are also available
+     * within the experimental section of the API.
+     * At the time of this writing, they include :
+     * ZSTD_c_rsyncable
+     * ZSTD_c_format
+     * ZSTD_c_forceMaxWindow
+     * ZSTD_c_forceAttachDict
+     * ZSTD_c_literalCompressionMode
+     * ZSTD_c_srcSizeHint
+     * ZSTD_c_enableDedicatedDictSearch
+     * ZSTD_c_stableInBuffer
+     * ZSTD_c_stableOutBuffer
+     * ZSTD_c_blockDelimiters
+     * ZSTD_c_validateSequences
+     * ZSTD_c_blockSplitterLevel
+     * ZSTD_c_splitAfterSequences
+     * ZSTD_c_useRowMatchFinder
+     * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableSeqProducerFallback
+     * ZSTD_c_maxBlockSize
+     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+     * note : never ever use experimentalParam? names directly;
+     *        also, the enums values themselves are unstable and can still change.
+     */
+     ZSTD_c_experimentalParam1=500,
+     ZSTD_c_experimentalParam2=10,
+     ZSTD_c_experimentalParam3=1000,
+     ZSTD_c_experimentalParam4=1001,
+     ZSTD_c_experimentalParam5=1002,
+     /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */
+     ZSTD_c_experimentalParam7=1004,
+     ZSTD_c_experimentalParam8=1005,
+     ZSTD_c_experimentalParam9=1006,
+     ZSTD_c_experimentalParam10=1007,
+     ZSTD_c_experimentalParam11=1008,
+     ZSTD_c_experimentalParam12=1009,
+     ZSTD_c_experimentalParam13=1010,
+     ZSTD_c_experimentalParam14=1011,
+     ZSTD_c_experimentalParam15=1012,
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015,
+     ZSTD_c_experimentalParam19=1016,
+     ZSTD_c_experimentalParam20=1017
+} ZSTD_cParameter;
+
+typedef struct {
+    size_t error;
+    int lowerBound;
+    int upperBound;
+} ZSTD_bounds;
+
+/*! ZSTD_cParam_getBounds() :
+ *  All parameters must belong to an interval with lower and upper bounds,
+ *  otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ *         - an error status field, which must be tested using ZSTD_isError()
+ *         - lower and upper bounds, both inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
+
+/*! ZSTD_CCtx_setParameter() :
+ *  Set one compression parameter, selected by enum ZSTD_cParameter.
+ *  All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
+ *  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ *  Setting a parameter is generally only possible during frame initialization (before starting compression).
+ *  Exception : when using multi-threading mode (nbWorkers >= 1),
+ *              the following parameters can be updated _during_ compression (within same frame):
+ *              => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
+ *              new parameters will be active for next job only (after a flush()).
+ * @return : an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtx_setPledgedSrcSize() :
+ *  Total input data size to be compressed as a single frame.
+ *  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+ *  This value will also be controlled at end of frame, and trigger an error if not respected.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
+ *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ *           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
+ *  Note 2 : pledgedSrcSize is only valid once, for the next frame.
+ *           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
+ *  Note 3 : Whenever all input data is provided and consumed in a single round,
+ *           for example with ZSTD_compress2(),
+ *           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
+ *           this value is automatically overridden by srcSize instead.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
+typedef enum {
+    ZSTD_reset_session_only = 1,
+    ZSTD_reset_parameters = 2,
+    ZSTD_reset_session_and_parameters = 3
+} ZSTD_ResetDirective;
+
+/*! ZSTD_CCtx_reset() :
+ *  There are 2 different things that can be reset, independently or jointly :
+ *  - The session : will stop compressing current frame, and make CCtx ready to start a new one.
+ *                  Useful after an error, or to interrupt any ongoing compression.
+ *                  Any internal data not yet flushed is cancelled.
+ *                  Compression parameters and dictionary remain unchanged.
+ *                  They will be used to compress next frame.
+ *                  Resetting session never fails.
+ *  - The parameters : changes all parameters back to "default".
+ *                  This also removes any reference to any dictionary or external sequence producer.
+ *                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
+ *                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
+ *  - Both : similar to resetting the session, followed by resetting parameters.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
+
+/*! ZSTD_compress2() :
+ *  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+ *  (note that this entry point doesn't even expose a compression level parameter).
+ *  ZSTD_compress2() always starts a new frame.
+ *  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ *  - The function is always blocking, returns when compression is completed.
+ *  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
+ *        enough space to successfully compress the data, though it is possible it fails for other reasons.
+ * @return : compressed size written into `dst` (<= `dstCapacity),
+ *           or an error code if it fails (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
+                                   void* dst, size_t dstCapacity,
+                             const void* src, size_t srcSize);
+
+
+/***********************************************
+*  Advanced decompression API (Requires v1.4.0+)
+************************************************/
+
+/* The advanced API pushes parameters one by one into an existing DCtx context.
+ * Parameters are sticky, and remain valid for all following frames
+ * using the same DCtx context.
+ * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
+ * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
+ *        Therefore, no new decompression function is necessary.
+ */
+
+typedef enum {
+
+    ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
+                              * the streaming API will refuse to allocate memory buffer
+                              * in order to protect the host from unreasonable memory requirements.
+                              * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+                              * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
+                              * Special: value 0 means "use default maximum windowLog". */
+
+    /* note : additional experimental parameters are also available
+     * within the experimental section of the API.
+     * At the time of this writing, they include :
+     * ZSTD_d_format
+     * ZSTD_d_stableOutBuffer
+     * ZSTD_d_forceIgnoreChecksum
+     * ZSTD_d_refMultipleDDicts
+     * ZSTD_d_disableHuffmanAssembly
+     * ZSTD_d_maxBlockSize
+     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+     * note : never ever use experimentalParam? names directly
+     */
+     ZSTD_d_experimentalParam1=1000,
+     ZSTD_d_experimentalParam2=1001,
+     ZSTD_d_experimentalParam3=1002,
+     ZSTD_d_experimentalParam4=1003,
+     ZSTD_d_experimentalParam5=1004,
+     ZSTD_d_experimentalParam6=1005
+
+} ZSTD_dParameter;
+
+/*! ZSTD_dParam_getBounds() :
+ *  All parameters must belong to an interval with lower and upper bounds,
+ *  otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ *         - an error status field, which must be tested using ZSTD_isError()
+ *         - both lower and upper bounds, inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
+
+/*! ZSTD_DCtx_setParameter() :
+ *  Set one compression parameter, selected by enum ZSTD_dParameter.
+ *  All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+ *  Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ *  Setting a parameter is only possible during frame initialization (before starting decompression).
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
+
+/*! ZSTD_DCtx_reset() :
+ *  Return a DCtx to clean state.
+ *  Session and parameters can be reset jointly or separately.
+ *  Parameters can only be reset when no active frame is being decompressed.
+ * @return : 0, or an error code, which can be tested with ZSTD_isError()
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
+
+
+/****************************
+*  Streaming
+****************************/
+
+typedef struct ZSTD_inBuffer_s {
+  const void* src;    /**< start of input buffer */
+  size_t size;        /**< size of input buffer */
+  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+  void*  dst;         /**< start of output buffer */
+  size_t size;        /**< size of output buffer */
+  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+
+
+/*-***********************************************************************
+*  Streaming compression - HowTo
+*
+*  A ZSTD_CStream object is required to track streaming operation.
+*  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+*  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+*  It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+*
+*  For parallel execution, use one separate ZSTD_CStream per thread.
+*
+*  note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+*
+*  Parameters are sticky : when starting a new compression on the same context,
+*  it will reuse the same sticky parameters as previous compression session.
+*  When in doubt, it's recommended to fully initialize the context before usage.
+*  Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
+*  ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
+*  set more specific parameters, the pledged source size, or load a dictionary.
+*
+*  Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
+*  consume input stream. The function will automatically update both `pos`
+*  fields within `input` and `output`.
+*  Note that the function may not consume the entire input, for example, because
+*  the output buffer is already full, in which case `input.pos < input.size`.
+*  The caller must check if input has been entirely consumed.
+*  If not, the caller must make some room to receive more compressed data,
+*  and then present again remaining input data.
+*  note: ZSTD_e_continue is guaranteed to make some forward progress when called,
+*        but doesn't guarantee maximal forward progress. This is especially relevant
+*        when compressing with multiple threads. The call won't block if it can
+*        consume some input, but if it can't it will wait for some, but not all,
+*        output to be flushed.
+* @return : provides a minimum amount of data remaining to be flushed from internal buffers
+*           or an error code, which can be tested using ZSTD_isError().
+*
+*  At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
+*  using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
+*  Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
+*  In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
+*  You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
+*  operation.
+*  note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
+*        block until the flush is complete or the output buffer is full.
+*  @return : 0 if internal buffers are entirely flushed,
+*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+*  Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
+*  It will perform a flush and write frame epilogue.
+*  The epilogue is required for decoders to consider a frame completed.
+*  flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
+*  You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
+*  start a new frame.
+*  note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
+*        block until the flush is complete or the output buffer is full.
+*  @return : 0 if frame fully completed and fully flushed,
+*            >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************/
+
+typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
+                                 /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
+/*===== ZSTD_CStream management functions =====*/
+ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
+ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);  /* accept NULL pointer */
+
+/*===== Streaming compression functions =====*/
+typedef enum {
+    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+    ZSTD_e_flush=1,    /* flush any data provided so far,
+                        * it creates (at least) one new block, that can be decoded immediately on reception;
+                        * frame will continue: any future data can still reference previously compressed data, improving compression.
+                        * note : multithreaded compression will block to flush as much output as possible. */
+    ZSTD_e_end=2       /* flush any remaining data _and_ close current frame.
+                        * note that frame is only closed after compressed data is fully flushed (return value == 0).
+                        * After that point, any additional data starts a new frame.
+                        * note : each frame is independent (does not reference any content from previous frame).
+                        : note : multithreaded compression will block to flush as much output as possible. */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream2() : Requires v1.4.0+
+ *  Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ *  - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+ *  - output->pos must be <= dstCapacity, input->pos must be <= srcSize
+ *  - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ *  - endOp must be a valid directive
+ *  - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ *  - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
+ *                                                  and then immediately returns, just indicating that there is some data remaining to be flushed.
+ *                                                  The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ *  - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+ *  - @return provides a minimum amount of data remaining to be flushed from internal buffers
+ *            or an error code, which can be tested using ZSTD_isError().
+ *            if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ *            This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+ *            For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+ *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ *            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+ *            Before starting a new compression job, or changing compression parameters,
+ *            it is required to fully flush internal buffers.
+ *  - note: if an operation ends with an error, it may leave @cctx in an undefined state.
+ *          Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state.
+ *          In order to be re-employed after an error, a state must be reset,
+ *          which can be done explicitly (ZSTD_CCtx_reset()),
+ *          or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx())
+ */
+ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+                                         ZSTD_outBuffer* output,
+                                         ZSTD_inBuffer* input,
+                                         ZSTD_EndDirective endOp);
+
+
+/* These buffer sizes are softly recommended.
+ * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
+ * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
+ * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
+ *
+ * However, note that these recommendations are from the perspective of a C caller program.
+ * If the streaming interface is invoked from some other language,
+ * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
+ * a major performance rule is to reduce crossing such interface to an absolute minimum.
+ * It's not rare that performance ends being spent more into the interface, rather than compression itself.
+ * In which cases, prefer using large buffers, as large as practical,
+ * for both input and output, to reduce the nb of roundtrips.
+ */
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
+
+
+/* *****************************************************************************
+ * This following is a legacy streaming API, available since v1.0+ .
+ * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
+ * It is redundant, but remains fully supported.
+ ******************************************************************************/
+
+/*!
+ * Equivalent to:
+ *
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ * to compress with a dictionary.
+ */
+ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+/*!
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
+ */
+ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
+ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
+ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+
+/*-***************************************************************************
+*  Streaming decompression - HowTo
+*
+*  A ZSTD_DStream object is required to track streaming operations.
+*  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+*  ZSTD_DStream objects can be re-employed multiple times.
+*
+*  Use ZSTD_initDStream() to start a new decompression operation.
+* @return : recommended first input size
+*  Alternatively, use advanced API to set specific properties.
+*
+*  Use ZSTD_decompressStream() repetitively to consume your input.
+*  The function will update both `pos` fields.
+*  If `input.pos < input.size`, some input has not been consumed.
+*  It's up to the caller to present again remaining data.
+*
+*  The function tries to flush all data decoded immediately, respecting output buffer size.
+*  If `output.pos < output.size`, decoder has flushed everything it could.
+*
+*  However, when `output.pos == output.size`, it's more difficult to know.
+*  If @return > 0, the frame is not complete, meaning
+*  either there is still some data left to flush within internal buffers,
+*  or there is more input to read to complete the frame (or both).
+*  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+*  Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+* @return : 0 when a frame is completely decoded and fully flushed,
+*        or an error code, which can be tested using ZSTD_isError(),
+*        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
+*                                the return value is a suggested next input size (just a hint for better latency)
+*                                that will never request more than the remaining content of the compressed frame.
+* *******************************************************************************/
+
+typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+                                 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
+/*===== ZSTD_DStream management functions =====*/
+ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
+ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer */
+
+/*===== Streaming decompression functions =====*/
+
+/*! ZSTD_initDStream() :
+ * Initialize/reset DStream state for new decompression operation.
+ * Call before new decompression operation using same DStream.
+ *
+ * Note : This function is redundant with the advanced API and equivalent to:
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_refDDict(zds, NULL);
+ */
+ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+
+/*! ZSTD_decompressStream() :
+ * Streaming decompression function.
+ * Call repetitively to consume full input updating it as necessary.
+ * Function will update both input and output `pos` fields exposing current state via these fields:
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
+ *   on the next call.
+ * - `output.pos < output.size`, decoder flushed internal output buffer.
+ * - `output.pos == output.size`, unflushed data potentially present in the internal buffers,
+ *   check ZSTD_decompressStream() @return value,
+ *   if > 0, invoke it again to flush remaining data to output.
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * @return : 0 when a frame is completely decoded and fully flushed,
+ *           or an error code, which can be tested using ZSTD_isError(),
+ *           or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ *
+ * Note: when an operation returns with an error code, the @zds state may be left in undefined state.
+ *       It's UB to invoke `ZSTD_decompressStream()` on such a state.
+ *       In order to re-use such a state, it must be first reset,
+ *       which can be done explicitly (`ZSTD_DCtx_reset()`),
+ *       or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`)
+ */
+ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+
+
+/**************************
+*  Simple dictionary API
+***************************/
+/*! ZSTD_compress_usingDict() :
+ *  Compression at an explicit compression level using a Dictionary.
+ *  A dictionary can be any arbitrary data segment (also called a prefix),
+ *  or a buffer with specified information (see zdict.h).
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *         It's intended for a dictionary used only once.
+ *  Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+                                           void* dst, size_t dstCapacity,
+                                     const void* src, size_t srcSize,
+                                     const void* dict,size_t dictSize,
+                                           int compressionLevel);
+
+/*! ZSTD_decompress_usingDict() :
+ *  Decompression using a known Dictionary.
+ *  Dictionary must be identical to the one used during compression.
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *         It's intended for a dictionary used only once.
+ *  Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+                                             void* dst, size_t dstCapacity,
+                                       const void* src, size_t srcSize,
+                                       const void* dict,size_t dictSize);
+
+
+/***********************************
+ *  Bulk processing dictionary API
+ **********************************/
+typedef struct ZSTD_CDict_s ZSTD_CDict;
+
+/*! ZSTD_createCDict() :
+ *  When compressing multiple messages or blocks using the same dictionary,
+ *  it's recommended to digest the dictionary only once, since it's a costly operation.
+ *  ZSTD_createCDict() will create a state from digesting a dictionary.
+ *  The resulting state can be used for future compression operations with very limited startup cost.
+ *  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ *  Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+ *  Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+ *      in which case the only thing that it transports is the @compressionLevel.
+ *      This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+ *      expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+                                         int compressionLevel);
+
+/*! ZSTD_freeCDict() :
+ *  Function frees memory allocated by ZSTD_createCDict().
+ *  If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
+
+/*! ZSTD_compress_usingCDict() :
+ *  Compression using a digested Dictionary.
+ *  Recommended when same dictionary is used multiple times.
+ *  Note : compression level is _decided at dictionary creation time_,
+ *     and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
+ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+                                            void* dst, size_t dstCapacity,
+                                      const void* src, size_t srcSize,
+                                      const ZSTD_CDict* cdict);
+
+
+typedef struct ZSTD_DDict_s ZSTD_DDict;
+
+/*! ZSTD_createDDict() :
+ *  Create a digested dictionary, ready to start decompression operation without startup delay.
+ *  dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_freeDDict() :
+ *  Function frees memory allocated with ZSTD_createDDict()
+ *  If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
+
+/*! ZSTD_decompress_usingDDict() :
+ *  Decompression using a digested Dictionary.
+ *  Recommended when same dictionary is used multiple times. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+                                              void* dst, size_t dstCapacity,
+                                        const void* src, size_t srcSize,
+                                        const ZSTD_DDict* ddict);
+
+
+/********************************
+ *  Dictionary helper functions
+ *******************************/
+
+/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
+ *  Provides the dictID stored within dictionary.
+ *  if @return == 0, the dictionary is not conformant with Zstandard specification.
+ *  It can still be loaded, but as a content-only dictionary. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
+ *  Provides the dictID of the dictionary loaded into `cdict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
+
+/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
+ *  Provides the dictID of the dictionary loaded into `ddict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
+ *  Provides the dictID required to decompressed the frame stored within `src`.
+ *  If @return == 0, the dictID could not be decoded.
+ *  This could for one of the following reasons :
+ *  - The frame does not require a dictionary to be decoded (most common case).
+ *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
+ *    Note : this use case also happens when using a non-conformant dictionary.
+ *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ *  - This is not a Zstandard frame.
+ *  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
+
+/*******************************************************************************
+ * Advanced dictionary and prefix API (Requires v1.4.0+)
+ *
+ * This API allows dictionaries to be used with ZSTD_compress2(),
+ * ZSTD_compressStream2(), and ZSTD_decompressDCtx().
+ * Dictionaries are sticky, they remain valid when same context is reused,
+ * they only reset when the context is reset
+ * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
+ * In contrast, Prefixes are single-use.
+ ******************************************************************************/
+
+
+/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
+ *  Create an internal CDict from `dict` buffer.
+ *  Decompression will have to use same dictionary.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
+ *           meaning "return to no-dictionary mode".
+ *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ *           until parameters are reset, a new dictionary is loaded, or the dictionary
+ *           is explicitly invalidated by loading a NULL dictionary.
+ *  Note 2 : Loading a dictionary involves building tables.
+ *           It's also a CPU consuming operation, with non-negligible impact on latency.
+ *           Tables are dependent on compression parameters, and for this reason,
+ *           compression parameters can no longer be changed after loading a dictionary.
+ *  Note 3 :`dict` content will be copied internally.
+ *           Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
+ *           In such a case, dictionary buffer must outlive its users.
+ *  Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
+ *           to precisely select how dictionary content must be interpreted.
+ *  Note 5 : This method does not benefit from LDM (long distance mode).
+ *           If you want to employ LDM on some large dictionary content,
+ *           prefer employing ZSTD_CCtx_refPrefix() described below.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
+ *  Reference a prepared dictionary, to be used for all future compressed frames.
+ *  Note that compression parameters are enforced from within CDict,
+ *  and supersede any compression parameter previously set within CCtx.
+ *  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+ *  The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
+ *  The dictionary will remain valid for future compressed frames using same CCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Referencing a NULL CDict means "return to no-dictionary mode".
+ *  Note 1 : Currently, only one dictionary can be managed.
+ *           Referencing a new dictionary effectively "discards" any previous one.
+ *  Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
+ *  Reference a prefix (single-usage dictionary) for next compressed frame.
+ *  A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
+ *  Decompression will need same prefix to properly regenerate data.
+ *  Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ *  but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ *  This method is compatible with LDM (long distance mode).
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
+ *  Note 1 : Prefix buffer is referenced. It **must** outlive compression.
+ *           Its content must remain unmodified during compression.
+ *  Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ *           ensure that the window size is large enough to contain the entire source.
+ *           See ZSTD_c_windowLog.
+ *  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ *           It's a CPU consuming operation, with non-negligible impact on latency.
+ *           If there is a need to use the same prefix multiple times, consider loadDictionary instead.
+ *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
+ *           Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
+                                 const void* prefix, size_t prefixSize);
+
+/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
+ *  Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ *  The dictionary remains valid for all future frames, until explicitly invalidated, or
+ *  a new dictionary is loaded.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ *            meaning "return to no-dictionary mode".
+ *  Note 1 : Loading a dictionary involves building tables,
+ *           which has a non-negligible impact on CPU usage and latency.
+ *           It's recommended to "load once, use many times", to amortize the cost
+ *  Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
+ *           Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
+ *  Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
+ *           how dictionary content is loaded and interpreted.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
+ *  Reference a prepared dictionary, to be used to decompress next frames.
+ *  The dictionary remains active for decompression of future frames using same DCtx.
+ *
+ *  If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
+ *  will store the DDict references in a table, and the DDict used for decompression
+ *  will be determined at decompression time, as per the dict ID in the frame.
+ *  The memory for the table is allocated on the first call to refDDict, and can be
+ *  freed with ZSTD_freeDCtx().
+ *
+ *  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ *  will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special: referencing a NULL DDict means "return to no-dictionary mode".
+ *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
+ *  Reference a prefix (single-usage dictionary) to decompress next frame.
+ *  This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ *  and must use the same prefix as the one used during compression.
+ *  Prefix is **only used once**. Reference is discarded at end of frame.
+ *  End of frame is reached when ZSTD_decompressStream() returns 0.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ *  Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
+ *           Prefix buffer must remain unmodified up to the end of frame,
+ *           reached when ZSTD_decompressStream() returns 0.
+ *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
+ *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
+ *  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ *           A full dictionary is more costly, as it requires building tables.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
+                                 const void* prefix, size_t prefixSize);
+
+/* ===   Memory management   === */
+
+/*! ZSTD_sizeof_*() : Requires v1.4.0+
+ *  These functions give the _current_ memory usage of selected object.
+ *  Note that object memory usage can evolve (increase or decrease) over time. */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+#endif  /* ZSTD_H_235446 */
+
+
+/* **************************************************************************************
+ *   ADVANCED AND EXPERIMENTAL FUNCTIONS
+ ****************************************************************************************
+ * The definitions in the following section are considered experimental.
+ * They are provided for advanced scenarios.
+ * They should never be used with a dynamic library, as prototypes may change in the future.
+ * Use them only in association with static linking.
+ * ***************************************************************************************/
+
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
+#include <limits.h>   /* INT_MAX */
+
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZSTDLIB_STATIC_API
+#  if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#    define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
+#  elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#    define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
+#  else
+#    define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
+#  endif
+#endif
+
+/****************************************************************************************
+ *   experimental API (static linking only)
+ ****************************************************************************************
+ * The following symbols and constants
+ * are not planned to join "stable API" status in the near future.
+ * They can still change in future versions.
+ * Some of them are planned to remain in the static_only section indefinitely.
+ * Some of them might be removed in the future (especially when redundant with existing stable functions)
+ * ***************************************************************************************/
+
+#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1)   /* minimum input size required to query frame header size */
+#define ZSTD_FRAMEHEADERSIZE_MIN(format)    ((format) == ZSTD_f_zstd1 ? 6 : 2)
+#define ZSTD_FRAMEHEADERSIZE_MAX   18   /* can be useful for static allocation */
+#define ZSTD_SKIPPABLEHEADERSIZE    8
+
+/* compression parameter bounds */
+#define ZSTD_WINDOWLOG_MAX_32    30
+#define ZSTD_WINDOWLOG_MAX_64    31
+#define ZSTD_WINDOWLOG_MAX     ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MIN       10
+#define ZSTD_HASHLOG_MAX       ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
+#define ZSTD_HASHLOG_MIN          6
+#define ZSTD_CHAINLOG_MAX_32     29
+#define ZSTD_CHAINLOG_MAX_64     30
+#define ZSTD_CHAINLOG_MAX      ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
+#define ZSTD_CHAINLOG_MIN        ZSTD_HASHLOG_MIN
+#define ZSTD_SEARCHLOG_MAX      (ZSTD_WINDOWLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN        1
+#define ZSTD_MINMATCH_MAX         7   /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_MINMATCH_MIN         3   /* only for ZSTD_btopt+, faster strategies are limited to 4 */
+#define ZSTD_TARGETLENGTH_MAX    ZSTD_BLOCKSIZE_MAX
+#define ZSTD_TARGETLENGTH_MIN     0   /* note : comparing this constant to an unsigned results in a tautological test */
+#define ZSTD_STRATEGY_MIN        ZSTD_fast
+#define ZSTD_STRATEGY_MAX        ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
+
+
+#define ZSTD_OVERLAPLOG_MIN       0
+#define ZSTD_OVERLAPLOG_MAX       9
+
+#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27   /* by default, the streaming decoder will refuse any frame
+                                           * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
+                                           * to preserve host's memory from unreasonable requirements.
+                                           * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
+                                           * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
+
+
+/* LDM parameter bounds */
+#define ZSTD_LDM_HASHLOG_MIN      ZSTD_HASHLOG_MIN
+#define ZSTD_LDM_HASHLOG_MAX      ZSTD_HASHLOG_MAX
+#define ZSTD_LDM_MINMATCH_MIN        4
+#define ZSTD_LDM_MINMATCH_MAX     4096
+#define ZSTD_LDM_BUCKETSIZELOG_MIN   1
+#define ZSTD_LDM_BUCKETSIZELOG_MAX   8
+#define ZSTD_LDM_HASHRATELOG_MIN     0
+#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
+
+/* Advanced parameter bounds */
+#define ZSTD_TARGETCBLOCKSIZE_MIN   1340 /* suitable to fit into an ethernet / wifi / 4G transport frame */
+#define ZSTD_TARGETCBLOCKSIZE_MAX   ZSTD_BLOCKSIZE_MAX
+#define ZSTD_SRCSIZEHINT_MIN        0
+#define ZSTD_SRCSIZEHINT_MAX        INT_MAX
+
+
+/* ---  Advanced types  --- */
+
+typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
+
+typedef struct {
+    unsigned int offset;      /* The offset of the match. (NOT the same as the offset code)
+                               * If offset == 0 and matchLength == 0, this sequence represents the last
+                               * literals in the block of litLength size.
+                               */
+
+    unsigned int litLength;   /* Literal length of the sequence. */
+    unsigned int matchLength; /* Match length of the sequence. */
+
+                              /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
+                               * In this case, we will treat the sequence as a marker for a block boundary.
+                               */
+
+    unsigned int rep;         /* Represents which repeat offset is represented by the field 'offset'.
+                               * Ranges from [0, 3].
+                               *
+                               * Repeat offsets are essentially previous offsets from previous sequences sorted in
+                               * recency order. For more detail, see doc/zstd_compression_format.md
+                               *
+                               * If rep == 0, then 'offset' does not contain a repeat offset.
+                               * If rep > 0:
+                               *  If litLength != 0:
+                               *      rep == 1 --> offset == repeat_offset_1
+                               *      rep == 2 --> offset == repeat_offset_2
+                               *      rep == 3 --> offset == repeat_offset_3
+                               *  If litLength == 0:
+                               *      rep == 1 --> offset == repeat_offset_2
+                               *      rep == 2 --> offset == repeat_offset_3
+                               *      rep == 3 --> offset == repeat_offset_1 - 1
+                               *
+                               * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
+                               * 'rep', but repeat offsets do not necessarily need to be calculated from an external
+                               * sequence provider's perspective. For example, ZSTD_compressSequences() does not
+                               * use this 'rep' field at all (as of now).
+                               */
+} ZSTD_Sequence;
+
+typedef struct {
+    unsigned windowLog;       /**< largest match distance : larger == more compression, more memory needed during decompression */
+    unsigned chainLog;        /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+    unsigned hashLog;         /**< dispatch table : larger == faster, more memory */
+    unsigned searchLog;       /**< nb of searches : larger == more compression, slower */
+    unsigned minMatch;        /**< match length searched : larger == faster decompression, sometimes less compression */
+    unsigned targetLength;    /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+    ZSTD_strategy strategy;   /**< see ZSTD_strategy definition above */
+} ZSTD_compressionParameters;
+
+typedef struct {
+    int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+    int checksumFlag;    /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
+    int noDictIDFlag;    /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
+} ZSTD_frameParameters;
+
+typedef struct {
+    ZSTD_compressionParameters cParams;
+    ZSTD_frameParameters fParams;
+} ZSTD_parameters;
+
+typedef enum {
+    ZSTD_dct_auto = 0,       /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
+    ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
+    ZSTD_dct_fullDict = 2    /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
+} ZSTD_dictContentType_e;
+
+typedef enum {
+    ZSTD_dlm_byCopy = 0,  /**< Copy dictionary content internally */
+    ZSTD_dlm_byRef = 1    /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
+} ZSTD_dictLoadMethod_e;
+
+typedef enum {
+    ZSTD_f_zstd1 = 0,           /* zstd frame format, specified in zstd_compression_format.md (default) */
+    ZSTD_f_zstd1_magicless = 1  /* Variant of zstd frame format, without initial 4-bytes magic number.
+                                 * Useful to save 4 bytes per generated frame.
+                                 * Decoder cannot recognise automatically this format, requiring this instruction. */
+} ZSTD_format_e;
+
+typedef enum {
+    /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
+    ZSTD_d_validateChecksum = 0,
+    ZSTD_d_ignoreChecksum = 1
+} ZSTD_forceIgnoreChecksum_e;
+
+typedef enum {
+    /* Note: this enum controls ZSTD_d_refMultipleDDicts */
+    ZSTD_rmd_refSingleDDict = 0,
+    ZSTD_rmd_refMultipleDDicts = 1
+} ZSTD_refMultipleDDicts_e;
+
+typedef enum {
+    /* Note: this enum and the behavior it controls are effectively internal
+     * implementation details of the compressor. They are expected to continue
+     * to evolve and should be considered only in the context of extremely
+     * advanced performance tuning.
+     *
+     * Zstd currently supports the use of a CDict in three ways:
+     *
+     * - The contents of the CDict can be copied into the working context. This
+     *   means that the compression can search both the dictionary and input
+     *   while operating on a single set of internal tables. This makes
+     *   the compression faster per-byte of input. However, the initial copy of
+     *   the CDict's tables incurs a fixed cost at the beginning of the
+     *   compression. For small compressions (< 8 KB), that copy can dominate
+     *   the cost of the compression.
+     *
+     * - The CDict's tables can be used in-place. In this model, compression is
+     *   slower per input byte, because the compressor has to search two sets of
+     *   tables. However, this model incurs no start-up cost (as long as the
+     *   working context's tables can be reused). For small inputs, this can be
+     *   faster than copying the CDict's tables.
+     *
+     * - The CDict's tables are not used at all, and instead we use the working
+     *   context alone to reload the dictionary and use params based on the source
+     *   size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+     *   This method is effective when the dictionary sizes are very small relative
+     *   to the input size, and the input size is fairly large to begin with.
+     *
+     * Zstd has a simple internal heuristic that selects which strategy to use
+     * at the beginning of a compression. However, if experimentation shows that
+     * Zstd is making poor choices, it is possible to override that choice with
+     * this enum.
+     */
+    ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
+    ZSTD_dictForceAttach   = 1, /* Never copy the dictionary. */
+    ZSTD_dictForceCopy     = 2, /* Always copy the dictionary. */
+    ZSTD_dictForceLoad     = 3  /* Always reload the dictionary */
+} ZSTD_dictAttachPref_e;
+
+typedef enum {
+  ZSTD_lcm_auto = 0,          /**< Automatically determine the compression mode based on the compression level.
+                               *   Negative compression levels will be uncompressed, and positive compression
+                               *   levels will be compressed. */
+  ZSTD_lcm_huffman = 1,       /**< Always attempt Huffman compression. Uncompressed literals will still be
+                               *   emitted if Huffman compression is not profitable. */
+  ZSTD_lcm_uncompressed = 2   /**< Always emit uncompressed literals. */
+} ZSTD_literalCompressionMode_e;
+
+typedef enum {
+  /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
+   * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
+   * or ZSTD_ps_disable allow for a force enable/disable the feature.
+   */
+  ZSTD_ps_auto = 0,         /* Let the library automatically determine whether the feature shall be enabled */
+  ZSTD_ps_enable = 1,       /* Force-enable the feature */
+  ZSTD_ps_disable = 2       /* Do not use the feature */
+} ZSTD_paramSwitch_e;
+
+/***************************************
+*  Frame header and size functions
+***************************************/
+
+/*! ZSTD_findDecompressedSize() :
+ *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ *  `srcSize` must be the _exact_ size of this series
+ *       (i.e. there should be a frame boundary at `src + srcSize`)
+ *  @return : - decompressed size of all data in all successive frames
+ *            - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+ *            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ *   note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ *            When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ *            In which case, it's necessary to use streaming mode to decompress data.
+ *   note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ *   note 3 : decompressed size can be very large (64-bits value),
+ *            potentially larger than what local system can handle as a single memory segment.
+ *            In which case, it's necessary to use streaming mode to decompress data.
+ *   note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ *            Always ensure result fits within application's authorized limits.
+ *            Each application can set its own limits.
+ *   note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+ *            read each contained frame header.  This is fast as most of the data is skipped,
+ *            however it does mean that all frame data must be present and valid. */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_decompressBound() :
+ *  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ *  `srcSize` must be the _exact_ size of this series
+ *       (i.e. there should be a frame boundary at `src + srcSize`)
+ *  @return : - upper-bound for the decompressed size of all data in all successive frames
+ *            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ *  note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame.
+ *  note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
+ *            in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
+ *  note 3  : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
+ *              upper-bound = # blocks * min(128 KB, Window_Size)
+ */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
+
+/*! ZSTD_frameHeaderSize() :
+ *  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
+ * @return : size of the Frame Header,
+ *           or an error code (if srcSize is too small) */
+ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+
+typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
+typedef struct {
+    unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
+    unsigned long long windowSize;       /* can be very large, up to <= frameContentSize */
+    unsigned blockSizeMax;
+    ZSTD_frameType_e frameType;          /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
+    unsigned headerSize;
+    unsigned dictID;
+    unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
+} ZSTD_frameHeader;
+
+/*! ZSTD_getFrameHeader() :
+ *  decode Frame Header, or requires larger `srcSize`.
+ * @return : 0, `zfhPtr` is correctly filled,
+ *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ *           or an error code, which can be tested using ZSTD_isError() */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
+/*! ZSTD_getFrameHeader_advanced() :
+ *  same as ZSTD_getFrameHeader(),
+ *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+
+/*! ZSTD_decompressionMargin() :
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ * and the input buffer must be at the end of the output buffer.
+ *
+ *  _______________________ Output Buffer ________________________
+ * |                                                              |
+ * |                                        ____ Input Buffer ____|
+ * |                                       |                      |
+ * v                                       v                      v
+ * |---------------------------------------|-----------|----------|
+ * ^                                                   ^          ^
+ * |___________________ Output_Size ___________________|_ Margin _|
+ *
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ * ZSTD_decompressDCtx().
+ * NOTE: This function supports multi-frame input.
+ *
+ * @param src The compressed frame(s)
+ * @param srcSize The size of the compressed frame(s)
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
+/*! ZSTD_DECOMPRESS_MARGIN() :
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ * the compressed frame, compute it from the original size and the blockSizeLog.
+ * See ZSTD_decompressionMargin() for details.
+ *
+ * WARNING: This macro does not support multi-frame input, the input must be a single
+ * zstd frame. If you need that support use the function, or implement it yourself.
+ *
+ * @param originalSize The original uncompressed size of the data.
+ * @param blockSize    The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+ *                     Unless you explicitly set the windowLog smaller than
+ *                     ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ */
+#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)(                                              \
+        ZSTD_FRAMEHEADERSIZE_MAX                                                              /* Frame header */ + \
+        4                                                                                         /* checksum */ + \
+        ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+        (blockSize)                                                                    /* One block of margin */   \
+    ))
+
+typedef enum {
+  ZSTD_sf_noBlockDelimiters = 0,         /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
+  ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
+} ZSTD_sequenceFormat_e;
+
+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ *  @return : upper-bound for the number of sequences that can be generated
+ *            from a buffer of srcSize bytes
+ *
+ *  note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
+/*! ZSTD_generateSequences() :
+ * WARNING: This function is meant for debugging and informational purposes ONLY!
+ * Its implementation is flawed, and it will be deleted in a future version.
+ * It is not guaranteed to succeed, as there are several cases where it will give
+ * up and fail. You should NOT use this function in production code.
+ *
+ * This function is deprecated, and will be removed in a future version.
+ *
+ * Generate sequences using ZSTD_compress2(), given a source buffer.
+ *
+ * @param zc The compression context to be used for ZSTD_compress2(). Set any
+ *           compression parameters you need on this context.
+ * @param outSeqs The output sequences buffer of size @p outSeqsSize
+ * @param outSeqsSize The size of the output sequences buffer.
+ *                    ZSTD_sequenceBound(srcSize) is an upper bound on the number
+ *                    of sequences that can be generated.
+ * @param src The source buffer to generate sequences from of size @p srcSize.
+ * @param srcSize The size of the source buffer.
+ *
+ * Each block will end with a dummy sequence
+ * with offset == 0, matchLength == 0, and litLength == length of last literals.
+ * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
+ * simply acts as a block delimiter.
+ *
+ * @returns The number of sequences generated, necessarily less than
+ *          ZSTD_sequenceBound(srcSize), or an error code that can be checked
+ *          with ZSTD_isError().
+ */
+ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
+ZSTDLIB_STATIC_API size_t
+ZSTD_generateSequences(ZSTD_CCtx* zc,
+                       ZSTD_Sequence* outSeqs, size_t outSeqsSize,
+                       const void* src, size_t srcSize);
+
+/*! ZSTD_mergeBlockDelimiters() :
+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
+ * by merging them into the literals of the next sequence.
+ *
+ * As such, the final generated result has no explicit representation of block boundaries,
+ * and the final last literals segment is not represented in the sequences.
+ *
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
+ * @return : number of sequences left after merging
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
+
+/*! ZSTD_compressSequences() :
+ * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
+ * @src contains the entire input (not just the literals).
+ * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
+ * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
+ * The entire source is compressed into a single frame.
+ *
+ * The compression behavior changes based on cctx params. In particular:
+ *    If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ *    no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
+ *    the block size derived from the cctx, and sequences may be split. This is the default setting.
+ *
+ *    If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ *    block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
+ *
+ *    If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
+ *    behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
+ *    specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
+ *
+ *    In addition to the two adjustable experimental params, there are other important cctx params.
+ *    - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
+ *    - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
+ *    - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
+ *      is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
+ *
+ * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
+ * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
+ *         and cannot emit an RLE block that disagrees with the repcode history
+ * @return : final compressed size, or a ZSTD error code.
+ */
+ZSTDLIB_STATIC_API size_t
+ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
+                        const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+                        const void* src, size_t srcSize);
+
+
+/*! ZSTD_writeSkippableFrame() :
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, if the source size is not representable
+ * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+                                            const void* src, size_t srcSize, unsigned magicVariant);
+
+/*! ZSTD_readSkippableFrame() :
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
+ * in the magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
+                                            const void* src, size_t srcSize);
+
+/*! ZSTD_isSkippableFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ */
+ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
+
+
+
+/***************************************
+*  Memory management
+***************************************/
+
+/*! ZSTD_estimate*() :
+ *  These functions make it possible to estimate memory usage
+ *  of a future {D,C}Ctx, before its creation.
+ *  This is useful in combination with ZSTD_initStatic(),
+ *  which makes it possible to employ a static buffer for ZSTD_CCtx* state.
+ *
+ *  ZSTD_estimateCCtxSize() will provide a memory budget large enough
+ *  to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2()
+ *  associated with any compression level up to max specified one.
+ *  The estimate will assume the input may be arbitrarily large,
+ *  which is the worst case.
+ *
+ *  Note that the size estimation is specific for one-shot compression,
+ *  it is not valid for streaming (see ZSTD_estimateCStreamSize*())
+ *  nor other potential ways of using a ZSTD_CCtx* state.
+ *
+ *  When srcSize can be bound by a known and rather "small" value,
+ *  this knowledge can be used to provide a tighter budget estimation
+ *  because the ZSTD_CCtx* state will need less memory for small inputs.
+ *  This tighter estimation can be provided by employing more advanced functions
+ *  ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
+ *  and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
+ *  Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
+ *
+ *  Note : only single-threaded compression is supported.
+ *  ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
+
+/*! ZSTD_estimateCStreamSize() :
+ *  ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression
+ *  using any compression level up to the max specified one.
+ *  It will also consider src size to be arbitrarily "large", which is a worst case scenario.
+ *  If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
+ *  ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  Note : CStream size estimation is only correct for single-threaded compression.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ *  Size estimates assume that no external sequence producer is registered.
+ *
+ *  ZSTD_DStream memory budget depends on frame's window Size.
+ *  This information can be passed manually, using ZSTD_estimateDStreamSize,
+ *  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
+ *  Any frame requesting a window size larger than max specified one will be rejected.
+ *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+ *         an internal ?Dict will be created, which additional size is not estimated here.
+ *         In this case, get total size by adding ZSTD_estimate?DictSize
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
+
+/*! ZSTD_estimate?DictSize() :
+ *  ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+ *  ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
+ *  Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
+
+/*! ZSTD_initStatic*() :
+ *  Initialize an object using a pre-allocated fixed-size buffer.
+ *  workspace: The memory area to emplace the object into.
+ *             Provided pointer *must be 8-bytes aligned*.
+ *             Buffer must outlive object.
+ *  workspaceSize: Use ZSTD_estimate*Size() to determine
+ *                 how large workspace must be to support target scenario.
+ * @return : pointer to object (same address as workspace, just different type),
+ *           or NULL if error (size too small, incorrect alignment, etc.)
+ *  Note : zstd will never resize nor malloc() when using a static buffer.
+ *         If the object requires more memory than available,
+ *         zstd will just error out (typically ZSTD_error_memory_allocation).
+ *  Note 2 : there is no corresponding "free" function.
+ *           Since workspace is allocated externally, it must be freed externally too.
+ *  Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
+ *           into its associated cParams.
+ *  Limitation 1 : currently not compatible with internal dictionary creation, triggered by
+ *                 ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
+ *  Limitation 2 : static cctx currently not compatible with multi-threading.
+ *  Limitation 3 : static dctx is incompatible with legacy support.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx*    ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticCCtx() */
+
+ZSTDLIB_STATIC_API ZSTD_DCtx*    ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticDCtx() */
+
+ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
+                                        void* workspace, size_t workspaceSize,
+                                        const void* dict, size_t dictSize,
+                                        ZSTD_dictLoadMethod_e dictLoadMethod,
+                                        ZSTD_dictContentType_e dictContentType,
+                                        ZSTD_compressionParameters cParams);
+
+ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
+                                        void* workspace, size_t workspaceSize,
+                                        const void* dict, size_t dictSize,
+                                        ZSTD_dictLoadMethod_e dictLoadMethod,
+                                        ZSTD_dictContentType_e dictContentType);
+
+
+/*! Custom memory allocation :
+ *  These prototypes make it possible to pass your own allocation/free functions.
+ *  ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
+ *  All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
+ */
+typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+
+#if defined(__clang__) && __clang_major__ >= 5
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
+#endif
+ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< this constant defers to stdlib's functions */
+#if defined(__clang__) && __clang_major__ >= 5
+#pragma clang diagnostic pop
+#endif
+
+ZSTDLIB_STATIC_API ZSTD_CCtx*    ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DCtx*    ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+                                                  ZSTD_dictLoadMethod_e dictLoadMethod,
+                                                  ZSTD_dictContentType_e dictContentType,
+                                                  ZSTD_compressionParameters cParams,
+                                                  ZSTD_customMem customMem);
+
+/*! Thread pool :
+ *  These prototypes make it possible to share a thread pool among multiple compression contexts.
+ *  This can limit resources for applications with multiple threads where each one uses
+ *  a threaded compression mode (via ZSTD_c_nbWorkers parameter).
+ *  ZSTD_createThreadPool creates a new thread pool with a given number of threads.
+ *  Note that the lifetime of such pool must exist while being used.
+ *  ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
+ *  to use an internal thread pool).
+ *  ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
+ */
+typedef struct POOL_ctx_s ZSTD_threadPool;
+ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
+ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);  /* accept NULL pointer */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
+
+
+/*
+ * This API is temporary and is expected to change or disappear in the future!
+ */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
+    const void* dict, size_t dictSize,
+    ZSTD_dictLoadMethod_e dictLoadMethod,
+    ZSTD_dictContentType_e dictContentType,
+    const ZSTD_CCtx_params* cctxParams,
+    ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
+    const void* dict, size_t dictSize,
+    ZSTD_dictLoadMethod_e dictLoadMethod,
+    ZSTD_dictContentType_e dictContentType,
+    ZSTD_customMem customMem);
+
+
+/***************************************
+*  Advanced compression functions
+***************************************/
+
+/*! ZSTD_createCDict_byReference() :
+ *  Create a digested dictionary for compression
+ *  Dictionary content is just referenced, not duplicated.
+ *  As a consequence, `dictBuffer` **must** outlive CDict,
+ *  and its content must remain unmodified throughout the lifetime of CDict.
+ *  note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
+/*! ZSTD_getCParams() :
+ * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+ * `estimatedSrcSize` value is optional, select 0 if not known */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_getParams() :
+ *  same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+ *  All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
+ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_checkCParams() :
+ *  Ensure param values remain within authorized range.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+
+/*! ZSTD_adjustCParams() :
+ *  optimize params for a given `srcSize` and `dictSize`.
+ * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
+ * `dictSize` must be `0` when there is no dictionary.
+ *  cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
+ *  This function never fails (wide contract) */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+
+/*! ZSTD_CCtx_setCParams() :
+ *  Set all parameters provided within @p cparams into the working @p cctx.
+ *  Note : if modifying parameters during compression (MT mode only),
+ *         note that changes to the .windowLog parameter will be ignored.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ *         On failure, no parameters are updated.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
+/*! ZSTD_CCtx_setFParams() :
+ *  Set all parameters provided within @p fparams into the working @p cctx.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
+
+/*! ZSTD_CCtx_setParams() :
+ *  Set all parameters provided within @p params into the working @p cctx.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
+
+/*! ZSTD_compress_advanced() :
+ *  Note : this function is now DEPRECATED.
+ *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
+ *  This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+                              void* dst, size_t dstCapacity,
+                        const void* src, size_t srcSize,
+                        const void* dict,size_t dictSize,
+                              ZSTD_parameters params);
+
+/*! ZSTD_compress_usingCDict_advanced() :
+ *  Note : this function is now DEPRECATED.
+ *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
+ *  This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+                                              void* dst, size_t dstCapacity,
+                                        const void* src, size_t srcSize,
+                                        const ZSTD_CDict* cdict,
+                                              ZSTD_frameParameters fParams);
+
+
+/*! ZSTD_CCtx_loadDictionary_byReference() :
+ *  Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
+ *  It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_loadDictionary_advanced() :
+ *  Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
+ *  how to load the dictionary (by copy ? by reference ?)
+ *  and how to interpret it (automatic ? force raw mode ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_CCtx_refPrefix_advanced() :
+ *  Same as ZSTD_CCtx_refPrefix(), but gives finer control over
+ *  how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/* ===   experimental parameters   === */
+/* these parameters can be used with ZSTD_setParameter()
+ * they are not guaranteed to remain supported in the future */
+
+ /* Enables rsyncable mode,
+  * which makes compressed files more rsync friendly
+  * by adding periodic synchronization points to the compressed data.
+  * The target average block size is ZSTD_c_jobSize / 2.
+  * It's possible to modify the job size to increase or decrease
+  * the granularity of the synchronization point.
+  * Once the jobSize is smaller than the window size,
+  * it will result in compression ratio degradation.
+  * NOTE 1: rsyncable mode only works when multithreading is enabled.
+  * NOTE 2: rsyncable performs poorly in combination with long range mode,
+  * since it will decrease the effectiveness of synchronization points,
+  * though mileage may vary.
+  * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
+  * If the selected compression level is already running significantly slower,
+  * the overall speed won't be significantly impacted.
+  */
+ #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
+
+/* Select a compression format.
+ * The value must be of type ZSTD_format_e.
+ * See ZSTD_format_e enum definition for details */
+#define ZSTD_c_format ZSTD_c_experimentalParam2
+
+/* Force back-reference distances to remain < windowSize,
+ * even when referencing into Dictionary content (default:0) */
+#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
+
+/* Controls whether the contents of a CDict
+ * are used in place, or copied into the working context.
+ * Accepts values from the ZSTD_dictAttachPref_e enum.
+ * See the comments on that enum for an explanation of the feature. */
+#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
+
+/* Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never compress literals.
+ * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
+ * may still be emitted if huffman is not beneficial to use.)
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * literals compression based on the compression parameters - specifically,
+ * negative compression levels do not use literal compression.
+ */
+#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
+
+/* User's best guess of source size.
+ * Hint is not valid when srcSizeHint == 0.
+ * There is no guarantee that hint is close to actual source size,
+ * but compression ratio may regress significantly if guess considerably underestimates */
+#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
+
+/* Controls whether the new and experimental "dedicated dictionary search
+ * structure" can be used. This feature is still rough around the edges, be
+ * prepared for surprising behavior!
+ *
+ * How to use it:
+ *
+ * When using a CDict, whether to use this feature or not is controlled at
+ * CDict creation, and it must be set in a CCtxParams set passed into that
+ * construction (via ZSTD_createCDict_advanced2()). A compression will then
+ * use the feature or not based on how the CDict was constructed; the value of
+ * this param, set in the CCtx, will have no effect.
+ *
+ * However, when a dictionary buffer is passed into a CCtx, such as via
+ * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
+ * whether the CDict that is created internally can use the feature or not.
+ *
+ * What it does:
+ *
+ * Normally, the internal data structures of the CDict are analogous to what
+ * would be stored in a CCtx after compressing the contents of a dictionary.
+ * To an approximation, a compression using a dictionary can then use those
+ * data structures to simply continue what is effectively a streaming
+ * compression where the simulated compression of the dictionary left off.
+ * Which is to say, the search structures in the CDict are normally the same
+ * format as in the CCtx.
+ *
+ * It is possible to do better, since the CDict is not like a CCtx: the search
+ * structures are written once during CDict creation, and then are only read
+ * after that, while the search structures in the CCtx are both read and
+ * written as the compression goes along. This means we can choose a search
+ * structure for the dictionary that is read-optimized.
+ *
+ * This feature enables the use of that different structure.
+ *
+ * Note that some of the members of the ZSTD_compressionParameters struct have
+ * different semantics and constraints in the dedicated search structure. It is
+ * highly recommended that you simply set a compression level in the CCtxParams
+ * you pass into the CDict creation call, and avoid messing with the cParams
+ * directly.
+ *
+ * Effects:
+ *
+ * This will only have any effect when the selected ZSTD_strategy
+ * implementation supports this feature. Currently, that's limited to
+ * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
+ *
+ * Note that this means that the CDict tables can no longer be copied into the
+ * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
+ * usable. The dictionary can only be attached or reloaded.
+ *
+ * In general, you should expect compression to be faster--sometimes very much
+ * so--and CDict creation to be slightly slower. Eventually, we will probably
+ * make this mode the default.
+ */
+#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
+
+/* ZSTD_c_stableInBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the compressor that input data presented with ZSTD_inBuffer
+ * will ALWAYS be the same between calls.
+ * Technically, the @src pointer must never be changed,
+ * and the @pos field can only be updated by zstd.
+ * However, it's possible to increase the @size field,
+ * allowing scenarios where more data can be appended after compressions starts.
+ * These conditions are checked by the compressor,
+ * and compression will fail if they are not respected.
+ * Also, data in the ZSTD_inBuffer within the range [src, src + pos)
+ * MUST not be modified during compression or it will result in data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an input window buffer,
+ * because the user guarantees it can reference the ZSTD_inBuffer until
+ * the frame is complete. But, it will still allocate an output buffer
+ * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
+ * avoid the memcpy() from the input buffer to the input window buffer.
+ *
+ * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, compression WILL fail if conditions are not respected.
+ *
+ * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
+ * not be modified during compression or it will result in data corruption.
+ * This is because zstd needs to reference data in the ZSTD_inBuffer to find
+ * matches. Normally zstd maintains its own window buffer for this purpose,
+ * but passing this flag tells zstd to rely on user provided buffer instead.
+ */
+#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
+
+/* ZSTD_c_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells he compressor that the ZSTD_outBuffer will not be resized between
+ * calls. Specifically: (out.size - out.pos) will never grow. This gives the
+ * compressor the freedom to say: If the compressed data doesn't fit in the
+ * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
+ * always decompress directly into the output buffer, instead of decompressing
+ * into an internal buffer and copying to the output buffer.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer. It will still allocate the
+ * input window buffer (see ZSTD_c_stableInBuffer).
+ *
+ * Zstd will check that (out.size - out.pos) never grows and return an error
+ * if it does. While not strictly necessary, this should prevent surprises.
+ */
+#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
+
+/* ZSTD_c_blockDelimiters
+ * Default is 0 == ZSTD_sf_noBlockDelimiters.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ *
+ * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
+ * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
+ * See the definition of ZSTD_Sequence for more specifics.
+ */
+#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
+
+/* ZSTD_c_validateSequences
+ * Default is 0 == disabled. Set to 1 to enable sequence validation.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
+ * during function execution.
+ *
+ * Without validation, providing a sequence that does not conform to the zstd spec will cause
+ * undefined behavior, and may produce a corrupted block.
+ *
+ * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
+ * specifics regarding offset/matchlength requirements) then the function will bail out and
+ * return an error.
+ *
+ */
+#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
+
+/* ZSTD_c_blockSplitterLevel
+ * note: this parameter only influences the first splitter stage,
+ *       which is active before producing the sequences.
+ *       ZSTD_c_splitAfterSequences controls the next splitter stage,
+ *       which is active after sequence production.
+ *       Note that both can be combined.
+ * Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included.
+ * 0 means "auto", which will select a value depending on current ZSTD_c_strategy.
+ * 1 means no splitting.
+ * Then, values from 2 to 6 are sorted in increasing cpu load order.
+ *
+ * Note that currently the first block is never split,
+ * to ensure expansion guarantees in presence of incompressible data.
+ */
+#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6
+#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20
+
+/* ZSTD_c_splitAfterSequences
+ * This is a stronger splitter algorithm,
+ * based on actual sequences previously produced by the selected parser.
+ * It's also slower, and as a consequence, mostly used for high compression levels.
+ * While the post-splitter does overlap with the pre-splitter,
+ * both can nonetheless be combined,
+ * notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX,
+ * resulting in higher compression ratio than just one of them.
+ *
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use block splitter.
+ * Set to ZSTD_ps_enable to always use block splitter.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * block splitting based on the compression parameters.
+ */
+#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13
+
+/* ZSTD_c_useRowMatchFinder
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use row-based matchfinder.
+ * Set to ZSTD_ps_enable to force usage of row-based matchfinder.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * the row-based matchfinder based on support for SIMD instructions and the window log.
+ * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
+ */
+#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
+
+/* ZSTD_c_deterministicRefPrefix
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Zstd produces different results for prefix compression when the prefix is
+ * directly adjacent to the data about to be compressed vs. when it isn't.
+ * This is because zstd detects that the two buffers are contiguous and it can
+ * use a more efficient match finding algorithm. However, this produces different
+ * results than when the two buffers are non-contiguous. This flag forces zstd
+ * to always load the prefix in non-contiguous mode, even if it happens to be
+ * adjacent to the data, to guarantee determinism.
+ *
+ * If you really care about determinism when using a dictionary or prefix,
+ * like when doing delta compression, you should select this option. It comes
+ * at a speed penalty of about ~2.5% if the dictionary and data happened to be
+ * contiguous, and is free if they weren't contiguous. We don't expect that
+ * intentionally making the dictionary and data contiguous will be worth the
+ * cost to memcpy() the data.
+ */
+#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
+
+/* ZSTD_c_prefetchCDictTables
+ * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
+ *
+ * In some situations, zstd uses CDict tables in-place rather than copying them
+ * into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
+ * In such situations, compression speed is seriously impacted when CDict tables are
+ * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
+ * when they are used in-place.
+ *
+ * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
+ * For sufficiently large inputs, zstd will by default memcpy() CDict tables
+ * into the working context, so there is no need to prefetch. This parameter is
+ * targeted at a middle range of input sizes, where a prefetch is cheap enough to be
+ * useful but memcpy() is too expensive. The exact range of input sizes where this
+ * makes sense is best determined by careful experimentation.
+ *
+ * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
+ * but in the future zstd may conditionally enable this feature via an auto-detection
+ * heuristic for cold CDicts.
+ * Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
+ */
+#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
+
+/* ZSTD_c_enableSeqProducerFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal sequence producer if an
+ * external sequence producer is registered and returns an error code. This fallback
+ * is block-by-block: the internal sequence producer will only be called for blocks
+ * where the external sequence producer returns an error code. Fallback parsing will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full Block-Level Sequence Producer API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
+
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() inaccurate). Only currently meant to be used for testing.
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
+/* ZSTD_c_searchForExternalRepcodes
+ * This parameter affects how zstd parses external sequences, such as sequences
+ * provided through the compressSequences() API or from an external block-level
+ * sequence producer.
+ *
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
+ * external sequences, even if those repcodes are not explicitly indicated in
+ * the "rep" field. Note that this is the only way to exploit repcode matches
+ * while using compressSequences() or an external sequence producer, since zstd
+ * currently ignores the "rep" field of external sequences.
+ *
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
+ * external sequences, regardless of whether the "rep" field has been set. This
+ * reduces sequence compression overhead by about 25% while sacrificing some
+ * compression ratio.
+ *
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
+ * based on compression level.
+ *
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
+ */
+#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
+
+
+/*! ZSTD_CCtx_getParameter() :
+ *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
+ *  and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+
+
+/*! ZSTD_CCtx_params :
+ *  Quick howto :
+ *  - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
+ *  - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
+ *                                     an existing ZSTD_CCtx_params structure.
+ *                                     This is similar to
+ *                                     ZSTD_CCtx_setParameter().
+ *  - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+ *                                    an existing CCtx.
+ *                                    These parameters will be applied to
+ *                                    all subsequent frames.
+ *  - ZSTD_compressStream2() : Do compression using the CCtx.
+ *  - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
+ *
+ *  This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+ *  for static allocation of CCtx for single-threaded compression.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  /* accept NULL pointer */
+
+/*! ZSTD_CCtxParams_reset() :
+ *  Reset params to default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+
+/*! ZSTD_CCtxParams_init() :
+ *  Initializes the compression parameters of cctxParams according to
+ *  compression level. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+
+/*! ZSTD_CCtxParams_init_advanced() :
+ *  Initializes the compression and frame parameters of cctxParams according to
+ *  params. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+
+/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
+ *  Similar to ZSTD_CCtx_setParameter.
+ *  Set one compression parameter, selected by enum ZSTD_cParameter.
+ *  Parameters must be applied to a ZSTD_CCtx using
+ *  ZSTD_CCtx_setParametersUsingCCtxParams().
+ * @result : a code representing success or failure (which can be tested with
+ *           ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtxParams_getParameter() :
+ * Similar to ZSTD_CCtx_getParameter.
+ * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+
+/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
+ *  Apply a set of ZSTD_CCtx_params to the compression context.
+ *  This can be done even after compression is started,
+ *    if nbWorkers==0, this will have no impact until a new compression is started.
+ *    if nbWorkers>=1, new parameters will be picked up at next job,
+ *       with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+        ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
+
+/*! ZSTD_compressStream2_simpleArgs() :
+ *  Same as ZSTD_compressStream2(),
+ *  but using only integral types as arguments.
+ *  This variant might be helpful for binders from dynamic languages
+ *  which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
+                            ZSTD_CCtx* cctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos,
+                            ZSTD_EndDirective endOp);
+
+
+/***************************************
+*  Advanced decompression functions
+***************************************/
+
+/*! ZSTD_isFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier.
+ *  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ *  Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ *  Note 3 : Skippable Frame Identifiers are considered valid. */
+ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
+
+/*! ZSTD_createDDict_byReference() :
+ *  Create a digested dictionary, ready to start decompression operation without startup delay.
+ *  Dictionary content is referenced, and therefore stays in dictBuffer.
+ *  It is important that dictBuffer outlives DDict,
+ *  it must remain read accessible throughout the lifetime of DDict */
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_byReference() :
+ *  Same as ZSTD_DCtx_loadDictionary(),
+ *  but references `dict` content instead of copying it into `dctx`.
+ *  This saves memory if `dict` remains around.,
+ *  However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_advanced() :
+ *  Same as ZSTD_DCtx_loadDictionary(),
+ *  but gives direct control over
+ *  how to load the dictionary (by copy ? by reference ?)
+ *  and how to interpret it (automatic ? force raw mode ? full mode only ?). */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_refPrefix_advanced() :
+ *  Same as ZSTD_DCtx_refPrefix(), but gives finer control over
+ *  how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_setMaxWindowSize() :
+ *  Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+ *  This protects a decoder context from reserving too much memory for itself (potential attack scenario).
+ *  This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ *  By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+
+/*! ZSTD_DCtx_getParameter() :
+ *  Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
+ *  and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
+
+/* ZSTD_d_format
+ * experimental parameter,
+ * allowing selection between ZSTD_format_e input compression formats
+ */
+#define ZSTD_d_format ZSTD_d_experimentalParam1
+/* ZSTD_d_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
+ * between calls, except for the modifications that zstd makes to pos (the
+ * caller must not modify pos). This is checked by the decompressor, and
+ * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
+ * MUST be large enough to fit the entire decompressed frame. This will be
+ * checked when the frame content size is known. The data in the ZSTD_outBuffer
+ * in the range [dst, dst + pos) MUST not be modified during decompression
+ * or you will get data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer, but it will still allocate
+ * an input buffer large enough to fit any compressed block. This will also
+ * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
+ * If you need to avoid the input buffer allocation use the buffer-less
+ * streaming API.
+ *
+ * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, decompression WILL fail if you violate the preconditions.
+ *
+ * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
+ * not be modified during decompression or you will get data corruption. This
+ * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
+ * matches. Normally zstd maintains its own buffer for this purpose, but passing
+ * this flag tells zstd to use the user provided buffer.
+ */
+#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
+
+/* ZSTD_d_forceIgnoreChecksum
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * Tells the decompressor to skip checksum validation during decompression, regardless
+ * of whether checksumming was specified during compression. This offers some
+ * slight performance benefits, and may be useful for debugging.
+ * Param has values of type ZSTD_forceIgnoreChecksum_e
+ */
+#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
+
+/* ZSTD_d_refMultipleDDicts
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * If enabled and dctx is allocated on the heap, then additional memory will be allocated
+ * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
+ * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
+ * store all references. At decompression time, the appropriate dictID is selected
+ * from the set of DDicts based on the dictID in the frame.
+ *
+ * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
+ *
+ * Param has values of byte ZSTD_refMultipleDDicts_e
+ *
+ * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
+ * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
+ * Memory is allocated as per ZSTD_DCtx::customMem.
+ *
+ * Although this function allocates memory for the table, the user is still responsible for
+ * memory management of the underlying ZSTD_DDict* themselves.
+ */
+#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
+
+/* ZSTD_d_disableHuffmanAssembly
+ * Set to 1 to disable the Huffman assembly implementation.
+ * The default value is 0, which allows zstd to use the Huffman assembly
+ * implementation if available.
+ *
+ * This parameter can be used to disable Huffman assembly at runtime.
+ * If you want to disable it at compile time you can define the macro
+ * ZSTD_DISABLE_ASM.
+ */
+#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
+
+/* ZSTD_d_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * Forces the decompressor to reject blocks whose content size is
+ * larger than the configured maxBlockSize. When maxBlockSize is
+ * larger than the windowSize, the windowSize is used instead.
+ * This saves memory on the decoder when you know all blocks are small.
+ *
+ * This option is typically used in conjunction with ZSTD_c_maxBlockSize.
+ *
+ * WARNING: This causes the decoder to reject otherwise valid frames
+ * that have block sizes larger than the configured maxBlockSize.
+ */
+#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6
+
+
+/*! ZSTD_DCtx_setFormat() :
+ *  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
+ *  Instruct the decoder context about what kind of data to decode next.
+ *  This instruction is mandatory to decode data without a fully-formed header,
+ *  such ZSTD_f_zstd1_magicless for example.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
+ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+
+/*! ZSTD_decompressStream_simpleArgs() :
+ *  Same as ZSTD_decompressStream(),
+ *  but using only integral types as arguments.
+ *  This can be helpful for binders from dynamic languages
+ *  which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
+                            ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos);
+
+
+/********************************************************************
+*  Advanced streaming functions
+*  Warning : most of these functions are now redundant with the Advanced API.
+*  Once Advanced API reaches "stable" status,
+*  redundant functions will be deprecated, and then at some point removed.
+********************************************************************/
+
+/*=====   Advanced Streaming compression functions  =====*/
+
+/*! ZSTD_initCStream_srcSize() :
+ * This function is DEPRECATED, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *
+ * pledgedSrcSize must be correct. If it is not known at init time, use
+ * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
+ * "0" also disables frame content size field. It may be enabled in the future.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+                         int compressionLevel,
+                         unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingDict() :
+ * This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * Creates of an internal CDict (incompatible with static CCtx), except if
+ * dict == NULL or dictSize < 8, in which case no dict is used.
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
+ * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+                     const void* dict, size_t dictSize,
+                           int compressionLevel);
+
+/*! ZSTD_initCStream_advanced() :
+ * This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setParams(zcs, params);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
+ * pledgedSrcSize must be correct.
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+                    const void* dict, size_t dictSize,
+                          ZSTD_parameters params,
+                          unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingCDict() :
+ * This function is DEPRECATED, and equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * note : cdict will just be referenced, and must outlive compression session
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+
+/*! ZSTD_initCStream_usingCDict_advanced() :
+ *   This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setFParams(zcs, fParams);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *     ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+                               const ZSTD_CDict* cdict,
+                                     ZSTD_frameParameters fParams,
+                                     unsigned long long pledgedSrcSize);
+
+/*! ZSTD_resetCStream() :
+ * This function is DEPRECATED, and is equivalent to:
+ *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
+ *       ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
+ *       explicitly specified.
+ *
+ *  start a new frame, using same parameters from previous frame.
+ *  This is typically useful to skip dictionary loading stage, since it will reuse it in-place.
+ *  Note that zcs must be init at least once before using ZSTD_resetCStream().
+ *  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
+ *  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
+ *  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ *  but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError())
+ *  This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+
+
+typedef struct {
+    unsigned long long ingested;   /* nb input bytes read and buffered */
+    unsigned long long consumed;   /* nb input bytes actually compressed */
+    unsigned long long produced;   /* nb of compressed bytes generated and buffered */
+    unsigned long long flushed;    /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
+    unsigned currentJobID;         /* MT only : latest started job nb */
+    unsigned nbActiveWorkers;      /* MT only : nb of workers actively compressing at probe time */
+} ZSTD_frameProgression;
+
+/* ZSTD_getFrameProgression() :
+ * tells how much data has been ingested (read from input)
+ * consumed (input actually compressed) and produced (output) for current frame.
+ * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
+ * Aggregates progression inside active worker threads.
+ */
+ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
+
+/*! ZSTD_toFlushNow() :
+ *  Tell how many bytes are ready to be flushed immediately.
+ *  Useful for multithreading scenarios (nbWorkers >= 1).
+ *  Probe the oldest active job, defined as oldest job not yet entirely flushed,
+ *  and check its output buffer.
+ * @return : amount of data stored in oldest job and ready to be flushed immediately.
+ *  if @return == 0, it means either :
+ *  + there is no active job (could be checked with ZSTD_frameProgression()), or
+ *  + oldest job is still actively compressing data,
+ *    but everything it has produced has also been flushed so far,
+ *    therefore flush speed is limited by production speed of oldest job
+ *    irrespective of the speed of concurrent (and newer) jobs.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
+
+
+/*=====   Advanced Streaming decompression functions  =====*/
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
+ *
+ * note: no dictionary will be used if dict == NULL or dictSize < 8
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *     ZSTD_DCtx_refDDict(zds, ddict);
+ *
+ * note : ddict is referenced, it must outlive decompression session
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *
+ * reuse decompression parameters from previous init; saves dictionary loading
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+
+
+/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
+ *
+ * *** OVERVIEW ***
+ * The Block-Level Sequence Producer API allows users to provide their own custom
+ * sequence producer which libzstd invokes to process each block. The produced list
+ * of sequences (literals and matches) is then post-processed by libzstd to produce
+ * valid compressed blocks.
+ *
+ * This block-level offload API is a more granular complement of the existing
+ * frame-level offload API compressSequences() (introduced in v1.5.1). It offers
+ * an easier migration story for applications already integrated with libzstd: the
+ * user application continues to invoke the same compression functions
+ * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
+ * from the specific advantages of the external sequence producer. For example,
+ * the sequence producer could be tuned to take advantage of known characteristics
+ * of the input, to offer better speed / ratio, or could leverage hardware
+ * acceleration not available within libzstd itself.
+ *
+ * See contrib/externalSequenceProducer for an example program employing the
+ * Block-Level Sequence Producer API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_sequenceProducer_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ *   - sequenceProducerState: a pointer to a user-managed state for the sequence
+ *     producer.
+ *
+ *   - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
+ *     outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
+ *     backing outSeqs is managed by the CCtx.
+ *
+ *   - src, srcSize: an input buffer for the sequence producer to parse.
+ *     srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ *   - dict, dictSize: a history buffer, which may be empty, which the sequence
+ *     producer may reference as it parses the src buffer. Currently, zstd will
+ *     always pass dictSize == 0 into external sequence producers, but this will
+ *     change in the future.
+ *
+ *   - compressionLevel: a signed integer representing the zstd compression level
+ *     set by the user for the current operation. The sequence producer may choose
+ *     to use this information to change its compression strategy and speed/ratio
+ *     tradeoff. Note: the compression level does not reflect zstd parameters set
+ *     through the advanced API.
+ *
+ *   - windowSize: a size_t representing the maximum allowed offset for external
+ *     sequences. Note that sequence offsets are sometimes allowed to exceed the
+ *     windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ *     for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ *   - The sum of matchLengths and literalLengths must equal srcSize.
+ *   - All sequences in the parse, except for the final sequence, must have
+ *     matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
+ *     matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
+ *   - All offsets must respect the windowSize parameter as specified in
+ *     doc/zstd_compression_format.md.
+ *   - If the final sequence has matchLength == 0, it must also have offset == 0.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal sequence producer or fail the compression operation. The user can
+ * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
+ * cParam. Fallback compression will follow any other cParam settings, such as
+ * compression level, the same as in a normal compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
+ * function by calling
+ *         ZSTD_registerSequenceProducer(cctx,
+ *                                       sequenceProducerState,
+ *                                       sequenceProducer)
+ * This setting will persist until the next parameter reset of the CCtx.
+ *
+ * The sequenceProducerState must be initialized by the user before calling
+ * ZSTD_registerSequenceProducer(). The user is responsible for destroying the
+ * sequenceProducerState.
+ *
+ * *** LIMITATIONS ***
+ * This API is compatible with all zstd compression APIs which respect advanced parameters.
+ * However, there are three limitations:
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
+ * external sequence producer.
+ *   - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
+ *     cases (see its documentation for details). Users must explicitly set
+ *     ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
+ *     sequence producer is registered.
+ *   - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ *     whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ *     check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
+ *     Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
+ *
+ * Second, history buffers are not currently supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external sequence producer (for now). This has two implications:
+ *   - Dictionaries are not currently supported. Compression will *not* fail if the user
+ *     references a dictionary, but the dictionary won't have any effect.
+ *   - Stream history is not currently supported. All advanced compression APIs, including
+ *     streaming APIs, work with external sequence producers, but each block is treated as
+ *     an independent chunk without history from previous blocks.
+ *
+ * Third, multi-threading within a single compression is not currently supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
+
+typedef size_t (*ZSTD_sequenceProducer_F) (
+  void* sequenceProducerState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+/*! ZSTD_registerSequenceProducer() :
+ * Instruct zstd to use a block-level external sequence producer function.
+ *
+ * The sequenceProducerState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * Sequence producer registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on compression
+ * APIs which respect advanced parameters, such as compress2() and compressStream2().
+ * Older compression APIs such as compressCCtx(), which predate the introduction of
+ * "advanced parameters", will ignore any external sequence producer setting.
+ *
+ * The sequence producer can be "cleared" by registering a NULL function pointer. This
+ * removes all limitations described above in the "LIMITATIONS" section of the API docs.
+ *
+ * The user is strongly encouraged to read the full API documentation (above) before
+ * calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerSequenceProducer(
+  ZSTD_CCtx* cctx,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F sequenceProducer
+);
+
+/*! ZSTD_CCtxParams_registerSequenceProducer() :
+ * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params.
+ * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(),
+ * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx().
+ *
+ * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx()
+ * is required, then this function is for you. Otherwise, you probably don't need it.
+ *
+ * See tests/zstreamtest.c for example usage. */
+ZSTDLIB_STATIC_API void
+ZSTD_CCtxParams_registerSequenceProducer(
+  ZSTD_CCtx_params* params,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F sequenceProducer
+);
+
+
+/*********************************************************************
+*  Buffer-less and synchronous inner streaming functions (DEPRECATED)
+*
+*  This API is deprecated, and will be removed in a future version.
+*  It allows streaming (de)compression with user allocated buffers.
+*  However, it is hard to use, and not as well tested as the rest of
+*  our API.
+*
+*  Please use the normal streaming API instead: ZSTD_compressStream2,
+*  and ZSTD_decompressStream.
+*  If there is functionality that you need, but it doesn't provide,
+*  please open an issue on our GitHub.
+********************************************************************* */
+
+/**
+  Buffer-less streaming compression (synchronous mode)
+
+  A ZSTD_CCtx object is required to track streaming operations.
+  Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
+  ZSTD_CCtx object can be reused multiple times within successive compression operations.
+
+  Start by initializing a context.
+  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
+
+  Then, consume your input using ZSTD_compressContinue().
+  There are some important considerations to keep in mind when using this advanced function :
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
+  - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
+  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+    Worst case evaluation is provided by ZSTD_compressBound().
+    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+  - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+    It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+  - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+    In which case, it will "discard" the relevant memory section from its history.
+
+  Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
+  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+  Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
+
+  `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again.
+*/
+
+/*=====   Buffer-less streaming compression functions  =====*/
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
+
+ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+/**
+  Buffer-less streaming decompression (synchronous mode)
+
+  A ZSTD_DCtx object is required to track streaming operations.
+  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+  A ZSTD_DCtx object can be reused multiple times.
+
+  First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
+  Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+  Data fragment must be large enough to ensure successful decoding.
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
+           errorCode, which can be tested using ZSTD_isError().
+
+  It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+  such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
+  Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
+  As a consequence, check that values remain within valid application range.
+  For example, do not allocate memory blindly, check that `windowSize` is within expectation.
+  Each application can set its own limits, depending on local restrictions.
+  For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
+
+  ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
+  ZSTD_decompressContinue() is very sensitive to contiguity,
+  if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
+  or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
+  There are multiple ways to guarantee this condition.
+
+  The most memory efficient way is to use a round buffer of sufficient size.
+  Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
+  which can return an error code if required value is too large for current system (in 32-bits mode).
+  In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
+  up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
+  which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
+  At which point, decoding can resume from the beginning of the buffer.
+  Note that already decoded data stored in the buffer should be flushed before being overwritten.
+
+  There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
+
+  Finally, if you control the compression process, you can also ignore all buffer size rules,
+  as long as the encoder and decoder progress in "lock-step",
+  aka use exactly the same buffer sizes, break contiguity at the same place, etc.
+
+  Once buffers are setup, start decompression, with ZSTD_decompressBegin().
+  If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
+
+  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
+  ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+  ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
+
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
+  It can also be an error code, which can be tested with ZSTD_isError().
+
+  A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+  Context can then be reset to start a new decompression.
+
+  Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
+  This information is not required to properly decode a frame.
+
+  == Special case : skippable frames ==
+
+  Skippable frames allow integration of user-defined data into a flow of concatenated frames.
+  Skippable frames will be ignored (skipped) by decompressor.
+  The format of skippable frames is as follows :
+  a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
+  b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
+  c) Frame Content - any content (User Data) of length equal to Frame Size
+  For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
+  For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
+*/
+
+/*=====   Buffer-less streaming decompression functions  =====*/
+
+ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* misc */
+ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+
+
+
+
+/* ========================================= */
+/**       Block level API (DEPRECATED)       */
+/* ========================================= */
+
+/*!
+
+    This API is deprecated in favor of the regular compression API.
+    You can get the frame header down to 2 bytes by setting:
+      - ZSTD_c_format = ZSTD_f_zstd1_magicless
+      - ZSTD_c_contentSizeFlag = 0
+      - ZSTD_c_checksumFlag = 0
+      - ZSTD_c_dictIDFlag = 0
+
+    This API is not as well tested as our normal API, so we recommend not using it.
+    We will be removing it in a future version. If the normal API doesn't provide
+    the functionality you need, please open a GitHub issue.
+
+    Block functions produce and decode raw zstd blocks, without frame metadata.
+    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+    But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
+
+    A few rules to respect :
+    - Compressing and decompressing require a context structure
+      + Use ZSTD_createCCtx() and ZSTD_createDCtx()
+    - It is necessary to init context before starting
+      + compression : any ZSTD_compressBegin*() variant, including with dictionary
+      + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+      + If input is larger than a block size, it's necessary to split input data into multiple blocks
+      + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
+        Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
+    - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
+      ===> In which case, nothing is produced into `dst` !
+      + User __must__ test for such outcome and deal directly with uncompressed data
+      + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
+        Doing so would mess up with statistics history, leading to potential data corruption.
+      + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
+      + In case of multiple successive blocks, should some of them be uncompressed,
+        decoder must be informed of their existence in order to follow proper history.
+        Use ZSTD_insertBlock() for such a case.
+*/
+
+/*=====   Raw zstd block functions  =====*/
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
+
+#endif   /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
+
+#if defined (__cplusplus)
+}
+#endif
+/**** ended inlining ../zstd.h ****/
+#define FSE_STATIC_LINKING_ONLY
+/**** skipping file: fse.h ****/
+/**** skipping file: huf.h ****/
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
+#endif
+/**** start inlining xxhash.h ****/
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Header File
+ * Copyright (c) Yann Collet - Meta Platforms, Inc
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* Local adaptations for Zstandard */
+
+#ifndef XXH_NO_XXH3
+# define XXH_NO_XXH3
+#endif
+
+#ifndef XXH_NAMESPACE
+# define XXH_NAMESPACE ZSTD_
+#endif
+
+/*!
+ * @mainpage xxHash
+ *
+ * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed
+ * limits.
+ *
+ * It is proposed in four flavors, in three families:
+ * 1. @ref XXH32_family
+ *   - Classic 32-bit hash function. Simple, compact, and runs on almost all
+ *     32-bit and 64-bit systems.
+ * 2. @ref XXH64_family
+ *   - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most
+ *     64-bit systems (but _not_ 32-bit systems).
+ * 3. @ref XXH3_family
+ *   - Modern 64-bit and 128-bit hash function family which features improved
+ *     strength and performance across the board, especially on smaller data.
+ *     It benefits greatly from SIMD and 64-bit without requiring it.
+ *
+ * Benchmarks
+ * ---
+ * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04.
+ * The open source benchmark program is compiled with clang v10.0 using -O3 flag.
+ *
+ * | Hash Name            | ISA ext | Width | Large Data Speed | Small Data Velocity |
+ * | -------------------- | ------- | ----: | ---------------: | ------------------: |
+ * | XXH3_64bits()        | @b AVX2 |    64 |        59.4 GB/s |               133.1 |
+ * | MeowHash             | AES-NI  |   128 |        58.2 GB/s |                52.5 |
+ * | XXH3_128bits()       | @b AVX2 |   128 |        57.9 GB/s |               118.1 |
+ * | CLHash               | PCLMUL  |    64 |        37.1 GB/s |                58.1 |
+ * | XXH3_64bits()        | @b SSE2 |    64 |        31.5 GB/s |               133.1 |
+ * | XXH3_128bits()       | @b SSE2 |   128 |        29.6 GB/s |               118.1 |
+ * | RAM sequential read  |         |   N/A |        28.0 GB/s |                 N/A |
+ * | ahash                | AES-NI  |    64 |        22.5 GB/s |               107.2 |
+ * | City64               |         |    64 |        22.0 GB/s |                76.6 |
+ * | T1ha2                |         |    64 |        22.0 GB/s |                99.0 |
+ * | City128              |         |   128 |        21.7 GB/s |                57.7 |
+ * | FarmHash             | AES-NI  |    64 |        21.3 GB/s |                71.9 |
+ * | XXH64()              |         |    64 |        19.4 GB/s |                71.0 |
+ * | SpookyHash           |         |    64 |        19.3 GB/s |                53.2 |
+ * | Mum                  |         |    64 |        18.0 GB/s |                67.0 |
+ * | CRC32C               | SSE4.2  |    32 |        13.0 GB/s |                57.9 |
+ * | XXH32()              |         |    32 |         9.7 GB/s |                71.9 |
+ * | City32               |         |    32 |         9.1 GB/s |                66.0 |
+ * | Blake3*              | @b AVX2 |   256 |         4.4 GB/s |                 8.1 |
+ * | Murmur3              |         |    32 |         3.9 GB/s |                56.1 |
+ * | SipHash*             |         |    64 |         3.0 GB/s |                43.2 |
+ * | Blake3*              | @b SSE2 |   256 |         2.4 GB/s |                 8.1 |
+ * | HighwayHash          |         |    64 |         1.4 GB/s |                 6.0 |
+ * | FNV64                |         |    64 |         1.2 GB/s |                62.7 |
+ * | Blake2*              |         |   256 |         1.1 GB/s |                 5.1 |
+ * | SHA1*                |         |   160 |         0.8 GB/s |                 5.6 |
+ * | MD5*                 |         |   128 |         0.6 GB/s |                 7.8 |
+ * @note
+ *   - Hashes which require a specific ISA extension are noted. SSE2 is also noted,
+ *     even though it is mandatory on x64.
+ *   - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic
+ *     by modern standards.
+ *   - Small data velocity is a rough average of algorithm's efficiency for small
+ *     data. For more accurate information, see the wiki.
+ *   - More benchmarks and strength tests are found on the wiki:
+ *         https://github.com/Cyan4973/xxHash/wiki
+ *
+ * Usage
+ * ------
+ * All xxHash variants use a similar API. Changing the algorithm is a trivial
+ * substitution.
+ *
+ * @pre
+ *    For functions which take an input and length parameter, the following
+ *    requirements are assumed:
+ *    - The range from [`input`, `input + length`) is valid, readable memory.
+ *      - The only exception is if the `length` is `0`, `input` may be `NULL`.
+ *    - For C++, the objects must have the *TriviallyCopyable* property, as the
+ *      functions access bytes directly as if it was an array of `unsigned char`.
+ *
+ * @anchor single_shot_example
+ * **Single Shot**
+ *
+ * These functions are stateless functions which hash a contiguous block of memory,
+ * immediately returning the result. They are the easiest and usually the fastest
+ * option.
+ *
+ * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits()
+ *
+ * @code{.c}
+ *   #include <string.h>
+ *   #include "xxhash.h"
+ *
+ *   // Example for a function which hashes a null terminated string with XXH32().
+ *   XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed)
+ *   {
+ *       // NULL pointers are only valid if the length is zero
+ *       size_t length = (string == NULL) ? 0 : strlen(string);
+ *       return XXH32(string, length, seed);
+ *   }
+ * @endcode
+ *
+ *
+ * @anchor streaming_example
+ * **Streaming**
+ *
+ * These groups of functions allow incremental hashing of unknown size, even
+ * more than what would fit in a size_t.
+ *
+ * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset()
+ *
+ * @code{.c}
+ *   #include <stdio.h>
+ *   #include <assert.h>
+ *   #include "xxhash.h"
+ *   // Example for a function which hashes a FILE incrementally with XXH3_64bits().
+ *   XXH64_hash_t hashFile(FILE* f)
+ *   {
+ *       // Allocate a state struct. Do not just use malloc() or new.
+ *       XXH3_state_t* state = XXH3_createState();
+ *       assert(state != NULL && "Out of memory!");
+ *       // Reset the state to start a new hashing session.
+ *       XXH3_64bits_reset(state);
+ *       char buffer[4096];
+ *       size_t count;
+ *       // Read the file in chunks
+ *       while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) {
+ *           // Run update() as many times as necessary to process the data
+ *           XXH3_64bits_update(state, buffer, count);
+ *       }
+ *       // Retrieve the finalized hash. This will not change the state.
+ *       XXH64_hash_t result = XXH3_64bits_digest(state);
+ *       // Free the state. Do not use free().
+ *       XXH3_freeState(state);
+ *       return result;
+ *   }
+ * @endcode
+ *
+ * Streaming functions generate the xxHash value from an incremental input.
+ * This method is slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * An XXH state must first be allocated using `XXH*_createState()`.
+ *
+ * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
+ *
+ * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
+ *
+ * The function returns an error code, with 0 meaning OK, and any other value
+ * meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a
+ * digest, and generate new hash values later on by invoking `XXH*_digest()`.
+ *
+ * When done, release the state using `XXH*_freeState()`.
+ *
+ *
+ * @anchor canonical_representation_example
+ * **Canonical Representation**
+ *
+ * The default return values from XXH functions are unsigned 32, 64 and 128 bit
+ * integers.
+ * This the simplest and fastest format for further post-processing.
+ *
+ * However, this leaves open the question of what is the order on the byte level,
+ * since little and big endian conventions will store the same number differently.
+ *
+ * The canonical representation settles this issue by mandating big-endian
+ * convention, the same convention as human-readable numbers (large digits first).
+ *
+ * When writing hash values to storage, sending them over a network, or printing
+ * them, it's highly recommended to use the canonical representation to ensure
+ * portability across a wider range of systems, present and future.
+ *
+ * The following functions allow transformation of hash values to and from
+ * canonical format.
+ *
+ * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(),
+ * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(),
+ * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(),
+ *
+ * @code{.c}
+ *   #include <stdio.h>
+ *   #include "xxhash.h"
+ *
+ *   // Example for a function which prints XXH32_hash_t in human readable format
+ *   void printXxh32(XXH32_hash_t hash)
+ *   {
+ *       XXH32_canonical_t cano;
+ *       XXH32_canonicalFromHash(&cano, hash);
+ *       size_t i;
+ *       for(i = 0; i < sizeof(cano.digest); ++i) {
+ *           printf("%02x", cano.digest[i]);
+ *       }
+ *       printf("\n");
+ *   }
+ *
+ *   // Example for a function which converts XXH32_canonical_t to XXH32_hash_t
+ *   XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano)
+ *   {
+ *       XXH32_hash_t hash = XXH32_hashFromCanonical(&cano);
+ *       return hash;
+ *   }
+ * @endcode
+ *
+ *
+ * @file xxhash.h
+ * xxHash prototypes and implementation
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* ****************************
+ *  INLINE mode
+ ******************************/
+/*!
+ * @defgroup public Public API
+ * Contains details on the public xxHash functions.
+ * @{
+ */
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Gives access to internal state declaration, required for static allocation.
+ *
+ * Incompatible with dynamic linking, due to risks of ABI changes.
+ *
+ * Usage:
+ * @code{.c}
+ *     #define XXH_STATIC_LINKING_ONLY
+ *     #include "xxhash.h"
+ * @endcode
+ */
+#  define XXH_STATIC_LINKING_ONLY
+/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */
+
+/*!
+ * @brief Gives access to internal definitions.
+ *
+ * Usage:
+ * @code{.c}
+ *     #define XXH_STATIC_LINKING_ONLY
+ *     #define XXH_IMPLEMENTATION
+ *     #include "xxhash.h"
+ * @endcode
+ */
+#  define XXH_IMPLEMENTATION
+/* Do not undef XXH_IMPLEMENTATION for Doxygen */
+
+/*!
+ * @brief Exposes the implementation and marks all functions as `inline`.
+ *
+ * Use these build macros to inline xxhash into the target unit.
+ * Inlining improves performance on small inputs, especially when the length is
+ * expressed as a compile-time constant:
+ *
+ *  https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
+ *
+ * It also keeps xxHash symbols private to the unit, so they are not exported.
+ *
+ * Usage:
+ * @code{.c}
+ *     #define XXH_INLINE_ALL
+ *     #include "xxhash.h"
+ * @endcode
+ * Do not compile and link xxhash.o as a separate object, as it is not useful.
+ */
+#  define XXH_INLINE_ALL
+#  undef XXH_INLINE_ALL
+/*!
+ * @brief Exposes the implementation without marking functions as inline.
+ */
+#  define XXH_PRIVATE_API
+#  undef XXH_PRIVATE_API
+/*!
+ * @brief Emulate a namespace by transparently prefixing all symbols.
+ *
+ * If you want to include _and expose_ xxHash functions from within your own
+ * library, but also want to avoid symbol collisions with other libraries which
+ * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix
+ * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE
+ * (therefore, avoid empty or numeric values).
+ *
+ * Note that no change is required within the calling program as long as it
+ * includes `xxhash.h`: Regular symbol names will be automatically translated
+ * by this header.
+ */
+#  define XXH_NAMESPACE /* YOUR NAME HERE */
+#  undef XXH_NAMESPACE
+#endif
+
+#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
+    && !defined(XXH_INLINE_ALL_31684351384)
+   /* this section should be traversed only once */
+#  define XXH_INLINE_ALL_31684351384
+   /* give access to the advanced API, required to compile implementations */
+#  undef XXH_STATIC_LINKING_ONLY   /* avoid macro redef */
+#  define XXH_STATIC_LINKING_ONLY
+   /* make all functions private */
+#  undef XXH_PUBLIC_API
+#  if defined(__GNUC__)
+#    define XXH_PUBLIC_API static __inline __attribute__((unused))
+#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#    define XXH_PUBLIC_API static inline
+#  elif defined(_MSC_VER)
+#    define XXH_PUBLIC_API static __inline
+#  else
+     /* note: this version may generate warnings for unused static functions */
+#    define XXH_PUBLIC_API static
+#  endif
+
+   /*
+    * This part deals with the special case where a unit wants to inline xxHash,
+    * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
+    * such as part of some previously included *.h header file.
+    * Without further action, the new include would just be ignored,
+    * and functions would effectively _not_ be inlined (silent failure).
+    * The following macros solve this situation by prefixing all inlined names,
+    * avoiding naming collision with previous inclusions.
+    */
+   /* Before that, we unconditionally #undef all symbols,
+    * in case they were already defined with XXH_NAMESPACE.
+    * They will then be redefined for XXH_INLINE_ALL
+    */
+#  undef XXH_versionNumber
+    /* XXH32 */
+#  undef XXH32
+#  undef XXH32_createState
+#  undef XXH32_freeState
+#  undef XXH32_reset
+#  undef XXH32_update
+#  undef XXH32_digest
+#  undef XXH32_copyState
+#  undef XXH32_canonicalFromHash
+#  undef XXH32_hashFromCanonical
+    /* XXH64 */
+#  undef XXH64
+#  undef XXH64_createState
+#  undef XXH64_freeState
+#  undef XXH64_reset
+#  undef XXH64_update
+#  undef XXH64_digest
+#  undef XXH64_copyState
+#  undef XXH64_canonicalFromHash
+#  undef XXH64_hashFromCanonical
+    /* XXH3_64bits */
+#  undef XXH3_64bits
+#  undef XXH3_64bits_withSecret
+#  undef XXH3_64bits_withSeed
+#  undef XXH3_64bits_withSecretandSeed
+#  undef XXH3_createState
+#  undef XXH3_freeState
+#  undef XXH3_copyState
+#  undef XXH3_64bits_reset
+#  undef XXH3_64bits_reset_withSeed
+#  undef XXH3_64bits_reset_withSecret
+#  undef XXH3_64bits_update
+#  undef XXH3_64bits_digest
+#  undef XXH3_generateSecret
+    /* XXH3_128bits */
+#  undef XXH128
+#  undef XXH3_128bits
+#  undef XXH3_128bits_withSeed
+#  undef XXH3_128bits_withSecret
+#  undef XXH3_128bits_reset
+#  undef XXH3_128bits_reset_withSeed
+#  undef XXH3_128bits_reset_withSecret
+#  undef XXH3_128bits_reset_withSecretandSeed
+#  undef XXH3_128bits_update
+#  undef XXH3_128bits_digest
+#  undef XXH128_isEqual
+#  undef XXH128_cmp
+#  undef XXH128_canonicalFromHash
+#  undef XXH128_hashFromCanonical
+    /* Finally, free the namespace itself */
+#  undef XXH_NAMESPACE
+
+    /* employ the namespace for XXH_INLINE_ALL */
+#  define XXH_NAMESPACE XXH_INLINE_
+   /*
+    * Some identifiers (enums, type names) are not symbols,
+    * but they must nonetheless be renamed to avoid redeclaration.
+    * Alternative solution: do not redeclare them.
+    * However, this requires some #ifdefs, and has a more dispersed impact.
+    * Meanwhile, renaming can be achieved in a single place.
+    */
+#  define XXH_IPREF(Id)   XXH_NAMESPACE ## Id
+#  define XXH_OK XXH_IPREF(XXH_OK)
+#  define XXH_ERROR XXH_IPREF(XXH_ERROR)
+#  define XXH_errorcode XXH_IPREF(XXH_errorcode)
+#  define XXH32_canonical_t  XXH_IPREF(XXH32_canonical_t)
+#  define XXH64_canonical_t  XXH_IPREF(XXH64_canonical_t)
+#  define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
+#  define XXH32_state_s XXH_IPREF(XXH32_state_s)
+#  define XXH32_state_t XXH_IPREF(XXH32_state_t)
+#  define XXH64_state_s XXH_IPREF(XXH64_state_s)
+#  define XXH64_state_t XXH_IPREF(XXH64_state_t)
+#  define XXH3_state_s  XXH_IPREF(XXH3_state_s)
+#  define XXH3_state_t  XXH_IPREF(XXH3_state_t)
+#  define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
+   /* Ensure the header is parsed again, even if it was previously included */
+#  undef XXHASH_H_5627135585666179
+#  undef XXHASH_H_STATIC_13879238742
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+
+/* ****************************************************************
+ *  Stable API
+ *****************************************************************/
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+/*! @brief Marks a global symbol. */
+#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
+#  if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
+#    ifdef XXH_EXPORT
+#      define XXH_PUBLIC_API __declspec(dllexport)
+#    elif XXH_IMPORT
+#      define XXH_PUBLIC_API __declspec(dllimport)
+#    endif
+#  else
+#    define XXH_PUBLIC_API   /* do nothing */
+#  endif
+#endif
+
+#ifdef XXH_NAMESPACE
+#  define XXH_CAT(A,B) A##B
+#  define XXH_NAME2(A,B) XXH_CAT(A,B)
+#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+/* XXH32 */
+#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+/* XXH64 */
+#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+/* XXH3_64bits */
+#  define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
+#  define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
+#  define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
+#  define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
+#  define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
+#  define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
+#  define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
+#  define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
+#  define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
+#  define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
+#  define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
+#  define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
+#  define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
+#  define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
+#  define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
+/* XXH3_128bits */
+#  define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
+#  define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
+#  define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
+#  define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
+#  define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
+#  define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
+#  define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
+#  define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
+#  define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
+#  define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
+#  define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
+#  define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
+#  define XXH128_cmp     XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
+#  define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
+#  define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
+#endif
+
+
+/* *************************************
+*  Compiler specifics
+***************************************/
+
+/* specific declaration modes for Windows */
+#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
+#  if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
+#    ifdef XXH_EXPORT
+#      define XXH_PUBLIC_API __declspec(dllexport)
+#    elif XXH_IMPORT
+#      define XXH_PUBLIC_API __declspec(dllimport)
+#    endif
+#  else
+#    define XXH_PUBLIC_API   /* do nothing */
+#  endif
+#endif
+
+#if defined (__GNUC__)
+# define XXH_CONSTF  __attribute__((const))
+# define XXH_PUREF   __attribute__((pure))
+# define XXH_MALLOCF __attribute__((malloc))
+#else
+# define XXH_CONSTF  /* disable */
+# define XXH_PUREF
+# define XXH_MALLOCF
+#endif
+
+/* *************************************
+*  Version
+***************************************/
+#define XXH_VERSION_MAJOR    0
+#define XXH_VERSION_MINOR    8
+#define XXH_VERSION_RELEASE  2
+/*! @brief Version number, encoded as two digits each */
+#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+
+/*!
+ * @brief Obtains the xxHash version.
+ *
+ * This is mostly useful when xxHash is compiled as a shared library,
+ * since the returned value comes from the library, as opposed to header file.
+ *
+ * @return @ref XXH_VERSION_NUMBER of the invoked library.
+ */
+XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);
+
+
+/* ****************************
+*  Common basic types
+******************************/
+#include <stddef.h>   /* size_t */
+/*!
+ * @brief Exit code for the streaming API.
+ */
+typedef enum {
+    XXH_OK = 0, /*!< OK */
+    XXH_ERROR   /*!< Error */
+} XXH_errorcode;
+
+
+/*-**********************************************************************
+*  32-bit hash
+************************************************************************/
+#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
+/*!
+ * @brief An unsigned 32-bit integer.
+ *
+ * Not necessarily defined to `uint32_t` but functionally equivalent.
+ */
+typedef uint32_t XXH32_hash_t;
+
+#elif !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   ifdef _AIX
+#     include <inttypes.h>
+#   else
+#     include <stdint.h>
+#   endif
+    typedef uint32_t XXH32_hash_t;
+
+#else
+#   include <limits.h>
+#   if UINT_MAX == 0xFFFFFFFFUL
+      typedef unsigned int XXH32_hash_t;
+#   elif ULONG_MAX == 0xFFFFFFFFUL
+      typedef unsigned long XXH32_hash_t;
+#   else
+#     error "unsupported platform: need a 32-bit type"
+#   endif
+#endif
+
+/*!
+ * @}
+ *
+ * @defgroup XXH32_family XXH32 family
+ * @ingroup public
+ * Contains functions used in the classic 32-bit xxHash algorithm.
+ *
+ * @note
+ *   XXH32 is useful for older platforms, with no or poor 64-bit performance.
+ *   Note that the @ref XXH3_family provides competitive speed for both 32-bit
+ *   and 64-bit systems, and offers true 64/128 bit hash results.
+ *
+ * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families
+ * @see @ref XXH32_impl for implementation details
+ * @{
+ */
+
+/*!
+ * @brief Calculates the 32-bit hash of @p input using xxHash32.
+ *
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed The 32-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 32-bit xxHash32 value.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
+
+#ifndef XXH_NO_STREAM
+/*!
+ * @typedef struct XXH32_state_s XXH32_state_t
+ * @brief The opaque state struct for the XXH32 streaming API.
+ *
+ * @see XXH32_state_s for details.
+ */
+typedef struct XXH32_state_s XXH32_state_t;
+
+/*!
+ * @brief Allocates an @ref XXH32_state_t.
+ *
+ * @return An allocated pointer of @ref XXH32_state_t on success.
+ * @return `NULL` on failure.
+ *
+ * @note Must be freed with XXH32_freeState().
+ */
+XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
+/*!
+ * @brief Frees an @ref XXH32_state_t.
+ *
+ * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
+ *
+ * @return @ref XXH_OK.
+ *
+ * @note @p statePtr must be allocated with XXH32_createState().
+ *
+ */
+XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
+/*!
+ * @brief Copies one @ref XXH32_state_t to another.
+ *
+ * @param dst_state The state to copy to.
+ * @param src_state The state to copy from.
+ * @pre
+ *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
+ */
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+/*!
+ * @brief Resets an @ref XXH32_state_t to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ * @param seed The 32-bit seed to alter the hash result predictably.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note This function resets and seeds a state. Call it before @ref XXH32_update().
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, XXH32_hash_t seed);
+
+/*!
+ * @brief Consumes a block of @p input to an @ref XXH32_state_t.
+ *
+ * @param statePtr The state struct to update.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ * @pre
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note Call this to incrementally consume blocks of data.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+
+/*!
+ * @brief Returns the calculated hash value from an @ref XXH32_state_t.
+ *
+ * @param statePtr The state struct to calculate the hash from.
+ *
+ * @pre
+ *  @p statePtr must not be `NULL`.
+ *
+ * @return The calculated 32-bit xxHash32 value from that state.
+ *
+ * @note
+ *   Calling XXH32_digest() will not affect @p statePtr, so you can update,
+ *   digest, and update again.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
+#endif /* !XXH_NO_STREAM */
+
+/*******   Canonical representation   *******/
+
+/*!
+ * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
+ */
+typedef struct {
+    unsigned char digest[4]; /*!< Hash bytes, big endian */
+} XXH32_canonical_t;
+
+/*!
+ * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
+ *
+ * @param dst  The @ref XXH32_canonical_t pointer to be stored to.
+ * @param hash The @ref XXH32_hash_t to be converted.
+ *
+ * @pre
+ *   @p dst must not be `NULL`.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
+ */
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+
+/*!
+ * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
+ *
+ * @param src The @ref XXH32_canonical_t to convert.
+ *
+ * @pre
+ *   @p src must not be `NULL`.
+ *
+ * @return The converted hash.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
+ */
+XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+
+/*! @cond Doxygen ignores this part */
+#ifdef __has_attribute
+# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
+#else
+# define XXH_HAS_ATTRIBUTE(x) 0
+#endif
+/*! @endcond */
+
+/*! @cond Doxygen ignores this part */
+/*
+ * C23 __STDC_VERSION__ number hasn't been specified yet. For now
+ * leave as `201711L` (C17 + 1).
+ * TODO: Update to correct value when its been specified.
+ */
+#define XXH_C23_VN 201711L
+/*! @endcond */
+
+/*! @cond Doxygen ignores this part */
+/* C-language Attributes are added in C23. */
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute)
+# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
+#else
+# define XXH_HAS_C_ATTRIBUTE(x) 0
+#endif
+/*! @endcond */
+
+/*! @cond Doxygen ignores this part */
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define XXH_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+/*! @endcond */
+
+/*! @cond Doxygen ignores this part */
+/*
+ * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
+ * introduced in CPP17 and C23.
+ * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
+ * C23   : https://en.cppreference.com/w/c/language/attributes/fallthrough
+ */
+#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough)
+# define XXH_FALLTHROUGH [[fallthrough]]
+#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
+# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__))
+#else
+# define XXH_FALLTHROUGH /* fallthrough */
+#endif
+/*! @endcond */
+
+/*! @cond Doxygen ignores this part */
+/*
+ * Define XXH_NOESCAPE for annotated pointers in public API.
+ * https://clang.llvm.org/docs/AttributeReference.html#noescape
+ * As of writing this, only supported by clang.
+ */
+#if XXH_HAS_ATTRIBUTE(noescape)
+# define XXH_NOESCAPE __attribute__((noescape))
+#else
+# define XXH_NOESCAPE
+#endif
+/*! @endcond */
+
+
+/*!
+ * @}
+ * @ingroup public
+ * @{
+ */
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+*  64-bit hash
+************************************************************************/
+#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
+/*!
+ * @brief An unsigned 64-bit integer.
+ *
+ * Not necessarily defined to `uint64_t` but functionally equivalent.
+ */
+typedef uint64_t XXH64_hash_t;
+#elif !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   ifdef _AIX
+#     include <inttypes.h>
+#   else
+#     include <stdint.h>
+#   endif
+   typedef uint64_t XXH64_hash_t;
+#else
+#  include <limits.h>
+#  if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
+     /* LP64 ABI says uint64_t is unsigned long */
+     typedef unsigned long XXH64_hash_t;
+#  else
+     /* the following type must have a width of 64-bit */
+     typedef unsigned long long XXH64_hash_t;
+#  endif
+#endif
+
+/*!
+ * @}
+ *
+ * @defgroup XXH64_family XXH64 family
+ * @ingroup public
+ * @{
+ * Contains functions used in the classic 64-bit xxHash algorithm.
+ *
+ * @note
+ *   XXH3 provides competitive speed for both 32-bit and 64-bit systems,
+ *   and offers true 64/128 bit hash results.
+ *   It provides better speed for systems with vector processing capabilities.
+ */
+
+/*!
+ * @brief Calculates the 64-bit hash of @p input using xxHash64.
+ *
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed The 64-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 64-bit xxHash64 value.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
+
+/*******   Streaming   *******/
+#ifndef XXH_NO_STREAM
+/*!
+ * @brief The opaque state struct for the XXH64 streaming API.
+ *
+ * @see XXH64_state_s for details.
+ */
+typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
+
+/*!
+ * @brief Allocates an @ref XXH64_state_t.
+ *
+ * @return An allocated pointer of @ref XXH64_state_t on success.
+ * @return `NULL` on failure.
+ *
+ * @note Must be freed with XXH64_freeState().
+ */
+XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
+
+/*!
+ * @brief Frees an @ref XXH64_state_t.
+ *
+ * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState().
+ *
+ * @return @ref XXH_OK.
+ *
+ * @note @p statePtr must be allocated with XXH64_createState().
+ */
+XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
+
+/*!
+ * @brief Copies one @ref XXH64_state_t to another.
+ *
+ * @param dst_state The state to copy to.
+ * @param src_state The state to copy from.
+ * @pre
+ *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
+ */
+XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+/*!
+ * @brief Resets an @ref XXH64_state_t to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ * @param seed The 64-bit seed to alter the hash result predictably.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note This function resets and seeds a state. Call it before @ref XXH64_update().
+ */
+XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);
+
+/*!
+ * @brief Consumes a block of @p input to an @ref XXH64_state_t.
+ *
+ * @param statePtr The state struct to update.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ * @pre
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note Call this to incrementally consume blocks of data.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
+
+/*!
+ * @brief Returns the calculated hash value from an @ref XXH64_state_t.
+ *
+ * @param statePtr The state struct to calculate the hash from.
+ *
+ * @pre
+ *  @p statePtr must not be `NULL`.
+ *
+ * @return The calculated 64-bit xxHash64 value from that state.
+ *
+ * @note
+ *   Calling XXH64_digest() will not affect @p statePtr, so you can update,
+ *   digest, and update again.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
+#endif /* !XXH_NO_STREAM */
+/*******   Canonical representation   *******/
+
+/*!
+ * @brief Canonical (big endian) representation of @ref XXH64_hash_t.
+ */
+typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
+
+/*!
+ * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t.
+ *
+ * @param dst The @ref XXH64_canonical_t pointer to be stored to.
+ * @param hash The @ref XXH64_hash_t to be converted.
+ *
+ * @pre
+ *   @p dst must not be `NULL`.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
+ */
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash);
+
+/*!
+ * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t.
+ *
+ * @param src The @ref XXH64_canonical_t to convert.
+ *
+ * @pre
+ *   @p src must not be `NULL`.
+ *
+ * @return The converted hash.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src);
+
+#ifndef XXH_NO_XXH3
+
+/*!
+ * @}
+ * ************************************************************************
+ * @defgroup XXH3_family XXH3 family
+ * @ingroup public
+ * @{
+ *
+ * XXH3 is a more recent hash algorithm featuring:
+ *  - Improved speed for both small and large inputs
+ *  - True 64-bit and 128-bit outputs
+ *  - SIMD acceleration
+ *  - Improved 32-bit viability
+ *
+ * Speed analysis methodology is explained here:
+ *
+ *    https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
+ *
+ * Compared to XXH64, expect XXH3 to run approximately
+ * ~2x faster on large inputs and >3x faster on small ones,
+ * exact differences vary depending on platform.
+ *
+ * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
+ * but does not require it.
+ * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3
+ * at competitive speeds, even without vector support. Further details are
+ * explained in the implementation.
+ *
+ * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD
+ * implementations for many common platforms:
+ *   - AVX512
+ *   - AVX2
+ *   - SSE2
+ *   - ARM NEON
+ *   - WebAssembly SIMD128
+ *   - POWER8 VSX
+ *   - s390x ZVector
+ * This can be controlled via the @ref XXH_VECTOR macro, but it automatically
+ * selects the best version according to predefined macros. For the x86 family, an
+ * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c.
+ *
+ * XXH3 implementation is portable:
+ * it has a generic C90 formulation that can be compiled on any platform,
+ * all implementations generate exactly the same hash value on all platforms.
+ * Starting from v0.8.0, it's also labelled "stable", meaning that
+ * any future version will also generate the same hash value.
+ *
+ * XXH3 offers 2 variants, _64bits and _128bits.
+ *
+ * When only 64 bits are needed, prefer invoking the _64bits variant, as it
+ * reduces the amount of mixing, resulting in faster speed on small inputs.
+ * It's also generally simpler to manipulate a scalar return type than a struct.
+ *
+ * The API supports one-shot hashing, streaming mode, and custom secrets.
+ */
+/*-**********************************************************************
+*  XXH3 64-bit variant
+************************************************************************/
+
+/*!
+ * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input.
+ *
+ * @param input  The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ *
+ * @pre
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 64-bit XXH3 hash value.
+ *
+ * @note
+ *   This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however
+ *   it may have slightly better performance due to constant propagation of the
+ *   defaults.
+ *
+ * @see
+ *    XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length);
+
+/*!
+ * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input.
+ *
+ * @param input  The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed   The 64-bit seed to alter the hash result predictably.
+ *
+ * @pre
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 64-bit XXH3 hash value.
+ *
+ * @note
+ *    seed == 0 produces the same results as @ref XXH3_64bits().
+ *
+ * This variant generates a custom secret on the fly based on default secret
+ * altered using the @p seed value.
+ *
+ * While this operation is decently fast, note that it's not completely free.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
+
+/*!
+ * The bare minimum size for a custom secret.
+ *
+ * @see
+ *  XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
+ *  XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
+ */
+#define XXH3_SECRET_SIZE_MIN 136
+
+/*!
+ * @brief Calculates 64-bit variant of XXH3 with a custom "secret".
+ *
+ * @param data       The block of data to be hashed, at least @p len bytes in size.
+ * @param len        The length of @p data, in bytes.
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @return The calculated 64-bit XXH3 hash value.
+ *
+ * @pre
+ *   The memory between @p data and @p data + @p len must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p data may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * It's possible to provide any blob of bytes as a "secret" to generate the hash.
+ * This makes it more difficult for an external actor to prepare an intentional collision.
+ * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
+ * However, the quality of the secret impacts the dispersion of the hash algorithm.
+ * Therefore, the secret _must_ look like a bunch of random bytes.
+ * Avoid "trivial" or structured data such as repeated sequences or a text document.
+ * Whenever in doubt about the "randomness" of the blob of bytes,
+ * consider employing @ref XXH3_generateSecret() instead (see below).
+ * It will generate a proper high entropy secret derived from the blob of bytes.
+ * Another advantage of using XXH3_generateSecret() is that
+ * it guarantees that all bits within the initial blob of bytes
+ * will impact every bit of the output.
+ * This is not necessarily the case when using the blob of bytes directly
+ * because, when hashing _small_ inputs, only a portion of the secret is employed.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
+
+
+/*******   Streaming   *******/
+#ifndef XXH_NO_STREAM
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
+ * As a consequence, streaming is slower than one-shot hashing.
+ * For better performance, prefer one-shot functions whenever applicable.
+ */
+
+/*!
+ * @brief The opaque state struct for the XXH3 streaming API.
+ *
+ * @see XXH3_state_s for details.
+ */
+typedef struct XXH3_state_s XXH3_state_t;
+XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
+
+/*!
+ * @brief Copies one @ref XXH3_state_t to another.
+ *
+ * @param dst_state The state to copy to.
+ * @param src_state The state to copy from.
+ * @pre
+ *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
+ */
+XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state);
+
+/*!
+ * @brief Resets an @ref XXH3_state_t to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ *   - This function resets `statePtr` and generate a secret with default parameters.
+ *   - Call this function before @ref XXH3_64bits_update().
+ *   - Digest will be equivalent to `XXH3_64bits()`.
+ *
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
+
+/*!
+ * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ * @param seed     The 64-bit seed to alter the hash result predictably.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ *   - This function resets `statePtr` and generate a secret from `seed`.
+ *   - Call this function before @ref XXH3_64bits_update().
+ *   - Digest will be equivalent to `XXH3_64bits_withSeed()`.
+ *
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
+
+/*!
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ *   `secret` is referenced, it _must outlive_ the hash streaming session.
+ *
+ * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
+ * and the quality of produced hash values depends on secret's entropy
+ * (secret's content should look like a bunch of random bytes).
+ * When in doubt about the randomness of a candidate `secret`,
+ * consider employing `XXH3_generateSecret()` instead (see below).
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
+
+/*!
+ * @brief Consumes a block of @p input to an @ref XXH3_state_t.
+ *
+ * @param statePtr The state struct to update.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ * @pre
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note Call this to incrementally consume blocks of data.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
+
+/*!
+ * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t.
+ *
+ * @param statePtr The state struct to calculate the hash from.
+ *
+ * @pre
+ *  @p statePtr must not be `NULL`.
+ *
+ * @return The calculated XXH3 64-bit hash value from that state.
+ *
+ * @note
+ *   Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
+ *   digest, and update again.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t  XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
+#endif /* !XXH_NO_STREAM */
+
+/* note : canonical representation of XXH3 is the same as XXH64
+ * since they both produce XXH64_hash_t values */
+
+
+/*-**********************************************************************
+*  XXH3 128-bit variant
+************************************************************************/
+
+/*!
+ * @brief The return value from 128-bit hashes.
+ *
+ * Stored in little endian order, although the fields themselves are in native
+ * endianness.
+ */
+typedef struct {
+    XXH64_hash_t low64;   /*!< `value & 0xFFFFFFFFFFFFFFFF` */
+    XXH64_hash_t high64;  /*!< `value >> 64` */
+} XXH128_hash_t;
+
+/*!
+ * @brief Calculates 128-bit unseeded variant of XXH3 of @p data.
+ *
+ * @param data The block of data to be hashed, at least @p length bytes in size.
+ * @param len  The length of @p data, in bytes.
+ *
+ * @return The calculated 128-bit variant of XXH3 value.
+ *
+ * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead
+ * for shorter inputs.
+ *
+ * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however
+ * it may have slightly better performance due to constant propagation of the
+ * defaults.
+ *
+ * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len);
+/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
+ *
+ * @param data The block of data to be hashed, at least @p length bytes in size.
+ * @param len  The length of @p data, in bytes.
+ * @param seed The 64-bit seed to alter the hash result predictably.
+ *
+ * @return The calculated 128-bit variant of XXH3 value.
+ *
+ * @note
+ *    seed == 0 produces the same results as @ref XXH3_64bits().
+ *
+ * This variant generates a custom secret on the fly based on default secret
+ * altered using the @p seed value.
+ *
+ * While this operation is decently fast, note that it's not completely free.
+ *
+ * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
+/*!
+ * @brief Calculates 128-bit variant of XXH3 with a custom "secret".
+ *
+ * @param data       The block of data to be hashed, at least @p len bytes in size.
+ * @param len        The length of @p data, in bytes.
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @return The calculated 128-bit variant of XXH3 value.
+ *
+ * It's possible to provide any blob of bytes as a "secret" to generate the hash.
+ * This makes it more difficult for an external actor to prepare an intentional collision.
+ * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
+ * However, the quality of the secret impacts the dispersion of the hash algorithm.
+ * Therefore, the secret _must_ look like a bunch of random bytes.
+ * Avoid "trivial" or structured data such as repeated sequences or a text document.
+ * Whenever in doubt about the "randomness" of the blob of bytes,
+ * consider employing @ref XXH3_generateSecret() instead (see below).
+ * It will generate a proper high entropy secret derived from the blob of bytes.
+ * Another advantage of using XXH3_generateSecret() is that
+ * it guarantees that all bits within the initial blob of bytes
+ * will impact every bit of the output.
+ * This is not necessarily the case when using the blob of bytes directly
+ * because, when hashing _small_ inputs, only a portion of the secret is employed.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
+
+/*******   Streaming   *******/
+#ifndef XXH_NO_STREAM
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
+ * As a consequence, streaming is slower than one-shot hashing.
+ * For better performance, prefer one-shot functions whenever applicable.
+ *
+ * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
+ * Use already declared XXH3_createState() and XXH3_freeState().
+ *
+ * All reset and streaming functions have same meaning as their 64-bit counterpart.
+ */
+
+/*!
+ * @brief Resets an @ref XXH3_state_t to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ *   - This function resets `statePtr` and generate a secret with default parameters.
+ *   - Call it before @ref XXH3_128bits_update().
+ *   - Digest will be equivalent to `XXH3_128bits()`.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
+
+/*!
+ * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ * @param seed     The 64-bit seed to alter the hash result predictably.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ *   - This function resets `statePtr` and generate a secret from `seed`.
+ *   - Call it before @ref XXH3_128bits_update().
+ *   - Digest will be equivalent to `XXH3_128bits_withSeed()`.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
+/*!
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr   The state struct to reset.
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * `secret` is referenced, it _must outlive_ the hash streaming session.
+ * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
+ * and the quality of produced hash values depends on secret's entropy
+ * (secret's content should look like a bunch of random bytes).
+ * When in doubt about the randomness of a candidate `secret`,
+ * consider employing `XXH3_generateSecret()` instead (see below).
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
+
+/*!
+ * @brief Consumes a block of @p input to an @ref XXH3_state_t.
+ *
+ * Call this to incrementally consume blocks of data.
+ *
+ * @param statePtr The state struct to update.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ *
+ * @pre
+ *   @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ *   The memory between @p input and @p input + @p length must be valid,
+ *   readable, contiguous memory. However, if @p length is `0`, @p input may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
+
+/*!
+ * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t.
+ *
+ * @param statePtr The state struct to calculate the hash from.
+ *
+ * @pre
+ *  @p statePtr must not be `NULL`.
+ *
+ * @return The calculated XXH3 128-bit hash value from that state.
+ *
+ * @note
+ *   Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
+ *   digest, and update again.
+ *
+ */
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
+#endif /* !XXH_NO_STREAM */
+
+/* Following helper functions make it possible to compare XXH128_hast_t values.
+ * Since XXH128_hash_t is a structure, this capability is not offered by the language.
+ * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
+
+/*!
+ * @brief Check equality of two XXH128_hash_t values
+ *
+ * @param h1 The 128-bit hash value.
+ * @param h2 Another 128-bit hash value.
+ *
+ * @return `1` if `h1` and `h2` are equal.
+ * @return `0` if they are not.
+ */
+XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
+
+/*!
+ * @brief Compares two @ref XXH128_hash_t
+ *
+ * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
+ *
+ * @param h128_1 Left-hand side value
+ * @param h128_2 Right-hand side value
+ *
+ * @return >0 if @p h128_1  > @p h128_2
+ * @return =0 if @p h128_1 == @p h128_2
+ * @return <0 if @p h128_1  < @p h128_2
+ */
+XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2);
+
+
+/*******   Canonical representation   *******/
+typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
+
+
+/*!
+ * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t.
+ *
+ * @param dst  The @ref XXH128_canonical_t pointer to be stored to.
+ * @param hash The @ref XXH128_hash_t to be converted.
+ *
+ * @pre
+ *   @p dst must not be `NULL`.
+ * @see @ref canonical_representation_example "Canonical Representation Example"
+ */
+XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash);
+
+/*!
+ * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t.
+ *
+ * @param src The @ref XXH128_canonical_t to convert.
+ *
+ * @pre
+ *   @p src must not be `NULL`.
+ *
+ * @return The converted hash.
+ * @see @ref canonical_representation_example "Canonical Representation Example"
+ */
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src);
+
+
+#endif  /* !XXH_NO_XXH3 */
+#endif  /* XXH_NO_LONG_LONG */
+
+/*!
+ * @}
+ */
+#endif /* XXHASH_H_5627135585666179 */
+
+
+
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
+#define XXHASH_H_STATIC_13879238742
+/* ****************************************************************************
+ * This section contains declarations which are not guaranteed to remain stable.
+ * They may change in future versions, becoming incompatible with a different
+ * version of the library.
+ * These declarations should only be used with static linking.
+ * Never use them in association with dynamic linking!
+ ***************************************************************************** */
+
+/*
+ * These definitions are only present to allow static allocation
+ * of XXH states, on stack or in a struct, for example.
+ * Never **ever** access their members directly.
+ */
+
+/*!
+ * @internal
+ * @brief Structure for XXH32 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
+ * an opaque type. This allows fields to safely be changed.
+ *
+ * Typedef'd to @ref XXH32_state_t.
+ * Do not access the members of this struct directly.
+ * @see XXH64_state_s, XXH3_state_s
+ */
+struct XXH32_state_s {
+   XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
+   XXH32_hash_t large_len;    /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
+   XXH32_hash_t v[4];         /*!< Accumulator lanes */
+   XXH32_hash_t mem32[4];     /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
+   XXH32_hash_t memsize;      /*!< Amount of data in @ref mem32 */
+   XXH32_hash_t reserved;     /*!< Reserved field. Do not read nor write to it. */
+};   /* typedef'd to XXH32_state_t */
+
+
+#ifndef XXH_NO_LONG_LONG  /* defined when there is no 64-bit support */
+
+/*!
+ * @internal
+ * @brief Structure for XXH64 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
+ * an opaque type. This allows fields to safely be changed.
+ *
+ * Typedef'd to @ref XXH64_state_t.
+ * Do not access the members of this struct directly.
+ * @see XXH32_state_s, XXH3_state_s
+ */
+struct XXH64_state_s {
+   XXH64_hash_t total_len;    /*!< Total length hashed. This is always 64-bit. */
+   XXH64_hash_t v[4];         /*!< Accumulator lanes */
+   XXH64_hash_t mem64[4];     /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
+   XXH32_hash_t memsize;      /*!< Amount of data in @ref mem64 */
+   XXH32_hash_t reserved32;   /*!< Reserved field, needed for padding anyways*/
+   XXH64_hash_t reserved64;   /*!< Reserved field. Do not read or write to it. */
+};   /* typedef'd to XXH64_state_t */
+
+#ifndef XXH_NO_XXH3
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
+#  include <stdalign.h>
+#  define XXH_ALIGN(n)      alignas(n)
+#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
+/* In C++ alignas() is a keyword */
+#  define XXH_ALIGN(n)      alignas(n)
+#elif defined(__GNUC__)
+#  define XXH_ALIGN(n)      __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+#  define XXH_ALIGN(n)      __declspec(align(n))
+#else
+#  define XXH_ALIGN(n)   /* disabled */
+#endif
+
+/* Old GCC versions only accept the attribute after the type in structures. */
+#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))   /* C11+ */ \
+    && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
+    && defined(__GNUC__)
+#   define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
+#else
+#   define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
+#endif
+
+/*!
+ * @brief The size of the internal XXH3 buffer.
+ *
+ * This is the optimal update size for incremental hashing.
+ *
+ * @see XXH3_64b_update(), XXH3_128b_update().
+ */
+#define XXH3_INTERNALBUFFER_SIZE 256
+
+/*!
+ * @internal
+ * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
+ *
+ * This is the size used in @ref XXH3_kSecret and the seeded functions.
+ *
+ * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
+ */
+#define XXH3_SECRET_DEFAULT_SIZE 192
+
+/*!
+ * @internal
+ * @brief Structure for XXH3 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
+ * Otherwise it is an opaque type.
+ * Never use this definition in combination with dynamic library.
+ * This allows fields to safely be changed in the future.
+ *
+ * @note ** This structure has a strict alignment requirement of 64 bytes!! **
+ * Do not allocate this with `malloc()` or `new`,
+ * it will not be sufficiently aligned.
+ * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
+ *
+ * Typedef'd to @ref XXH3_state_t.
+ * Do never access the members of this struct directly.
+ *
+ * @see XXH3_INITSTATE() for stack initialization.
+ * @see XXH3_createState(), XXH3_freeState().
+ * @see XXH32_state_s, XXH64_state_s
+ */
+struct XXH3_state_s {
+   XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
+       /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */
+   XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
+       /*!< Used to store a custom secret generated from a seed. */
+   XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
+       /*!< The internal buffer. @see XXH32_state_s::mem32 */
+   XXH32_hash_t bufferedSize;
+       /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
+   XXH32_hash_t useSeed;
+       /*!< Reserved field. Needed for padding on 64-bit. */
+   size_t nbStripesSoFar;
+       /*!< Number or stripes processed. */
+   XXH64_hash_t totalLen;
+       /*!< Total length hashed. 64-bit even on 32-bit targets. */
+   size_t nbStripesPerBlock;
+       /*!< Number of stripes per block. */
+   size_t secretLimit;
+       /*!< Size of @ref customSecret or @ref extSecret */
+   XXH64_hash_t seed;
+       /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
+   XXH64_hash_t reserved64;
+       /*!< Reserved field. */
+   const unsigned char* extSecret;
+       /*!< Reference to an external secret for the _withSecret variants, NULL
+        *   for other variants. */
+   /* note: there may be some padding at the end due to alignment on 64 bytes */
+}; /* typedef'd to XXH3_state_t */
+
+#undef XXH_ALIGN_MEMBER
+
+/*!
+ * @brief Initializes a stack-allocated `XXH3_state_s`.
+ *
+ * When the @ref XXH3_state_t structure is merely emplaced on stack,
+ * it should be initialized with XXH3_INITSTATE() or a memset()
+ * in case its first reset uses XXH3_NNbits_reset_withSeed().
+ * This init can be omitted if the first reset uses default or _withSecret mode.
+ * This operation isn't necessary when the state is created with XXH3_createState().
+ * Note that this doesn't prepare the state for a streaming operation,
+ * it's still necessary to use XXH3_NNbits_reset*() afterwards.
+ */
+#define XXH3_INITSTATE(XXH3_state_ptr)                       \
+    do {                                                     \
+        XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \
+        tmp_xxh3_state_ptr->seed = 0;                        \
+        tmp_xxh3_state_ptr->extSecret = NULL;                \
+    } while(0)
+
+
+/*!
+ * @brief Calculates the 128-bit hash of @p data using XXH3.
+ *
+ * @param data The block of data to be hashed, at least @p len bytes in size.
+ * @param len  The length of @p data, in bytes.
+ * @param seed The 64-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ *   The memory between @p data and @p data + @p len must be valid,
+ *   readable, contiguous memory. However, if @p len is `0`, @p data may be
+ *   `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 128-bit XXH3 value.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
+
+
+/* ===   Experimental API   === */
+/* Symbols defined below must be considered tied to a specific library version. */
+
+/*!
+ * @brief Derive a high-entropy secret from any user-defined content, named customSeed.
+ *
+ * @param secretBuffer    A writable buffer for derived high-entropy secret data.
+ * @param secretSize      Size of secretBuffer, in bytes.  Must be >= XXH3_SECRET_DEFAULT_SIZE.
+ * @param customSeed      A user-defined content.
+ * @param customSeedSize  Size of customSeed, in bytes.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * The generated secret can be used in combination with `*_withSecret()` functions.
+ * The `_withSecret()` variants are useful to provide a higher level of protection
+ * than 64-bit seed, as it becomes much more difficult for an external actor to
+ * guess how to impact the calculation logic.
+ *
+ * The function accepts as input a custom seed of any length and any content,
+ * and derives from it a high-entropy secret of length @p secretSize into an
+ * already allocated buffer @p secretBuffer.
+ *
+ * The generated secret can then be used with any `*_withSecret()` variant.
+ * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(),
+ * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret()
+ * are part of this list. They all accept a `secret` parameter
+ * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN)
+ * _and_ feature very high entropy (consist of random-looking bytes).
+ * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can
+ * be employed to ensure proper quality.
+ *
+ * @p customSeed can be anything. It can have any size, even small ones,
+ * and its content can be anything, even "poor entropy" sources such as a bunch
+ * of zeroes. The resulting `secret` will nonetheless provide all required qualities.
+ *
+ * @pre
+ *   - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN
+ *   - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
+ *
+ * Example code:
+ * @code{.c}
+ *    #include <stdio.h>
+ *    #include <stdlib.h>
+ *    #include <string.h>
+ *    #define XXH_STATIC_LINKING_ONLY // expose unstable API
+ *    #include "xxhash.h"
+ *    // Hashes argv[2] using the entropy from argv[1].
+ *    int main(int argc, char* argv[])
+ *    {
+ *        char secret[XXH3_SECRET_SIZE_MIN];
+ *        if (argv != 3) { return 1; }
+ *        XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1]));
+ *        XXH64_hash_t h = XXH3_64bits_withSecret(
+ *             argv[2], strlen(argv[2]),
+ *             secret, sizeof(secret)
+ *        );
+ *        printf("%016llx\n", (unsigned long long) h);
+ *    }
+ * @endcode
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize);
+
+/*!
+ * @brief Generate the same secret as the _withSeed() variants.
+ *
+ * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes
+ * @param seed         The 64-bit seed to alter the hash result predictably.
+ *
+ * The generated secret can be used in combination with
+ *`*_withSecret()` and `_withSecretandSeed()` variants.
+ *
+ * Example C++ `std::string` hash class:
+ * @code{.cpp}
+ *    #include <string>
+ *    #define XXH_STATIC_LINKING_ONLY // expose unstable API
+ *    #include "xxhash.h"
+ *    // Slow, seeds each time
+ *    class HashSlow {
+ *        XXH64_hash_t seed;
+ *    public:
+ *        HashSlow(XXH64_hash_t s) : seed{s} {}
+ *        size_t operator()(const std::string& x) const {
+ *            return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)};
+ *        }
+ *    };
+ *    // Fast, caches the seeded secret for future uses.
+ *    class HashFast {
+ *        unsigned char secret[XXH3_SECRET_SIZE_MIN];
+ *    public:
+ *        HashFast(XXH64_hash_t s) {
+ *            XXH3_generateSecret_fromSeed(secret, seed);
+ *        }
+ *        size_t operator()(const std::string& x) const {
+ *            return size_t{
+ *                XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret))
+ *            };
+ *        }
+ *    };
+ * @endcode
+ */
+XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed);
+
+/*!
+ * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data.
+ *
+ * @param data       The block of data to be hashed, at least @p len bytes in size.
+ * @param len        The length of @p data, in bytes.
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ * @param seed       The 64-bit seed to alter the hash result predictably.
+ *
+ * These variants generate hash values using either
+ * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes)
+ * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX).
+ *
+ * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
+ * `_withSeed()` has to generate the secret on the fly for "large" keys.
+ * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
+ * `_withSecret()` has to generate the masks on the fly for "small" keys,
+ * which requires more instructions than _withSeed() variants.
+ * Therefore, _withSecretandSeed variant combines the best of both worlds.
+ *
+ * When @p secret has been generated by XXH3_generateSecret_fromSeed(),
+ * this variant produces *exactly* the same results as `_withSeed()` variant,
+ * hence offering only a pure speed benefit on "large" input,
+ * by skipping the need to regenerate the secret for every large input.
+ *
+ * Another usage scenario is to hash the secret to a 64-bit hash value,
+ * for example with XXH3_64bits(), which then becomes the seed,
+ * and then employ both the seed and the secret in _withSecretandSeed().
+ * On top of speed, an added benefit is that each bit in the secret
+ * has a 50% chance to swap each bit in the output, via its impact to the seed.
+ *
+ * This is not guaranteed when using the secret directly in "small data" scenarios,
+ * because only portions of the secret are employed for small data.
+ */
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t
+XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len,
+                              XXH_NOESCAPE const void* secret, size_t secretSize,
+                              XXH64_hash_t seed);
+/*!
+ * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
+ *
+ * @param input      The block of data to be hashed, at least @p len bytes in size.
+ * @param length     The length of @p data, in bytes.
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ * @param seed64     The 64-bit seed to alter the hash result predictably.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @see XXH3_64bits_withSecretandSeed()
+ */
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t
+XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length,
+                               XXH_NOESCAPE const void* secret, size_t secretSize,
+                               XXH64_hash_t seed64);
+#ifndef XXH_NO_STREAM
+/*!
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr   A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ * @param seed64     The 64-bit seed to alter the hash result predictably.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @see XXH3_64bits_withSecretandSeed()
+ */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
+                                    XXH_NOESCAPE const void* secret, size_t secretSize,
+                                    XXH64_hash_t seed64);
+/*!
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr   A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
+ * @param secret     The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ * @param seed64     The 64-bit seed to alter the hash result predictably.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @see XXH3_64bits_withSecretandSeed()
+ */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
+                                     XXH_NOESCAPE const void* secret, size_t secretSize,
+                                     XXH64_hash_t seed64);
+#endif /* !XXH_NO_STREAM */
+
+#endif  /* !XXH_NO_XXH3 */
+#endif  /* XXH_NO_LONG_LONG */
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+#  define XXH_IMPLEMENTATION
+#endif
+
+#endif  /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
+
+
+/* ======================================================================== */
+/* ======================================================================== */
+/* ======================================================================== */
+
+
+/*-**********************************************************************
+ * xxHash implementation
+ *-**********************************************************************
+ * xxHash's implementation used to be hosted inside xxhash.c.
+ *
+ * However, inlining requires implementation to be visible to the compiler,
+ * hence be included alongside the header.
+ * Previously, implementation was hosted inside xxhash.c,
+ * which was then #included when inlining was activated.
+ * This construction created issues with a few build and install systems,
+ * as it required xxhash.c to be stored in /include directory.
+ *
+ * xxHash implementation is now directly integrated within xxhash.h.
+ * As a consequence, xxhash.c is no longer needed in /include.
+ *
+ * xxhash.c is still available and is still useful.
+ * In a "normal" setup, when xxhash is not inlined,
+ * xxhash.h only exposes the prototypes and public symbols,
+ * while xxhash.c can be built into an object file xxhash.o
+ * which can then be linked into the final binary.
+ ************************************************************************/
+
+#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
+   || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
+#  define XXH_IMPLEM_13a8737387
+
+/* *************************************
+*  Tuning parameters
+***************************************/
+
+/*!
+ * @defgroup tuning Tuning parameters
+ * @{
+ *
+ * Various macros to control xxHash's behavior.
+ */
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Define this to disable 64-bit code.
+ *
+ * Useful if only using the @ref XXH32_family and you have a strict C90 compiler.
+ */
+#  define XXH_NO_LONG_LONG
+#  undef XXH_NO_LONG_LONG /* don't actually */
+/*!
+ * @brief Controls how unaligned memory is accessed.
+ *
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is
+ * safe and portable.
+ *
+ * Unfortunately, on some target/compiler combinations, the generated assembly
+ * is sub-optimal.
+ *
+ * The below switch allow selection of a different access method
+ * in the search for improved performance.
+ *
+ * @par Possible options:
+ *
+ *  - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
+ *   @par
+ *     Use `memcpy()`. Safe and portable. Note that most modern compilers will
+ *     eliminate the function call and treat it as an unaligned access.
+ *
+ *  - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))`
+ *   @par
+ *     Depends on compiler extensions and is therefore not portable.
+ *     This method is safe _if_ your compiler supports it,
+ *     and *generally* as fast or faster than `memcpy`.
+ *
+ *  - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
+ *  @par
+ *     Casts directly and dereferences. This method doesn't depend on the
+ *     compiler, but it violates the C standard as it directly dereferences an
+ *     unaligned pointer. It can generate buggy code on targets which do not
+ *     support unaligned memory accesses, but in some circumstances, it's the
+ *     only known way to get the most performance.
+ *
+ *  - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
+ *  @par
+ *     Also portable. This can generate the best code on old compilers which don't
+ *     inline small `memcpy()` calls, and it might also be faster on big-endian
+ *     systems which lack a native byteswap instruction. However, some compilers
+ *     will emit literal byteshifts even if the target supports unaligned access.
+ *
+ *
+ * @warning
+ *   Methods 1 and 2 rely on implementation-defined behavior. Use these with
+ *   care, as what works on one compiler/platform/optimization level may cause
+ *   another to read garbage data or even crash.
+ *
+ * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
+ *
+ * Prefer these methods in priority order (0 > 3 > 1 > 2)
+ */
+#  define XXH_FORCE_MEMORY_ACCESS 0
+
+/*!
+ * @def XXH_SIZE_OPT
+ * @brief Controls how much xxHash optimizes for size.
+ *
+ * xxHash, when compiled, tends to result in a rather large binary size. This
+ * is mostly due to heavy usage to forced inlining and constant folding of the
+ * @ref XXH3_family to increase performance.
+ *
+ * However, some developers prefer size over speed. This option can
+ * significantly reduce the size of the generated code. When using the `-Os`
+ * or `-Oz` options on GCC or Clang, this is defined to 1 by default,
+ * otherwise it is defined to 0.
+ *
+ * Most of these size optimizations can be controlled manually.
+ *
+ * This is a number from 0-2.
+ *  - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed
+ *    comes first.
+ *  - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more
+ *    conservative and disables hacks that increase code size. It implies the
+ *    options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0,
+ *    and @ref XXH3_NEON_LANES == 8 if they are not already defined.
+ *  - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible.
+ *    Performance may cry. For example, the single shot functions just use the
+ *    streaming API.
+ */
+#  define XXH_SIZE_OPT 0
+
+/*!
+ * @def XXH_FORCE_ALIGN_CHECK
+ * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
+ * and XXH64() only).
+ *
+ * This is an important performance trick for architectures without decent
+ * unaligned memory access performance.
+ *
+ * It checks for input alignment, and when conditions are met, uses a "fast
+ * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
+ * faster_ read speed.
+ *
+ * The check costs one initial branch per hash, which is generally negligible,
+ * but not zero.
+ *
+ * Moreover, it's not useful to generate an additional code path if memory
+ * access uses the same instruction for both aligned and unaligned
+ * addresses (e.g. x86 and aarch64).
+ *
+ * In these cases, the alignment check can be removed by setting this macro to 0.
+ * Then the code will always use unaligned memory access.
+ * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips
+ * which are platforms known to offer good unaligned memory accesses performance.
+ *
+ * It is also disabled by default when @ref XXH_SIZE_OPT >= 1.
+ *
+ * This option does not affect XXH3 (only XXH32 and XXH64).
+ */
+#  define XXH_FORCE_ALIGN_CHECK 0
+
+/*!
+ * @def XXH_NO_INLINE_HINTS
+ * @brief When non-zero, sets all functions to `static`.
+ *
+ * By default, xxHash tries to force the compiler to inline almost all internal
+ * functions.
+ *
+ * This can usually improve performance due to reduced jumping and improved
+ * constant folding, but significantly increases the size of the binary which
+ * might not be favorable.
+ *
+ * Additionally, sometimes the forced inlining can be detrimental to performance,
+ * depending on the architecture.
+ *
+ * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
+ * compiler full control on whether to inline or not.
+ *
+ * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if
+ * @ref XXH_SIZE_OPT >= 1, this will automatically be defined.
+ */
+#  define XXH_NO_INLINE_HINTS 0
+
+/*!
+ * @def XXH3_INLINE_SECRET
+ * @brief Determines whether to inline the XXH3 withSecret code.
+ *
+ * When the secret size is known, the compiler can improve the performance
+ * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret().
+ *
+ * However, if the secret size is not known, it doesn't have any benefit. This
+ * happens when xxHash is compiled into a global symbol. Therefore, if
+ * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0.
+ *
+ * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers
+ * that are *sometimes* force inline on -Og, and it is impossible to automatically
+ * detect this optimization level.
+ */
+#  define XXH3_INLINE_SECRET 0
+
+/*!
+ * @def XXH32_ENDJMP
+ * @brief Whether to use a jump for `XXH32_finalize`.
+ *
+ * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
+ * This is generally preferable for performance,
+ * but depending on exact architecture, a jmp may be preferable.
+ *
+ * This setting is only possibly making a difference for very small inputs.
+ */
+#  define XXH32_ENDJMP 0
+
+/*!
+ * @internal
+ * @brief Redefines old internal names.
+ *
+ * For compatibility with code that uses xxHash's internals before the names
+ * were changed to improve namespacing. There is no other reason to use this.
+ */
+#  define XXH_OLD_NAMES
+#  undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
+
+/*!
+ * @def XXH_NO_STREAM
+ * @brief Disables the streaming API.
+ *
+ * When xxHash is not inlined and the streaming functions are not used, disabling
+ * the streaming functions can improve code size significantly, especially with
+ * the @ref XXH3_family which tends to make constant folded copies of itself.
+ */
+#  define XXH_NO_STREAM
+#  undef XXH_NO_STREAM /* don't actually */
+#endif /* XXH_DOXYGEN */
+/*!
+ * @}
+ */
+
+#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+   /* prefer __packed__ structures (method 1) for GCC
+    * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy
+    * which for some reason does unaligned loads. */
+#  if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED))
+#    define XXH_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+#ifndef XXH_SIZE_OPT
+   /* default to 1 for -Os or -Oz */
+#  if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__)
+#    define XXH_SIZE_OPT 1
+#  else
+#    define XXH_SIZE_OPT 0
+#  endif
+#endif
+
+#ifndef XXH_FORCE_ALIGN_CHECK  /* can be defined externally */
+   /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */
+#  if XXH_SIZE_OPT >= 1 || \
+      defined(__i386)  || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \
+   || defined(_M_IX86) || defined(_M_X64)     || defined(_M_ARM64)    || defined(_M_ARM) /* visual */
+#    define XXH_FORCE_ALIGN_CHECK 0
+#  else
+#    define XXH_FORCE_ALIGN_CHECK 1
+#  endif
+#endif
+
+#ifndef XXH_NO_INLINE_HINTS
+#  if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__)  /* -O0, -fno-inline */
+#    define XXH_NO_INLINE_HINTS 1
+#  else
+#    define XXH_NO_INLINE_HINTS 0
+#  endif
+#endif
+
+#ifndef XXH3_INLINE_SECRET
+#  if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \
+     || !defined(XXH_INLINE_ALL)
+#    define XXH3_INLINE_SECRET 0
+#  else
+#    define XXH3_INLINE_SECRET 1
+#  endif
+#endif
+
+#ifndef XXH32_ENDJMP
+/* generally preferable for performance */
+#  define XXH32_ENDJMP 0
+#endif
+
+/*!
+ * @defgroup impl Implementation
+ * @{
+ */
+
+
+/* *************************************
+*  Includes & Memory related functions
+***************************************/
+#if defined(XXH_NO_STREAM)
+/* nothing */
+#elif defined(XXH_NO_STDLIB)
+
+/* When requesting to disable any mention of stdlib,
+ * the library loses the ability to invoked malloc / free.
+ * In practice, it means that functions like `XXH*_createState()`
+ * will always fail, and return NULL.
+ * This flag is useful in situations where
+ * xxhash.h is integrated into some kernel, embedded or limited environment
+ * without access to dynamic allocation.
+ */
+
+static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
+static void XXH_free(void* p) { (void)p; }
+
+#else
+
+/*
+ * Modify the local functions below should you wish to use
+ * different memory routines for malloc() and free()
+ */
+#include <stdlib.h>
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than malloc().
+ */
+static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than free().
+ */
+static void XXH_free(void* p) { free(p); }
+
+#endif  /* XXH_NO_STDLIB */
+
+#include <string.h>
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than memcpy().
+ */
+static void* XXH_memcpy(void* dest, const void* src, size_t size)
+{
+    return memcpy(dest,src,size);
+}
+
+#include <limits.h>   /* ULLONG_MAX */
+
+
+/* *************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER /* Visual Studio warning fix */
+#  pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+#if XXH_NO_INLINE_HINTS  /* disable inlining hints */
+#  if defined(__GNUC__) || defined(__clang__)
+#    define XXH_FORCE_INLINE static __attribute__((unused))
+#  else
+#    define XXH_FORCE_INLINE static
+#  endif
+#  define XXH_NO_INLINE static
+/* enable inlining hints */
+#elif defined(__GNUC__) || defined(__clang__)
+#  define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
+#  define XXH_NO_INLINE static __attribute__((noinline))
+#elif defined(_MSC_VER)  /* Visual Studio */
+#  define XXH_FORCE_INLINE static __forceinline
+#  define XXH_NO_INLINE static __declspec(noinline)
+#elif defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))   /* C99 */
+#  define XXH_FORCE_INLINE static inline
+#  define XXH_NO_INLINE static
+#else
+#  define XXH_FORCE_INLINE static
+#  define XXH_NO_INLINE static
+#endif
+
+#if XXH3_INLINE_SECRET
+#  define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE
+#else
+#  define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
+#endif
+
+
+/* *************************************
+*  Debug
+***************************************/
+/*!
+ * @ingroup tuning
+ * @def XXH_DEBUGLEVEL
+ * @brief Sets the debugging level.
+ *
+ * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
+ * compiler's command line options. The value must be a number.
+ */
+#ifndef XXH_DEBUGLEVEL
+#  ifdef DEBUGLEVEL /* backwards compat */
+#    define XXH_DEBUGLEVEL DEBUGLEVEL
+#  else
+#    define XXH_DEBUGLEVEL 0
+#  endif
+#endif
+
+#if (XXH_DEBUGLEVEL>=1)
+#  include <assert.h>   /* note: can still be disabled with NDEBUG */
+#  define XXH_ASSERT(c)   assert(c)
+#else
+#  if defined(__INTEL_COMPILER)
+#    define XXH_ASSERT(c)   XXH_ASSUME((unsigned char) (c))
+#  else
+#    define XXH_ASSERT(c)   XXH_ASSUME(c)
+#  endif
+#endif
+
+/* note: use after variable declarations */
+#ifndef XXH_STATIC_ASSERT
+#  if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)    /* C11 */
+#    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
+#  elif defined(__cplusplus) && (__cplusplus >= 201103L)            /* C++11 */
+#    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
+#  else
+#    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
+#  endif
+#  define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
+#endif
+
+/*!
+ * @internal
+ * @def XXH_COMPILER_GUARD(var)
+ * @brief Used to prevent unwanted optimizations for @p var.
+ *
+ * It uses an empty GCC inline assembly statement with a register constraint
+ * which forces @p var into a general purpose register (eg eax, ebx, ecx
+ * on x86) and marks it as modified.
+ *
+ * This is used in a few places to avoid unwanted autovectorization (e.g.
+ * XXH32_round()). All vectorization we want is explicit via intrinsics,
+ * and _usually_ isn't wanted elsewhere.
+ *
+ * We also use it to prevent unwanted constant folding for AArch64 in
+ * XXH3_initCustomSecret_scalar().
+ */
+#if defined(__GNUC__) || defined(__clang__)
+#  define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var))
+#else
+#  define XXH_COMPILER_GUARD(var) ((void)0)
+#endif
+
+/* Specifically for NEON vectors which use the "w" constraint, on
+ * Clang. */
+#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__)
+#  define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var))
+#else
+#  define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0)
+#endif
+
+/* *************************************
+*  Basic Types
+***************************************/
+#if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# ifdef _AIX
+#   include <inttypes.h>
+# else
+#   include <stdint.h>
+# endif
+  typedef uint8_t xxh_u8;
+#else
+  typedef unsigned char xxh_u8;
+#endif
+typedef XXH32_hash_t xxh_u32;
+
+#ifdef XXH_OLD_NAMES
+#  warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly"
+#  define BYTE xxh_u8
+#  define U8   xxh_u8
+#  define U32  xxh_u32
+#endif
+
+/* ***   Memory access   *** */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_read32(const void* ptr)
+ * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit native endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readLE32(const void* ptr)
+ * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readBE32(const void* ptr)
+ * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit big endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
+ * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
+ * always @ref XXH_alignment::XXH_unaligned.
+ *
+ * @param ptr The pointer to read from.
+ * @param align Whether @p ptr is aligned.
+ * @pre
+ *   If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
+ *   aligned.
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
+ */
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE32 and XXH_readBE32.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/*
+ * Force direct memory access. Only works on CPU which support unaligned memory
+ * access in hardware.
+ */
+static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
+ * documentation claimed that it only increased the alignment, but actually it
+ * can decrease it on gcc, clang, and icc:
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
+ * https://gcc.godbolt.org/z/xYez1j67Y.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
+#endif
+static xxh_u32 XXH_read32(const void* ptr)
+{
+    typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32;
+    return *((const xxh_unalign32*)ptr);
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ */
+static xxh_u32 XXH_read32(const void* memPtr)
+{
+    xxh_u32 val;
+    XXH_memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ***   Endianness   *** */
+
+/*!
+ * @ingroup tuning
+ * @def XXH_CPU_LITTLE_ENDIAN
+ * @brief Whether the target is little endian.
+ *
+ * Defined to 1 if the target is little endian, or 0 if it is big endian.
+ * It can be defined externally, for example on the compiler command line.
+ *
+ * If it is not defined,
+ * a runtime check (which is usually constant folded) is used instead.
+ *
+ * @note
+ *   This is not necessarily defined to an integer constant.
+ *
+ * @see XXH_isLittleEndian() for the runtime check.
+ */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+/*
+ * Try to detect endianness automatically, to avoid the nonstandard behavior
+ * in `XXH_isLittleEndian()`
+ */
+#  if defined(_WIN32) /* Windows is always little endian */ \
+     || defined(__LITTLE_ENDIAN__) \
+     || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#    define XXH_CPU_LITTLE_ENDIAN 1
+#  elif defined(__BIG_ENDIAN__) \
+     || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#    define XXH_CPU_LITTLE_ENDIAN 0
+#  else
+/*!
+ * @internal
+ * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
+ *
+ * Most compilers will constant fold this.
+ */
+static int XXH_isLittleEndian(void)
+{
+    /*
+     * Portable and well-defined behavior.
+     * Don't use static: it is detrimental to performance.
+     */
+    const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
+    return one.c[0];
+}
+#   define XXH_CPU_LITTLE_ENDIAN   XXH_isLittleEndian()
+#  endif
+#endif
+
+
+
+
+/* ****************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#ifdef __has_builtin
+#  define XXH_HAS_BUILTIN(x) __has_builtin(x)
+#else
+#  define XXH_HAS_BUILTIN(x) 0
+#endif
+
+
+
+/*
+ * C23 and future versions have standard "unreachable()".
+ * Once it has been implemented reliably we can add it as an
+ * additional case:
+ *
+ * ```
+ * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN)
+ * #  include <stddef.h>
+ * #  ifdef unreachable
+ * #    define XXH_UNREACHABLE() unreachable()
+ * #  endif
+ * #endif
+ * ```
+ *
+ * Note C++23 also has std::unreachable() which can be detected
+ * as follows:
+ * ```
+ * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L)
+ * #  include <utility>
+ * #  define XXH_UNREACHABLE() std::unreachable()
+ * #endif
+ * ```
+ * NB: `__cpp_lib_unreachable` is defined in the `<version>` header.
+ * We don't use that as including `<utility>` in `extern "C"` blocks
+ * doesn't work on GCC12
+ */
+
+#if XXH_HAS_BUILTIN(__builtin_unreachable)
+#  define XXH_UNREACHABLE() __builtin_unreachable()
+
+#elif defined(_MSC_VER)
+#  define XXH_UNREACHABLE() __assume(0)
+
+#else
+#  define XXH_UNREACHABLE()
+#endif
+
+#if XXH_HAS_BUILTIN(__builtin_assume)
+#  define XXH_ASSUME(c) __builtin_assume(c)
+#else
+#  define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); }
+#endif
+
+/*!
+ * @internal
+ * @def XXH_rotl32(x,r)
+ * @brief 32-bit rotate left.
+ *
+ * @param x The 32-bit integer to be rotated.
+ * @param r The number of bits to rotate.
+ * @pre
+ *   @p r > 0 && @p r < 32
+ * @note
+ *   @p x and @p r may be evaluated multiple times.
+ * @return The rotated result.
+ */
+#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
+                               && XXH_HAS_BUILTIN(__builtin_rotateleft64)
+#  define XXH_rotl32 __builtin_rotateleft32
+#  define XXH_rotl64 __builtin_rotateleft64
+/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
+#elif defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
+#  define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
+#endif
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_swap32(xxh_u32 x)
+ * @brief A 32-bit byteswap.
+ *
+ * @param x The 32-bit integer to byteswap.
+ * @return @p x, byteswapped.
+ */
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#else
+static xxh_u32 XXH_swap32 (xxh_u32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* ***************************
+*  Memory reads
+*****************************/
+
+/*!
+ * @internal
+ * @brief Enum to indicate whether a pointer is aligned.
+ */
+typedef enum {
+    XXH_aligned,  /*!< Aligned */
+    XXH_unaligned /*!< Possibly unaligned */
+} XXH_alignment;
+
+/*
+ * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
+ *
+ * This is ideal for older compilers which don't inline memcpy.
+ */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[0]
+         | ((xxh_u32)bytePtr[1] << 8)
+         | ((xxh_u32)bytePtr[2] << 16)
+         | ((xxh_u32)bytePtr[3] << 24);
+}
+
+XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[3]
+         | ((xxh_u32)bytePtr[2] << 8)
+         | ((xxh_u32)bytePtr[1] << 16)
+         | ((xxh_u32)bytePtr[0] << 24);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+}
+
+static xxh_u32 XXH_readBE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u32
+XXH_readLE32_align(const void* ptr, XXH_alignment align)
+{
+    if (align==XXH_unaligned) {
+        return XXH_readLE32(ptr);
+    } else {
+        return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
+    }
+}
+
+
+/* *************************************
+*  Misc
+***************************************/
+/*! @ingroup public */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+*  32-bit hash functions
+*********************************************************************/
+/*!
+ * @}
+ * @defgroup XXH32_impl XXH32 implementation
+ * @ingroup impl
+ *
+ * Details on the XXH32 implementation.
+ * @{
+ */
+ /* #define instead of static const, to be used as initializers */
+#define XXH_PRIME32_1  0x9E3779B1U  /*!< 0b10011110001101110111100110110001 */
+#define XXH_PRIME32_2  0x85EBCA77U  /*!< 0b10000101111010111100101001110111 */
+#define XXH_PRIME32_3  0xC2B2AE3DU  /*!< 0b11000010101100101010111000111101 */
+#define XXH_PRIME32_4  0x27D4EB2FU  /*!< 0b00100111110101001110101100101111 */
+#define XXH_PRIME32_5  0x165667B1U  /*!< 0b00010110010101100110011110110001 */
+
+#ifdef XXH_OLD_NAMES
+#  define PRIME32_1 XXH_PRIME32_1
+#  define PRIME32_2 XXH_PRIME32_2
+#  define PRIME32_3 XXH_PRIME32_3
+#  define PRIME32_4 XXH_PRIME32_4
+#  define PRIME32_5 XXH_PRIME32_5
+#endif
+
+/*!
+ * @internal
+ * @brief Normal stripe processing routine.
+ *
+ * This shuffles the bits so that any bit from @p input impacts several bits in
+ * @p acc.
+ *
+ * @param acc The accumulator lane.
+ * @param input The stripe of input to mix.
+ * @return The mixed accumulator lane.
+ */
+static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
+{
+    acc += input * XXH_PRIME32_2;
+    acc  = XXH_rotl32(acc, 13);
+    acc *= XXH_PRIME32_1;
+#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
+    /*
+     * UGLY HACK:
+     * A compiler fence is the only thing that prevents GCC and Clang from
+     * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
+     * reason) without globally disabling SSE4.1.
+     *
+     * The reason we want to avoid vectorization is because despite working on
+     * 4 integers at a time, there are multiple factors slowing XXH32 down on
+     * SSE4:
+     * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
+     *   newer chips!) making it slightly slower to multiply four integers at
+     *   once compared to four integers independently. Even when pmulld was
+     *   fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
+     *   just to multiply unless doing a long operation.
+     *
+     * - Four instructions are required to rotate,
+     *      movqda tmp,  v // not required with VEX encoding
+     *      pslld  tmp, 13 // tmp <<= 13
+     *      psrld  v,   19 // x >>= 19
+     *      por    v,  tmp // x |= tmp
+     *   compared to one for scalar:
+     *      roll   v, 13    // reliably fast across the board
+     *      shldl  v, v, 13 // Sandy Bridge and later prefer this for some reason
+     *
+     * - Instruction level parallelism is actually more beneficial here because
+     *   the SIMD actually serializes this operation: While v1 is rotating, v2
+     *   can load data, while v3 can multiply. SSE forces them to operate
+     *   together.
+     *
+     * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing
+     * the loop. NEON is only faster on the A53, and with the newer cores, it is less
+     * than half the speed.
+     *
+     * Additionally, this is used on WASM SIMD128 because it JITs to the same
+     * SIMD instructions and has the same issue.
+     */
+    XXH_COMPILER_GUARD(acc);
+#endif
+    return acc;
+}
+
+/*!
+ * @internal
+ * @brief Mixes all bits to finalize the hash.
+ *
+ * The final mix ensures that all input bits have a chance to impact any bit in
+ * the output digest, resulting in an unbiased distribution.
+ *
+ * @param hash The hash to avalanche.
+ * @return The avalanched hash.
+ */
+static xxh_u32 XXH32_avalanche(xxh_u32 hash)
+{
+    hash ^= hash >> 15;
+    hash *= XXH_PRIME32_2;
+    hash ^= hash >> 13;
+    hash *= XXH_PRIME32_3;
+    hash ^= hash >> 16;
+    return hash;
+}
+
+#define XXH_get32bits(p) XXH_readLE32_align(p, align)
+
+/*!
+ * @internal
+ * @brief Processes the last 0-15 bytes of @p ptr.
+ *
+ * There may be up to 15 bytes remaining to consume from the input.
+ * This final stage will digest them to ensure that all input bytes are present
+ * in the final mix.
+ *
+ * @param hash The hash to finalize.
+ * @param ptr The pointer to the remaining input.
+ * @param len The remaining length, modulo 16.
+ * @param align Whether @p ptr is aligned.
+ * @return The finalized hash.
+ * @see XXH64_finalize().
+ */
+static XXH_PUREF xxh_u32
+XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+#define XXH_PROCESS1 do {                             \
+    hash += (*ptr++) * XXH_PRIME32_5;                 \
+    hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1;      \
+} while (0)
+
+#define XXH_PROCESS4 do {                             \
+    hash += XXH_get32bits(ptr) * XXH_PRIME32_3;       \
+    ptr += 4;                                         \
+    hash  = XXH_rotl32(hash, 17) * XXH_PRIME32_4;     \
+} while (0)
+
+    if (ptr==NULL) XXH_ASSERT(len == 0);
+
+    /* Compact rerolled version; generally faster */
+    if (!XXH32_ENDJMP) {
+        len &= 15;
+        while (len >= 4) {
+            XXH_PROCESS4;
+            len -= 4;
+        }
+        while (len > 0) {
+            XXH_PROCESS1;
+            --len;
+        }
+        return XXH32_avalanche(hash);
+    } else {
+         switch(len&15) /* or switch(bEnd - p) */ {
+           case 12:      XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 8:       XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 4:       XXH_PROCESS4;
+                         return XXH32_avalanche(hash);
+
+           case 13:      XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 9:       XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 5:       XXH_PROCESS4;
+                         XXH_PROCESS1;
+                         return XXH32_avalanche(hash);
+
+           case 14:      XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 10:      XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 6:       XXH_PROCESS4;
+                         XXH_PROCESS1;
+                         XXH_PROCESS1;
+                         return XXH32_avalanche(hash);
+
+           case 15:      XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 11:      XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 7:       XXH_PROCESS4;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 3:       XXH_PROCESS1;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 2:       XXH_PROCESS1;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 1:       XXH_PROCESS1;
+                         XXH_FALLTHROUGH;  /* fallthrough */
+           case 0:       return XXH32_avalanche(hash);
+        }
+        XXH_ASSERT(0);
+        return hash;   /* reaching this point is deemed impossible */
+    }
+}
+
+#ifdef XXH_OLD_NAMES
+#  define PROCESS1 XXH_PROCESS1
+#  define PROCESS4 XXH_PROCESS4
+#else
+#  undef XXH_PROCESS1
+#  undef XXH_PROCESS4
+#endif
+
+/*!
+ * @internal
+ * @brief The implementation for @ref XXH32().
+ *
+ * @param input , len , seed Directly passed from @ref XXH32().
+ * @param align Whether @p input is aligned.
+ * @return The calculated hash.
+ */
+XXH_FORCE_INLINE XXH_PUREF xxh_u32
+XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
+{
+    xxh_u32 h32;
+
+    if (input==NULL) XXH_ASSERT(len == 0);
+
+    if (len>=16) {
+        const xxh_u8* const bEnd = input + len;
+        const xxh_u8* const limit = bEnd - 15;
+        xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+        xxh_u32 v2 = seed + XXH_PRIME32_2;
+        xxh_u32 v3 = seed + 0;
+        xxh_u32 v4 = seed - XXH_PRIME32_1;
+
+        do {
+            v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
+            v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
+            v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
+            v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
+        } while (input < limit);
+
+        h32 = XXH_rotl32(v1, 1)  + XXH_rotl32(v2, 7)
+            + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    } else {
+        h32  = seed + XXH_PRIME32_5;
+    }
+
+    h32 += (xxh_u32)len;
+
+    return XXH32_finalize(h32, input, len&15, align);
+}
+
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
+{
+#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, (const xxh_u8*)input, len);
+    return XXH32_digest(&state);
+#else
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
+            return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+    }   }
+
+    return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+#endif
+}
+
+
+
+/*******   Hash streaming   *******/
+#ifndef XXH_NO_STREAM
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+    XXH_memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
+{
+    XXH_ASSERT(statePtr != NULL);
+    memset(statePtr, 0, sizeof(*statePtr));
+    statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+    statePtr->v[1] = seed + XXH_PRIME32_2;
+    statePtr->v[2] = seed + 0;
+    statePtr->v[3] = seed - XXH_PRIME32_1;
+    return XXH_OK;
+}
+
+
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH32_update(XXH32_state_t* state, const void* input, size_t len)
+{
+    if (input==NULL) {
+        XXH_ASSERT(len == 0);
+        return XXH_OK;
+    }
+
+    {   const xxh_u8* p = (const xxh_u8*)input;
+        const xxh_u8* const bEnd = p + len;
+
+        state->total_len_32 += (XXH32_hash_t)len;
+        state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
+
+        if (state->memsize + len < 16)  {   /* fill in tmp buffer */
+            XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
+            state->memsize += (XXH32_hash_t)len;
+            return XXH_OK;
+        }
+
+        if (state->memsize) {   /* some data left from previous update */
+            XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
+            {   const xxh_u32* p32 = state->mem32;
+                state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
+                state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
+                state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
+                state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
+            }
+            p += 16-state->memsize;
+            state->memsize = 0;
+        }
+
+        if (p <= bEnd-16) {
+            const xxh_u8* const limit = bEnd - 16;
+
+            do {
+                state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
+                state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
+                state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
+                state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
+            } while (p<=limit);
+
+        }
+
+        if (p < bEnd) {
+            XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+            state->memsize = (unsigned)(bEnd-p);
+        }
+    }
+
+    return XXH_OK;
+}
+
+
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
+{
+    xxh_u32 h32;
+
+    if (state->large_len) {
+        h32 = XXH_rotl32(state->v[0], 1)
+            + XXH_rotl32(state->v[1], 7)
+            + XXH_rotl32(state->v[2], 12)
+            + XXH_rotl32(state->v[3], 18);
+    } else {
+        h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
+    }
+
+    h32 += state->total_len_32;
+
+    return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
+}
+#endif /* !XXH_NO_STREAM */
+
+/*******   Canonical representation   *******/
+
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+    XXH_memcpy(dst, &hash, sizeof(*dst));
+}
+/*! @ingroup XXH32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+    return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+*  64-bit hash functions
+*********************************************************************/
+/*!
+ * @}
+ * @ingroup impl
+ * @{
+ */
+/*******   Memory access   *******/
+
+typedef XXH64_hash_t xxh_u64;
+
+#ifdef XXH_OLD_NAMES
+#  define U64 xxh_u64
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE64 and XXH_readBE64.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static xxh_u64 XXH_read64(const void* memPtr)
+{
+    return *(const xxh_u64*) memPtr;
+}
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
+ * documentation claimed that it only increased the alignment, but actually it
+ * can decrease it on gcc, clang, and icc:
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
+ * https://gcc.godbolt.org/z/xYez1j67Y.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
+#endif
+static xxh_u64 XXH_read64(const void* ptr)
+{
+    typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64;
+    return *((const xxh_unalign64*)ptr);
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ */
+static xxh_u64 XXH_read64(const void* memPtr)
+{
+    xxh_u64 val;
+    XXH_memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap64 _byteswap_uint64
+#elif XXH_GCC_VERSION >= 403
+#  define XXH_swap64 __builtin_bswap64
+#else
+static xxh_u64 XXH_swap64(xxh_u64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+
+/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[0]
+         | ((xxh_u64)bytePtr[1] << 8)
+         | ((xxh_u64)bytePtr[2] << 16)
+         | ((xxh_u64)bytePtr[3] << 24)
+         | ((xxh_u64)bytePtr[4] << 32)
+         | ((xxh_u64)bytePtr[5] << 40)
+         | ((xxh_u64)bytePtr[6] << 48)
+         | ((xxh_u64)bytePtr[7] << 56);
+}
+
+XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
+{
+    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+    return bytePtr[7]
+         | ((xxh_u64)bytePtr[6] << 8)
+         | ((xxh_u64)bytePtr[5] << 16)
+         | ((xxh_u64)bytePtr[4] << 24)
+         | ((xxh_u64)bytePtr[3] << 32)
+         | ((xxh_u64)bytePtr[2] << 40)
+         | ((xxh_u64)bytePtr[1] << 48)
+         | ((xxh_u64)bytePtr[0] << 56);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+}
+
+static xxh_u64 XXH_readBE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u64
+XXH_readLE64_align(const void* ptr, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return XXH_readLE64(ptr);
+    else
+        return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
+}
+
+
+/*******   xxh64   *******/
+/*!
+ * @}
+ * @defgroup XXH64_impl XXH64 implementation
+ * @ingroup impl
+ *
+ * Details on the XXH64 implementation.
+ * @{
+ */
+/* #define rather that static const, to be used as initializers */
+#define XXH_PRIME64_1  0x9E3779B185EBCA87ULL  /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
+#define XXH_PRIME64_2  0xC2B2AE3D27D4EB4FULL  /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
+#define XXH_PRIME64_3  0x165667B19E3779F9ULL  /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
+#define XXH_PRIME64_4  0x85EBCA77C2B2AE63ULL  /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
+#define XXH_PRIME64_5  0x27D4EB2F165667C5ULL  /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
+
+#ifdef XXH_OLD_NAMES
+#  define PRIME64_1 XXH_PRIME64_1
+#  define PRIME64_2 XXH_PRIME64_2
+#  define PRIME64_3 XXH_PRIME64_3
+#  define PRIME64_4 XXH_PRIME64_4
+#  define PRIME64_5 XXH_PRIME64_5
+#endif
+
+/*! @copydoc XXH32_round */
+static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
+{
+    acc += input * XXH_PRIME64_2;
+    acc  = XXH_rotl64(acc, 31);
+    acc *= XXH_PRIME64_1;
+#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
+    /*
+     * DISABLE AUTOVECTORIZATION:
+     * A compiler fence is used to prevent GCC and Clang from
+     * autovectorizing the XXH64 loop (pragmas and attributes don't work for some
+     * reason) without globally disabling AVX512.
+     *
+     * Autovectorization of XXH64 tends to be detrimental,
+     * though the exact outcome may change depending on exact cpu and compiler version.
+     * For information, it has been reported as detrimental for Skylake-X,
+     * but possibly beneficial for Zen4.
+     *
+     * The default is to disable auto-vectorization,
+     * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable.
+     */
+    XXH_COMPILER_GUARD(acc);
+#endif
+    return acc;
+}
+
+static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
+{
+    val  = XXH64_round(0, val);
+    acc ^= val;
+    acc  = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
+    return acc;
+}
+
+/*! @copydoc XXH32_avalanche */
+static xxh_u64 XXH64_avalanche(xxh_u64 hash)
+{
+    hash ^= hash >> 33;
+    hash *= XXH_PRIME64_2;
+    hash ^= hash >> 29;
+    hash *= XXH_PRIME64_3;
+    hash ^= hash >> 32;
+    return hash;
+}
+
+
+#define XXH_get64bits(p) XXH_readLE64_align(p, align)
+
+/*!
+ * @internal
+ * @brief Processes the last 0-31 bytes of @p ptr.
+ *
+ * There may be up to 31 bytes remaining to consume from the input.
+ * This final stage will digest them to ensure that all input bytes are present
+ * in the final mix.
+ *
+ * @param hash The hash to finalize.
+ * @param ptr The pointer to the remaining input.
+ * @param len The remaining length, modulo 32.
+ * @param align Whether @p ptr is aligned.
+ * @return The finalized hash
+ * @see XXH32_finalize().
+ */
+static XXH_PUREF xxh_u64
+XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+    if (ptr==NULL) XXH_ASSERT(len == 0);
+    len &= 31;
+    while (len >= 8) {
+        xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
+        ptr += 8;
+        hash ^= k1;
+        hash  = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
+        len -= 8;
+    }
+    if (len >= 4) {
+        hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
+        ptr += 4;
+        hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
+        len -= 4;
+    }
+    while (len > 0) {
+        hash ^= (*ptr++) * XXH_PRIME64_5;
+        hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1;
+        --len;
+    }
+    return  XXH64_avalanche(hash);
+}
+
+#ifdef XXH_OLD_NAMES
+#  define PROCESS1_64 XXH_PROCESS1_64
+#  define PROCESS4_64 XXH_PROCESS4_64
+#  define PROCESS8_64 XXH_PROCESS8_64
+#else
+#  undef XXH_PROCESS1_64
+#  undef XXH_PROCESS4_64
+#  undef XXH_PROCESS8_64
+#endif
+
+/*!
+ * @internal
+ * @brief The implementation for @ref XXH64().
+ *
+ * @param input , len , seed Directly passed from @ref XXH64().
+ * @param align Whether @p input is aligned.
+ * @return The calculated hash.
+ */
+XXH_FORCE_INLINE XXH_PUREF xxh_u64
+XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
+{
+    xxh_u64 h64;
+    if (input==NULL) XXH_ASSERT(len == 0);
+
+    if (len>=32) {
+        const xxh_u8* const bEnd = input + len;
+        const xxh_u8* const limit = bEnd - 31;
+        xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+        xxh_u64 v2 = seed + XXH_PRIME64_2;
+        xxh_u64 v3 = seed + 0;
+        xxh_u64 v4 = seed - XXH_PRIME64_1;
+
+        do {
+            v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
+            v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
+            v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
+            v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
+        } while (input<limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+
+    } else {
+        h64  = seed + XXH_PRIME64_5;
+    }
+
+    h64 += (xxh_u64) len;
+
+    return XXH64_finalize(h64, input, len, align);
+}
+
+
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
+{
+#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, (const xxh_u8*)input, len);
+    return XXH64_digest(&state);
+#else
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
+            return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+    }   }
+
+    return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+
+#endif
+}
+
+/*******   Hash Streaming   *******/
+#ifndef XXH_NO_STREAM
+/*! @ingroup XXH64_family*/
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+    XXH_memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed)
+{
+    XXH_ASSERT(statePtr != NULL);
+    memset(statePtr, 0, sizeof(*statePtr));
+    statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+    statePtr->v[1] = seed + XXH_PRIME64_2;
+    statePtr->v[2] = seed + 0;
+    statePtr->v[3] = seed - XXH_PRIME64_1;
+    return XXH_OK;
+}
+
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len)
+{
+    if (input==NULL) {
+        XXH_ASSERT(len == 0);
+        return XXH_OK;
+    }
+
+    {   const xxh_u8* p = (const xxh_u8*)input;
+        const xxh_u8* const bEnd = p + len;
+
+        state->total_len += len;
+
+        if (state->memsize + len < 32) {  /* fill in tmp buffer */
+            XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
+            state->memsize += (xxh_u32)len;
+            return XXH_OK;
+        }
+
+        if (state->memsize) {   /* tmp buffer is full */
+            XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
+            state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
+            state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
+            state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
+            state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
+            p += 32 - state->memsize;
+            state->memsize = 0;
+        }
+
+        if (p+32 <= bEnd) {
+            const xxh_u8* const limit = bEnd - 32;
+
+            do {
+                state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
+                state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
+                state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
+                state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
+            } while (p<=limit);
+
+        }
+
+        if (p < bEnd) {
+            XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+            state->memsize = (unsigned)(bEnd-p);
+        }
+    }
+
+    return XXH_OK;
+}
+
+
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state)
+{
+    xxh_u64 h64;
+
+    if (state->total_len >= 32) {
+        h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
+        h64 = XXH64_mergeRound(h64, state->v[0]);
+        h64 = XXH64_mergeRound(h64, state->v[1]);
+        h64 = XXH64_mergeRound(h64, state->v[2]);
+        h64 = XXH64_mergeRound(h64, state->v[3]);
+    } else {
+        h64  = state->v[2] /*seed*/ + XXH_PRIME64_5;
+    }
+
+    h64 += (xxh_u64) state->total_len;
+
+    return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
+}
+#endif /* !XXH_NO_STREAM */
+
+/******* Canonical representation   *******/
+
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+    XXH_memcpy(dst, &hash, sizeof(*dst));
+}
+
+/*! @ingroup XXH64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src)
+{
+    return XXH_readBE64(src);
+}
+
+#ifndef XXH_NO_XXH3
+
+/* *********************************************************************
+*  XXH3
+*  New generation hash designed for speed on small keys and vectorization
+************************************************************************ */
+/*!
+ * @}
+ * @defgroup XXH3_impl XXH3 implementation
+ * @ingroup impl
+ * @{
+ */
+
+/* ===   Compiler specifics   === */
+
+#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
+#  define XXH_RESTRICT   /* disable */
+#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* >= C99 */
+#  define XXH_RESTRICT   restrict
+#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
+   || (defined (__clang__)) \
+   || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
+   || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
+/*
+ * There are a LOT more compilers that recognize __restrict but this
+ * covers the major ones.
+ */
+#  define XXH_RESTRICT   __restrict
+#else
+#  define XXH_RESTRICT   /* disable */
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 3))  \
+  || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
+  || defined(__clang__)
+#    define XXH_likely(x) __builtin_expect(x, 1)
+#    define XXH_unlikely(x) __builtin_expect(x, 0)
+#else
+#    define XXH_likely(x) (x)
+#    define XXH_unlikely(x) (x)
+#endif
+
+#ifndef XXH_HAS_INCLUDE
+#  ifdef __has_include
+/*
+ * Not defined as XXH_HAS_INCLUDE(x) (function-like) because
+ * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
+ */
+#    define XXH_HAS_INCLUDE __has_include
+#  else
+#    define XXH_HAS_INCLUDE(x) 0
+#  endif
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+#  if defined(__ARM_FEATURE_SVE)
+#    include <arm_sve.h>
+#  endif
+#  if defined(__ARM_NEON__) || defined(__ARM_NEON) \
+   || (defined(_M_ARM) && _M_ARM >= 7) \
+   || defined(_M_ARM64) || defined(_M_ARM64EC) \
+   || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */
+#    define inline __inline__  /* circumvent a clang bug */
+#    include <arm_neon.h>
+#    undef inline
+#  elif defined(__AVX2__)
+#    include <immintrin.h>
+#  elif defined(__SSE2__)
+#    include <emmintrin.h>
+#  endif
+#endif
+
+#if defined(_MSC_VER)
+#  include <intrin.h>
+#endif
+
+/*
+ * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
+ * remaining a true 64-bit/128-bit hash function.
+ *
+ * This is done by prioritizing a subset of 64-bit operations that can be
+ * emulated without too many steps on the average 32-bit machine.
+ *
+ * For example, these two lines seem similar, and run equally fast on 64-bit:
+ *
+ *   xxh_u64 x;
+ *   x ^= (x >> 47); // good
+ *   x ^= (x >> 13); // bad
+ *
+ * However, to a 32-bit machine, there is a major difference.
+ *
+ * x ^= (x >> 47) looks like this:
+ *
+ *   x.lo ^= (x.hi >> (47 - 32));
+ *
+ * while x ^= (x >> 13) looks like this:
+ *
+ *   // note: funnel shifts are not usually cheap.
+ *   x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
+ *   x.hi ^= (x.hi >> 13);
+ *
+ * The first one is significantly faster than the second, simply because the
+ * shift is larger than 32. This means:
+ *  - All the bits we need are in the upper 32 bits, so we can ignore the lower
+ *    32 bits in the shift.
+ *  - The shift result will always fit in the lower 32 bits, and therefore,
+ *    we can ignore the upper 32 bits in the xor.
+ *
+ * Thanks to this optimization, XXH3 only requires these features to be efficient:
+ *
+ *  - Usable unaligned access
+ *  - A 32-bit or 64-bit ALU
+ *      - If 32-bit, a decent ADC instruction
+ *  - A 32 or 64-bit multiply with a 64-bit result
+ *  - For the 128-bit variant, a decent byteswap helps short inputs.
+ *
+ * The first two are already required by XXH32, and almost all 32-bit and 64-bit
+ * platforms which can run XXH32 can run XXH3 efficiently.
+ *
+ * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
+ * notable exception.
+ *
+ * First of all, Thumb-1 lacks support for the UMULL instruction which
+ * performs the important long multiply. This means numerous __aeabi_lmul
+ * calls.
+ *
+ * Second of all, the 8 functional registers are just not enough.
+ * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
+ * Lo registers, and this shuffling results in thousands more MOVs than A32.
+ *
+ * A32 and T32 don't have this limitation. They can access all 14 registers,
+ * do a 32->64 multiply with UMULL, and the flexible operand allowing free
+ * shifts is helpful, too.
+ *
+ * Therefore, we do a quick sanity check.
+ *
+ * If compiling Thumb-1 for a target which supports ARM instructions, we will
+ * emit a warning, as it is not a "sane" platform to compile for.
+ *
+ * Usually, if this happens, it is because of an accident and you probably need
+ * to specify -march, as you likely meant to compile for a newer architecture.
+ *
+ * Credit: large sections of the vectorial and asm source code paths
+ *         have been contributed by @easyaspi314
+ */
+#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
+#   warning "XXH3 is highly inefficient without ARM or Thumb-2."
+#endif
+
+/* ==========================================
+ * Vectorization detection
+ * ========================================== */
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @ingroup tuning
+ * @brief Overrides the vectorization implementation chosen for XXH3.
+ *
+ * Can be defined to 0 to disable SIMD or any of the values mentioned in
+ * @ref XXH_VECTOR_TYPE.
+ *
+ * If this is not defined, it uses predefined macros to determine the best
+ * implementation.
+ */
+#  define XXH_VECTOR XXH_SCALAR
+/*!
+ * @ingroup tuning
+ * @brief Possible values for @ref XXH_VECTOR.
+ *
+ * Note that these are actually implemented as macros.
+ *
+ * If this is not defined, it is detected automatically.
+ * internal macro XXH_X86DISPATCH overrides this.
+ */
+enum XXH_VECTOR_TYPE /* fake enum */ {
+    XXH_SCALAR = 0,  /*!< Portable scalar version */
+    XXH_SSE2   = 1,  /*!<
+                      * SSE2 for Pentium 4, Opteron, all x86_64.
+                      *
+                      * @note SSE2 is also guaranteed on Windows 10, macOS, and
+                      * Android x86.
+                      */
+    XXH_AVX2   = 2,  /*!< AVX2 for Haswell and Bulldozer */
+    XXH_AVX512 = 3,  /*!< AVX512 for Skylake and Icelake */
+    XXH_NEON   = 4,  /*!<
+                       * NEON for most ARMv7-A, all AArch64, and WASM SIMD128
+                       * via the SIMDeverywhere polyfill provided with the
+                       * Emscripten SDK.
+                       */
+    XXH_VSX    = 5,  /*!< VSX and ZVector for POWER8/z13 (64-bit) */
+    XXH_SVE    = 6,  /*!< SVE for some ARMv8-A and ARMv9-A */
+};
+/*!
+ * @ingroup tuning
+ * @brief Selects the minimum alignment for XXH3's accumulators.
+ *
+ * When using SIMD, this should match the alignment required for said vector
+ * type, so, for example, 32 for AVX2.
+ *
+ * Default: Auto detected.
+ */
+#  define XXH_ACC_ALIGN 8
+#endif
+
+/* Actual definition */
+#ifndef XXH_DOXYGEN
+#  define XXH_SCALAR 0
+#  define XXH_SSE2   1
+#  define XXH_AVX2   2
+#  define XXH_AVX512 3
+#  define XXH_NEON   4
+#  define XXH_VSX    5
+#  define XXH_SVE    6
+#endif
+
+#ifndef XXH_VECTOR    /* can be defined on command line */
+#  if defined(__ARM_FEATURE_SVE)
+#    define XXH_VECTOR XXH_SVE
+#  elif ( \
+        defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
+     || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \
+     || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \
+   ) && ( \
+        defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
+    || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
+   )
+#    define XXH_VECTOR XXH_NEON
+#  elif defined(__AVX512F__)
+#    define XXH_VECTOR XXH_AVX512
+#  elif defined(__AVX2__)
+#    define XXH_VECTOR XXH_AVX2
+#  elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
+#    define XXH_VECTOR XXH_SSE2
+#  elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
+     || (defined(__s390x__) && defined(__VEC__)) \
+     && defined(__GNUC__) /* TODO: IBM XL */
+#    define XXH_VECTOR XXH_VSX
+#  else
+#    define XXH_VECTOR XXH_SCALAR
+#  endif
+#endif
+
+/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */
+#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE)
+#  ifdef _MSC_VER
+#    pragma warning(once : 4606)
+#  else
+#    warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead."
+#  endif
+#  undef XXH_VECTOR
+#  define XXH_VECTOR XXH_SCALAR
+#endif
+
+/*
+ * Controls the alignment of the accumulator,
+ * for compatibility with aligned vector loads, which are usually faster.
+ */
+#ifndef XXH_ACC_ALIGN
+#  if defined(XXH_X86DISPATCH)
+#     define XXH_ACC_ALIGN 64  /* for compatibility with avx512 */
+#  elif XXH_VECTOR == XXH_SCALAR  /* scalar */
+#     define XXH_ACC_ALIGN 8
+#  elif XXH_VECTOR == XXH_SSE2  /* sse2 */
+#     define XXH_ACC_ALIGN 16
+#  elif XXH_VECTOR == XXH_AVX2  /* avx2 */
+#     define XXH_ACC_ALIGN 32
+#  elif XXH_VECTOR == XXH_NEON  /* neon */
+#     define XXH_ACC_ALIGN 16
+#  elif XXH_VECTOR == XXH_VSX   /* vsx */
+#     define XXH_ACC_ALIGN 16
+#  elif XXH_VECTOR == XXH_AVX512  /* avx512 */
+#     define XXH_ACC_ALIGN 64
+#  elif XXH_VECTOR == XXH_SVE   /* sve */
+#     define XXH_ACC_ALIGN 64
+#  endif
+#endif
+
+#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
+    || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
+#  define XXH_SEC_ALIGN XXH_ACC_ALIGN
+#elif XXH_VECTOR == XXH_SVE
+#  define XXH_SEC_ALIGN XXH_ACC_ALIGN
+#else
+#  define XXH_SEC_ALIGN 8
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+#  define XXH_ALIASING __attribute__((may_alias))
+#else
+#  define XXH_ALIASING /* nothing */
+#endif
+
+/*
+ * UGLY HACK:
+ * GCC usually generates the best code with -O3 for xxHash.
+ *
+ * However, when targeting AVX2, it is overzealous in its unrolling resulting
+ * in code roughly 3/4 the speed of Clang.
+ *
+ * There are other issues, such as GCC splitting _mm256_loadu_si256 into
+ * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
+ * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
+ *
+ * That is why when compiling the AVX2 version, it is recommended to use either
+ *   -O2 -mavx2 -march=haswell
+ * or
+ *   -O2 -mavx2 -mno-avx256-split-unaligned-load
+ * for decent performance, or to use Clang instead.
+ *
+ * Fortunately, we can control the first one with a pragma that forces GCC into
+ * -O2, but the other one we can't control without "failed to inline always
+ * inline function due to target mismatch" warnings.
+ */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+  && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+  && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
+#  pragma GCC push_options
+#  pragma GCC optimize("-O2")
+#endif
+
+#if XXH_VECTOR == XXH_NEON
+
+/*
+ * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3
+ * optimizes out the entire hashLong loop because of the aliasing violation.
+ *
+ * However, GCC is also inefficient at load-store optimization with vld1q/vst1q,
+ * so the only option is to mark it as aliasing.
+ */
+typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING;
+
+/*!
+ * @internal
+ * @brief `vld1q_u64` but faster and alignment-safe.
+ *
+ * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only
+ * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86).
+ *
+ * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it
+ * prohibits load-store optimizations. Therefore, a direct dereference is used.
+ *
+ * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe
+ * unaligned load.
+ */
+#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)
+XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */
+{
+    return *(xxh_aliasing_uint64x2_t const *)ptr;
+}
+#else
+XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr)
+{
+    return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr));
+}
+#endif
+
+/*!
+ * @internal
+ * @brief `vmlal_u32` on low and high halves of a vector.
+ *
+ * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with
+ * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32`
+ * with `vmlal_u32`.
+ */
+#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11
+XXH_FORCE_INLINE uint64x2_t
+XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
+{
+    /* Inline assembly is the only way */
+    __asm__("umlal   %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs));
+    return acc;
+}
+XXH_FORCE_INLINE uint64x2_t
+XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
+{
+    /* This intrinsic works as expected */
+    return vmlal_high_u32(acc, lhs, rhs);
+}
+#else
+/* Portable intrinsic versions */
+XXH_FORCE_INLINE uint64x2_t
+XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
+{
+    return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
+}
+/*! @copydoc XXH_vmlal_low_u32
+ * Assume the compiler converts this to vmlal_high_u32 on aarch64 */
+XXH_FORCE_INLINE uint64x2_t
+XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
+{
+    return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
+}
+#endif
+
+/*!
+ * @ingroup tuning
+ * @brief Controls the NEON to scalar ratio for XXH3
+ *
+ * This can be set to 2, 4, 6, or 8.
+ *
+ * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used.
+ *
+ * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those
+ * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU
+ * bandwidth.
+ *
+ * This is even more noticeable on the more advanced cores like the Cortex-A76 which
+ * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
+ *
+ * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes
+ * and 2 scalar lanes, which is chosen by default.
+ *
+ * This does not apply to Apple processors or 32-bit processors, which run better with
+ * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes.
+ *
+ * This change benefits CPUs with large micro-op buffers without negatively affecting
+ * most other CPUs:
+ *
+ *  | Chipset               | Dispatch type       | NEON only | 6:2 hybrid | Diff. |
+ *  |:----------------------|:--------------------|----------:|-----------:|------:|
+ *  | Snapdragon 730 (A76)  | 2 NEON/8 micro-ops  |  8.8 GB/s |  10.1 GB/s |  ~16% |
+ *  | Snapdragon 835 (A73)  | 2 NEON/3 micro-ops  |  5.1 GB/s |   5.3 GB/s |   ~5% |
+ *  | Marvell PXA1928 (A53) | In-order dual-issue |  1.9 GB/s |   1.9 GB/s |    0% |
+ *  | Apple M1              | 4 NEON/8 micro-ops  | 37.3 GB/s |  36.1 GB/s |  ~-3% |
+ *
+ * It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
+ *
+ * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning
+ * it effectively becomes worse 4.
+ *
+ * @see XXH3_accumulate_512_neon()
+ */
+# ifndef XXH3_NEON_LANES
+#  if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \
+   && !defined(__APPLE__) && XXH_SIZE_OPT <= 0
+#   define XXH3_NEON_LANES 6
+#  else
+#   define XXH3_NEON_LANES XXH_ACC_NB
+#  endif
+# endif
+#endif  /* XXH_VECTOR == XXH_NEON */
+
+/*
+ * VSX and Z Vector helpers.
+ *
+ * This is very messy, and any pull requests to clean this up are welcome.
+ *
+ * There are a lot of problems with supporting VSX and s390x, due to
+ * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
+ */
+#if XXH_VECTOR == XXH_VSX
+/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`,
+ * and `pixel`. This is a problem for obvious reasons.
+ *
+ * These keywords are unnecessary; the spec literally says they are
+ * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd
+ * after including the header.
+ *
+ * We use pragma push_macro/pop_macro to keep the namespace clean. */
+#  pragma push_macro("bool")
+#  pragma push_macro("vector")
+#  pragma push_macro("pixel")
+/* silence potential macro redefined warnings */
+#  undef bool
+#  undef vector
+#  undef pixel
+
+#  if defined(__s390x__)
+#    include <s390intrin.h>
+#  else
+#    include <altivec.h>
+#  endif
+
+/* Restore the original macro values, if applicable. */
+#  pragma pop_macro("pixel")
+#  pragma pop_macro("vector")
+#  pragma pop_macro("bool")
+
+typedef __vector unsigned long long xxh_u64x2;
+typedef __vector unsigned char xxh_u8x16;
+typedef __vector unsigned xxh_u32x4;
+
+/*
+ * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue.
+ */
+typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;
+
+# ifndef XXH_VSX_BE
+#  if defined(__BIG_ENDIAN__) \
+  || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#    define XXH_VSX_BE 1
+#  elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
+#    warning "-maltivec=be is not recommended. Please use native endianness."
+#    define XXH_VSX_BE 1
+#  else
+#    define XXH_VSX_BE 0
+#  endif
+# endif /* !defined(XXH_VSX_BE) */
+
+# if XXH_VSX_BE
+#  if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
+#    define XXH_vec_revb vec_revb
+#  else
+/*!
+ * A polyfill for POWER9's vec_revb().
+ */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
+{
+    xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+                                  0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
+    return vec_perm(val, val, vByteSwap);
+}
+#  endif
+# endif /* XXH_VSX_BE */
+
+/*!
+ * Performs an unaligned vector load and byte swaps it on big endian.
+ */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
+{
+    xxh_u64x2 ret;
+    XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
+# if XXH_VSX_BE
+    ret = XXH_vec_revb(ret);
+# endif
+    return ret;
+}
+
+/*
+ * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
+ *
+ * These intrinsics weren't added until GCC 8, despite existing for a while,
+ * and they are endian dependent. Also, their meaning swap depending on version.
+ * */
+# if defined(__s390x__)
+ /* s390x is always big endian, no issue on this platform */
+#  define XXH_vec_mulo vec_mulo
+#  define XXH_vec_mule vec_mule
+# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__)
+/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
+ /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */
+#  define XXH_vec_mulo __builtin_altivec_vmulouw
+#  define XXH_vec_mule __builtin_altivec_vmuleuw
+# else
+/* gcc needs inline assembly */
+/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
+{
+    xxh_u64x2 result;
+    __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+    return result;
+}
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
+{
+    xxh_u64x2 result;
+    __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+    return result;
+}
+# endif /* XXH_vec_mulo, XXH_vec_mule */
+#endif /* XXH_VECTOR == XXH_VSX */
+
+#if XXH_VECTOR == XXH_SVE
+#define ACCRND(acc, offset) \
+do { \
+    svuint64_t input_vec = svld1_u64(mask, xinput + offset);         \
+    svuint64_t secret_vec = svld1_u64(mask, xsecret + offset);       \
+    svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec);     \
+    svuint64_t swapped = svtbl_u64(input_vec, kSwap);                \
+    svuint64_t mixed_lo = svextw_u64_x(mask, mixed);                 \
+    svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32);            \
+    svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \
+    acc = svadd_u64_x(mask, acc, mul);                               \
+} while (0)
+#endif /* XXH_VECTOR == XXH_SVE */
+
+/* prefetch
+ * can be disabled, by declaring XXH_NO_PREFETCH build macro */
+#if defined(XXH_NO_PREFETCH)
+#  define XXH_PREFETCH(ptr)  (void)(ptr)  /* disabled */
+#else
+#  if XXH_SIZE_OPT >= 1
+#    define XXH_PREFETCH(ptr) (void)(ptr)
+#  elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))  /* _mm_prefetch() not defined outside of x86/x64 */
+#    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+#    define XXH_PREFETCH(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+#    define XXH_PREFETCH(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#  else
+#    define XXH_PREFETCH(ptr) (void)(ptr)  /* disabled */
+#  endif
+#endif  /* XXH_NO_PREFETCH */
+
+
+/* ==========================================
+ * XXH3 default settings
+ * ========================================== */
+
+#define XXH_SECRET_DEFAULT_SIZE 192   /* minimum XXH3_SECRET_SIZE_MIN */
+
+#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
+#  error "default keyset is not large enough"
+#endif
+
+/*! Pseudorandom secret taken directly from FARSH. */
+XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
+    0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
+    0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
+    0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
+    0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
+    0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
+    0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
+    0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
+    0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
+    0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
+    0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
+    0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
+    0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
+};
+
+static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL;  /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */
+static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL;  /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */
+
+#ifdef XXH_OLD_NAMES
+#  define kSecret XXH3_kSecret
+#endif
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Calculates a 32-bit to 64-bit long multiply.
+ *
+ * Implemented as a macro.
+ *
+ * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
+ * need to (but it shouldn't need to anyways, it is about 7 instructions to do
+ * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
+ * use that instead of the normal method.
+ *
+ * If you are compiling for platforms like Thumb-1 and don't have a better option,
+ * you may also want to write your own long multiply routine here.
+ *
+ * @param x, y Numbers to be multiplied
+ * @return 64-bit product of the low 32 bits of @p x and @p y.
+ */
+XXH_FORCE_INLINE xxh_u64
+XXH_mult32to64(xxh_u64 x, xxh_u64 y)
+{
+   return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
+}
+#elif defined(_MSC_VER) && defined(_M_IX86)
+#    define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
+#else
+/*
+ * Downcast + upcast is usually better than masking on older compilers like
+ * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
+ *
+ * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
+ * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
+ */
+#    define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
+#endif
+
+/*!
+ * @brief Calculates a 64->128-bit long multiply.
+ *
+ * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
+ * version.
+ *
+ * @param lhs , rhs The 64-bit integers to be multiplied
+ * @return The 128-bit result represented in an @ref XXH128_hash_t.
+ */
+static XXH128_hash_t
+XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
+{
+    /*
+     * GCC/Clang __uint128_t method.
+     *
+     * On most 64-bit targets, GCC and Clang define a __uint128_t type.
+     * This is usually the best way as it usually uses a native long 64-bit
+     * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
+     *
+     * Usually.
+     *
+     * Despite being a 32-bit platform, Clang (and emscripten) define this type
+     * despite not having the arithmetic for it. This results in a laggy
+     * compiler builtin call which calculates a full 128-bit multiply.
+     * In that case it is best to use the portable one.
+     * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
+     */
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \
+    && defined(__SIZEOF_INT128__) \
+    || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+
+    __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
+    XXH128_hash_t r128;
+    r128.low64  = (xxh_u64)(product);
+    r128.high64 = (xxh_u64)(product >> 64);
+    return r128;
+
+    /*
+     * MSVC for x64's _umul128 method.
+     *
+     * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
+     *
+     * This compiles to single operand MUL on x64.
+     */
+#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC)
+
+#ifndef _MSC_VER
+#   pragma intrinsic(_umul128)
+#endif
+    xxh_u64 product_high;
+    xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
+    XXH128_hash_t r128;
+    r128.low64  = product_low;
+    r128.high64 = product_high;
+    return r128;
+
+    /*
+     * MSVC for ARM64's __umulh method.
+     *
+     * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
+     */
+#elif defined(_M_ARM64) || defined(_M_ARM64EC)
+
+#ifndef _MSC_VER
+#   pragma intrinsic(__umulh)
+#endif
+    XXH128_hash_t r128;
+    r128.low64  = lhs * rhs;
+    r128.high64 = __umulh(lhs, rhs);
+    return r128;
+
+#else
+    /*
+     * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
+     *
+     * This is a fast and simple grade school multiply, which is shown below
+     * with base 10 arithmetic instead of base 0x100000000.
+     *
+     *           9 3 // D2 lhs = 93
+     *         x 7 5 // D2 rhs = 75
+     *     ----------
+     *           1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
+     *         4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
+     *         2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
+     *     + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
+     *     ---------
+     *         2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
+     *     + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
+     *     ---------
+     *       6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
+     *
+     * The reasons for adding the products like this are:
+     *  1. It avoids manual carry tracking. Just like how
+     *     (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
+     *     This avoids a lot of complexity.
+     *
+     *  2. It hints for, and on Clang, compiles to, the powerful UMAAL
+     *     instruction available in ARM's Digital Signal Processing extension
+     *     in 32-bit ARMv6 and later, which is shown below:
+     *
+     *         void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
+     *         {
+     *             xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
+     *             *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
+     *             *RdHi = (xxh_u32)(product >> 32);
+     *         }
+     *
+     *     This instruction was designed for efficient long multiplication, and
+     *     allows this to be calculated in only 4 instructions at speeds
+     *     comparable to some 64-bit ALUs.
+     *
+     *  3. It isn't terrible on other platforms. Usually this will be a couple
+     *     of 32-bit ADD/ADCs.
+     */
+
+    /* First calculate all of the cross products. */
+    xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
+    xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32,        rhs & 0xFFFFFFFF);
+    xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
+    xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32,        rhs >> 32);
+
+    /* Now add the products together. These will never overflow. */
+    xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
+    xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32)        + hi_hi;
+    xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
+
+    XXH128_hash_t r128;
+    r128.low64  = lower;
+    r128.high64 = upper;
+    return r128;
+#endif
+}
+
+/*!
+ * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
+ *
+ * The reason for the separate function is to prevent passing too many structs
+ * around by value. This will hopefully inline the multiply, but we don't force it.
+ *
+ * @param lhs , rhs The 64-bit integers to multiply
+ * @return The low 64 bits of the product XOR'd by the high 64 bits.
+ * @see XXH_mult64to128()
+ */
+static xxh_u64
+XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
+{
+    XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
+    return product.low64 ^ product.high64;
+}
+
+/*! Seems to produce slightly better code on GCC for some reason. */
+XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
+{
+    XXH_ASSERT(0 <= shift && shift < 64);
+    return v64 ^ (v64 >> shift);
+}
+
+/*
+ * This is a fast avalanche stage,
+ * suitable when input bits are already partially mixed
+ */
+static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
+{
+    h64 = XXH_xorshift64(h64, 37);
+    h64 *= PRIME_MX1;
+    h64 = XXH_xorshift64(h64, 32);
+    return h64;
+}
+
+/*
+ * This is a stronger avalanche,
+ * inspired by Pelle Evensen's rrmxmx
+ * preferable when input has not been previously mixed
+ */
+static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
+{
+    /* this mix is inspired by Pelle Evensen's rrmxmx */
+    h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
+    h64 *= PRIME_MX2;
+    h64 ^= (h64 >> 35) + len ;
+    h64 *= PRIME_MX2;
+    return XXH_xorshift64(h64, 28);
+}
+
+
+/* ==========================================
+ * Short keys
+ * ==========================================
+ * One of the shortcomings of XXH32 and XXH64 was that their performance was
+ * sub-optimal on short lengths. It used an iterative algorithm which strongly
+ * favored lengths that were a multiple of 4 or 8.
+ *
+ * Instead of iterating over individual inputs, we use a set of single shot
+ * functions which piece together a range of lengths and operate in constant time.
+ *
+ * Additionally, the number of multiplies has been significantly reduced. This
+ * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
+ *
+ * Depending on the platform, this may or may not be faster than XXH32, but it
+ * is almost guaranteed to be faster than XXH64.
+ */
+
+/*
+ * At very short lengths, there isn't enough input to fully hide secrets, or use
+ * the entire secret.
+ *
+ * There is also only a limited amount of mixing we can do before significantly
+ * impacting performance.
+ *
+ * Therefore, we use different sections of the secret and always mix two secret
+ * samples with an XOR. This should have no effect on performance on the
+ * seedless or withSeed variants because everything _should_ be constant folded
+ * by modern compilers.
+ *
+ * The XOR mixing hides individual parts of the secret and increases entropy.
+ *
+ * This adds an extra layer of strength for custom secrets.
+ */
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
+XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(1 <= len && len <= 3);
+    XXH_ASSERT(secret != NULL);
+    /*
+     * len = 1: combined = { input[0], 0x01, input[0], input[0] }
+     * len = 2: combined = { input[1], 0x02, input[0], input[1] }
+     * len = 3: combined = { input[2], 0x03, input[0], input[1] }
+     */
+    {   xxh_u8  const c1 = input[0];
+        xxh_u8  const c2 = input[len >> 1];
+        xxh_u8  const c3 = input[len - 1];
+        xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2  << 24)
+                               | ((xxh_u32)c3 <<  0) | ((xxh_u32)len << 8);
+        xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+        xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
+        return XXH64_avalanche(keyed);
+    }
+}
+
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
+XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(4 <= len && len <= 8);
+    seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+    {   xxh_u32 const input1 = XXH_readLE32(input);
+        xxh_u32 const input2 = XXH_readLE32(input + len - 4);
+        xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
+        xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
+        xxh_u64 const keyed = input64 ^ bitflip;
+        return XXH3_rrmxmx(keyed, len);
+    }
+}
+
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
+XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(9 <= len && len <= 16);
+    {   xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
+        xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
+        xxh_u64 const input_lo = XXH_readLE64(input)           ^ bitflip1;
+        xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
+        xxh_u64 const acc = len
+                          + XXH_swap64(input_lo) + input_hi
+                          + XXH3_mul128_fold64(input_lo, input_hi);
+        return XXH3_avalanche(acc);
+    }
+}
+
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
+XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(len <= 16);
+    {   if (XXH_likely(len >  8)) return XXH3_len_9to16_64b(input, len, secret, seed);
+        if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
+        if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
+        return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
+    }
+}
+
+/*
+ * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
+ * multiplication by zero, affecting hashes of lengths 17 to 240.
+ *
+ * However, they are very unlikely.
+ *
+ * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
+ * unseeded non-cryptographic hashes, it does not attempt to defend itself
+ * against specially crafted inputs, only random inputs.
+ *
+ * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
+ * cancelling out the secret is taken an arbitrary number of times (addressed
+ * in XXH3_accumulate_512), this collision is very unlikely with random inputs
+ * and/or proper seeding:
+ *
+ * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
+ * function that is only called up to 16 times per hash with up to 240 bytes of
+ * input.
+ *
+ * This is not too bad for a non-cryptographic hash function, especially with
+ * only 64 bit outputs.
+ *
+ * The 128-bit variant (which trades some speed for strength) is NOT affected
+ * by this, although it is always a good idea to use a proper seed if you care
+ * about strength.
+ */
+XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
+                                     const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
+{
+#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+  && defined(__i386__) && defined(__SSE2__)  /* x86 + SSE2 */ \
+  && !defined(XXH_ENABLE_AUTOVECTORIZE)      /* Define to disable like XXH32 hack */
+    /*
+     * UGLY HACK:
+     * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
+     * slower code.
+     *
+     * By forcing seed64 into a register, we disrupt the cost model and
+     * cause it to scalarize. See `XXH32_round()`
+     *
+     * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
+     * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
+     * GCC 9.2, despite both emitting scalar code.
+     *
+     * GCC generates much better scalar code than Clang for the rest of XXH3,
+     * which is why finding a more optimal codepath is an interest.
+     */
+    XXH_COMPILER_GUARD(seed64);
+#endif
+    {   xxh_u64 const input_lo = XXH_readLE64(input);
+        xxh_u64 const input_hi = XXH_readLE64(input+8);
+        return XXH3_mul128_fold64(
+            input_lo ^ (XXH_readLE64(secret)   + seed64),
+            input_hi ^ (XXH_readLE64(secret+8) - seed64)
+        );
+    }
+}
+
+/* For mid range keys, XXH3 uses a Mum-hash variant. */
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
+XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                     const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                     XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(16 < len && len <= 128);
+
+    {   xxh_u64 acc = len * XXH_PRIME64_1;
+#if XXH_SIZE_OPT >= 1
+        /* Smaller and cleaner, but slightly slower. */
+        unsigned int i = (unsigned int)(len - 1) / 32;
+        do {
+            acc += XXH3_mix16B(input+16 * i, secret+32*i, seed);
+            acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed);
+        } while (i-- != 0);
+#else
+        if (len > 32) {
+            if (len > 64) {
+                if (len > 96) {
+                    acc += XXH3_mix16B(input+48, secret+96, seed);
+                    acc += XXH3_mix16B(input+len-64, secret+112, seed);
+                }
+                acc += XXH3_mix16B(input+32, secret+64, seed);
+                acc += XXH3_mix16B(input+len-48, secret+80, seed);
+            }
+            acc += XXH3_mix16B(input+16, secret+32, seed);
+            acc += XXH3_mix16B(input+len-32, secret+48, seed);
+        }
+        acc += XXH3_mix16B(input+0, secret+0, seed);
+        acc += XXH3_mix16B(input+len-16, secret+16, seed);
+#endif
+        return XXH3_avalanche(acc);
+    }
+}
+
+/*!
+ * @brief Maximum size of "short" key in bytes.
+ */
+#define XXH3_MIDSIZE_MAX 240
+
+XXH_NO_INLINE XXH_PUREF XXH64_hash_t
+XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                      XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+    #define XXH3_MIDSIZE_STARTOFFSET 3
+    #define XXH3_MIDSIZE_LASTOFFSET  17
+
+    {   xxh_u64 acc = len * XXH_PRIME64_1;
+        xxh_u64 acc_end;
+        unsigned int const nbRounds = (unsigned int)len / 16;
+        unsigned int i;
+        XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+        for (i=0; i<8; i++) {
+            acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
+        }
+        /* last bytes */
+        acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
+        XXH_ASSERT(nbRounds >= 8);
+        acc = XXH3_avalanche(acc);
+#if defined(__clang__)                                /* Clang */ \
+    && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
+    && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
+        /*
+         * UGLY HACK:
+         * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
+         * In everywhere else, it uses scalar code.
+         *
+         * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
+         * would still be slower than UMAAL (see XXH_mult64to128).
+         *
+         * Unfortunately, Clang doesn't handle the long multiplies properly and
+         * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
+         * scalarized into an ugly mess of VMOV.32 instructions.
+         *
+         * This mess is difficult to avoid without turning autovectorization
+         * off completely, but they are usually relatively minor and/or not
+         * worth it to fix.
+         *
+         * This loop is the easiest to fix, as unlike XXH32, this pragma
+         * _actually works_ because it is a loop vectorization instead of an
+         * SLP vectorization.
+         */
+        #pragma clang loop vectorize(disable)
+#endif
+        for (i=8 ; i < nbRounds; i++) {
+            /*
+             * Prevents clang for unrolling the acc loop and interleaving with this one.
+             */
+            XXH_COMPILER_GUARD(acc);
+            acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
+        }
+        return XXH3_avalanche(acc + acc_end);
+    }
+}
+
+
+/* =======     Long Keys     ======= */
+
+#define XXH_STRIPE_LEN 64
+#define XXH_SECRET_CONSUME_RATE 8   /* nb of secret bytes consumed at each accumulation */
+#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
+
+#ifdef XXH_OLD_NAMES
+#  define STRIPE_LEN XXH_STRIPE_LEN
+#  define ACC_NB XXH_ACC_NB
+#endif
+
+#ifndef XXH_PREFETCH_DIST
+#  ifdef __clang__
+#    define XXH_PREFETCH_DIST 320
+#  else
+#    if (XXH_VECTOR == XXH_AVX512)
+#      define XXH_PREFETCH_DIST 512
+#    else
+#      define XXH_PREFETCH_DIST 384
+#    endif
+#  endif  /* __clang__ */
+#endif  /* XXH_PREFETCH_DIST */
+
+/*
+ * These macros are to generate an XXH3_accumulate() function.
+ * The two arguments select the name suffix and target attribute.
+ *
+ * The name of this symbol is XXH3_accumulate_<name>() and it calls
+ * XXH3_accumulate_512_<name>().
+ *
+ * It may be useful to hand implement this function if the compiler fails to
+ * optimize the inline function.
+ */
+#define XXH3_ACCUMULATE_TEMPLATE(name)                      \
+void                                                        \
+XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc,           \
+                       const xxh_u8* XXH_RESTRICT input,    \
+                       const xxh_u8* XXH_RESTRICT secret,   \
+                       size_t nbStripes)                    \
+{                                                           \
+    size_t n;                                               \
+    for (n = 0; n < nbStripes; n++ ) {                      \
+        const xxh_u8* const in = input + n*XXH_STRIPE_LEN;  \
+        XXH_PREFETCH(in + XXH_PREFETCH_DIST);               \
+        XXH3_accumulate_512_##name(                         \
+                 acc,                                       \
+                 in,                                        \
+                 secret + n*XXH_SECRET_CONSUME_RATE);       \
+    }                                                       \
+}
+
+
+XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
+{
+    if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
+    XXH_memcpy(dst, &v64, sizeof(v64));
+}
+
+/* Several intrinsic functions below are supposed to accept __int64 as argument,
+ * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
+ * However, several environments do not define __int64 type,
+ * requiring a workaround.
+ */
+#if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+    typedef int64_t xxh_i64;
+#else
+    /* the following type must have a width of 64-bit */
+    typedef long long xxh_i64;
+#endif
+
+
+/*
+ * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
+ *
+ * It is a hardened version of UMAC, based off of FARSH's implementation.
+ *
+ * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
+ * implementations, and it is ridiculously fast.
+ *
+ * We harden it by mixing the original input to the accumulators as well as the product.
+ *
+ * This means that in the (relatively likely) case of a multiply by zero, the
+ * original input is preserved.
+ *
+ * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
+ * cross-pollination, as otherwise the upper and lower halves would be
+ * essentially independent.
+ *
+ * This doesn't matter on 64-bit hashes since they all get merged together in
+ * the end, so we skip the extra step.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+#if (XXH_VECTOR == XXH_AVX512) \
+     || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
+
+#ifndef XXH_TARGET_AVX512
+# define XXH_TARGET_AVX512  /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
+                     const void* XXH_RESTRICT input,
+                     const void* XXH_RESTRICT secret)
+{
+    __m512i* const xacc = (__m512i *) acc;
+    XXH_ASSERT((((size_t)acc) & 63) == 0);
+    XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+
+    {
+        /* data_vec    = input[0]; */
+        __m512i const data_vec    = _mm512_loadu_si512   (input);
+        /* key_vec     = secret[0]; */
+        __m512i const key_vec     = _mm512_loadu_si512   (secret);
+        /* data_key    = data_vec ^ key_vec; */
+        __m512i const data_key    = _mm512_xor_si512     (data_vec, key_vec);
+        /* data_key_lo = data_key >> 32; */
+        __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32);
+        /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+        __m512i const product     = _mm512_mul_epu32     (data_key, data_key_lo);
+        /* xacc[0] += swap(data_vec); */
+        __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
+        __m512i const sum       = _mm512_add_epi64(*xacc, data_swap);
+        /* xacc[0] += product; */
+        *xacc = _mm512_add_epi64(product, sum);
+    }
+}
+XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512)
+
+/*
+ * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
+ *
+ * Multiplication isn't perfect, as explained by Google in HighwayHash:
+ *
+ *  // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
+ *  // varying degrees. In descending order of goodness, bytes
+ *  // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
+ *  // As expected, the upper and lower bytes are much worse.
+ *
+ * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
+ *
+ * Since our algorithm uses a pseudorandom secret to add some variance into the
+ * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
+ *
+ * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
+ * extraction.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 63) == 0);
+    XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+    {   __m512i* const xacc = (__m512i*) acc;
+        const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
+
+        /* xacc[0] ^= (xacc[0] >> 47) */
+        __m512i const acc_vec     = *xacc;
+        __m512i const shifted     = _mm512_srli_epi64    (acc_vec, 47);
+        /* xacc[0] ^= secret; */
+        __m512i const key_vec     = _mm512_loadu_si512   (secret);
+        __m512i const data_key    = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */);
+
+        /* xacc[0] *= XXH_PRIME32_1; */
+        __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32);
+        __m512i const prod_lo     = _mm512_mul_epu32     (data_key, prime32);
+        __m512i const prod_hi     = _mm512_mul_epu32     (data_key_hi, prime32);
+        *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
+    }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
+    XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
+    XXH_ASSERT(((size_t)customSecret & 63) == 0);
+    (void)(&XXH_writeLE64);
+    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
+        __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64);
+        __m512i const seed     = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos);
+
+        const __m512i* const src  = (const __m512i*) ((const void*) XXH3_kSecret);
+              __m512i* const dest = (      __m512i*) customSecret;
+        int i;
+        XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
+        XXH_ASSERT(((size_t)dest & 63) == 0);
+        for (i=0; i < nbRounds; ++i) {
+            dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed);
+    }   }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_AVX2) \
+    || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
+
+#ifndef XXH_TARGET_AVX2
+# define XXH_TARGET_AVX2  /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 31) == 0);
+    {   __m256i* const xacc    =       (__m256i *) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm256_loadu_si256 requires  a const __m256i * pointer for some reason. */
+        const         __m256i* const xinput  = (const __m256i *) input;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+        const         __m256i* const xsecret = (const __m256i *) secret;
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+            /* data_vec    = xinput[i]; */
+            __m256i const data_vec    = _mm256_loadu_si256    (xinput+i);
+            /* key_vec     = xsecret[i]; */
+            __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
+            /* data_key    = data_vec ^ key_vec; */
+            __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);
+            /* data_key_lo = data_key >> 32; */
+            __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32);
+            /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+            __m256i const product     = _mm256_mul_epu32     (data_key, data_key_lo);
+            /* xacc[i] += swap(data_vec); */
+            __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
+            __m256i const sum       = _mm256_add_epi64(xacc[i], data_swap);
+            /* xacc[i] += product; */
+            xacc[i] = _mm256_add_epi64(product, sum);
+    }   }
+}
+XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2)
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 31) == 0);
+    {   __m256i* const xacc = (__m256i*) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+        const         __m256i* const xsecret = (const __m256i *) secret;
+        const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+            /* xacc[i] ^= (xacc[i] >> 47) */
+            __m256i const acc_vec     = xacc[i];
+            __m256i const shifted     = _mm256_srli_epi64    (acc_vec, 47);
+            __m256i const data_vec    = _mm256_xor_si256     (acc_vec, shifted);
+            /* xacc[i] ^= xsecret; */
+            __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
+            __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);
+
+            /* xacc[i] *= XXH_PRIME32_1; */
+            __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32);
+            __m256i const prod_lo     = _mm256_mul_epu32     (data_key, prime32);
+            __m256i const prod_hi     = _mm256_mul_epu32     (data_key_hi, prime32);
+            xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
+        }
+    }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
+    XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
+    (void)(&XXH_writeLE64);
+    XXH_PREFETCH(customSecret);
+    {   __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
+
+        const __m256i* const src  = (const __m256i*) ((const void*) XXH3_kSecret);
+              __m256i*       dest = (      __m256i*) customSecret;
+
+#       if defined(__GNUC__) || defined(__clang__)
+        /*
+         * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+         *   - do not extract the secret from sse registers in the internal loop
+         *   - use less common registers, and avoid pushing these reg into stack
+         */
+        XXH_COMPILER_GUARD(dest);
+#       endif
+        XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
+        XXH_ASSERT(((size_t)dest & 31) == 0);
+
+        /* GCC -O2 need unroll loop manually */
+        dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed);
+        dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed);
+        dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed);
+        dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed);
+        dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed);
+        dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed);
+    }
+}
+
+#endif
+
+/* x86dispatch always generates SSE2 */
+#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
+
+#ifndef XXH_TARGET_SSE2
+# define XXH_TARGET_SSE2  /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+    /* SSE2 is just a half-scale version of the AVX2 version. */
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+    {   __m128i* const xacc    =       (__m128i *) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+        const         __m128i* const xinput  = (const __m128i *) input;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+        const         __m128i* const xsecret = (const __m128i *) secret;
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+            /* data_vec    = xinput[i]; */
+            __m128i const data_vec    = _mm_loadu_si128   (xinput+i);
+            /* key_vec     = xsecret[i]; */
+            __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
+            /* data_key    = data_vec ^ key_vec; */
+            __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);
+            /* data_key_lo = data_key >> 32; */
+            __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+            /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+            __m128i const product     = _mm_mul_epu32     (data_key, data_key_lo);
+            /* xacc[i] += swap(data_vec); */
+            __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
+            __m128i const sum       = _mm_add_epi64(xacc[i], data_swap);
+            /* xacc[i] += product; */
+            xacc[i] = _mm_add_epi64(product, sum);
+    }   }
+}
+XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2)
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+    {   __m128i* const xacc = (__m128i*) acc;
+        /* Unaligned. This is mainly for pointer arithmetic, and because
+         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+        const         __m128i* const xsecret = (const __m128i *) secret;
+        const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
+
+        size_t i;
+        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+            /* xacc[i] ^= (xacc[i] >> 47) */
+            __m128i const acc_vec     = xacc[i];
+            __m128i const shifted     = _mm_srli_epi64    (acc_vec, 47);
+            __m128i const data_vec    = _mm_xor_si128     (acc_vec, shifted);
+            /* xacc[i] ^= xsecret[i]; */
+            __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
+            __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);
+
+            /* xacc[i] *= XXH_PRIME32_1; */
+            __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+            __m128i const prod_lo     = _mm_mul_epu32     (data_key, prime32);
+            __m128i const prod_hi     = _mm_mul_epu32     (data_key_hi, prime32);
+            xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
+        }
+    }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+    (void)(&XXH_writeLE64);
+    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
+
+#       if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
+        /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
+        XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
+        __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
+#       else
+        __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
+#       endif
+        int i;
+
+        const void* const src16 = XXH3_kSecret;
+        __m128i* dst16 = (__m128i*) customSecret;
+#       if defined(__GNUC__) || defined(__clang__)
+        /*
+         * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+         *   - do not extract the secret from sse registers in the internal loop
+         *   - use less common registers, and avoid pushing these reg into stack
+         */
+        XXH_COMPILER_GUARD(dst16);
+#       endif
+        XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
+        XXH_ASSERT(((size_t)dst16 & 15) == 0);
+
+        for (i=0; i < nbRounds; ++i) {
+            dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
+    }   }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_NEON)
+
+/* forward declarations for the scalar routines */
+XXH_FORCE_INLINE void
+XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
+                 void const* XXH_RESTRICT secret, size_t lane);
+
+XXH_FORCE_INLINE void
+XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
+                         void const* XXH_RESTRICT secret, size_t lane);
+
+/*!
+ * @internal
+ * @brief The bulk processing loop for NEON and WASM SIMD128.
+ *
+ * The NEON code path is actually partially scalar when running on AArch64. This
+ * is to optimize the pipelining and can have up to 15% speedup depending on the
+ * CPU, and it also mitigates some GCC codegen issues.
+ *
+ * @see XXH3_NEON_LANES for configuring this and details about this optimization.
+ *
+ * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit
+ * integers instead of the other platforms which mask full 64-bit vectors,
+ * so the setup is more complicated than just shifting right.
+ *
+ * Additionally, there is an optimization for 4 lanes at once noted below.
+ *
+ * Since, as stated, the most optimal amount of lanes for Cortexes is 6,
+ * there needs to be *three* versions of the accumulate operation used
+ * for the remaining 2 lanes.
+ *
+ * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap
+ * nearly perfectly.
+ */
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+    XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
+    {   /* GCC for darwin arm64 does not like aliasing here */
+        xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc;
+        /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
+        uint8_t const* xinput = (const uint8_t *) input;
+        uint8_t const* xsecret  = (const uint8_t *) secret;
+
+        size_t i;
+#ifdef __wasm_simd128__
+        /*
+         * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret
+         * is constant propagated, which results in it converting it to this
+         * inside the loop:
+         *
+         *    a = v128.load(XXH3_kSecret +  0 + $secret_offset, offset = 0)
+         *    b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0)
+         *    ...
+         *
+         * This requires a full 32-bit address immediate (and therefore a 6 byte
+         * instruction) as well as an add for each offset.
+         *
+         * Putting an asm guard prevents it from folding (at the cost of losing
+         * the alignment hint), and uses the free offset in `v128.load` instead
+         * of adding secret_offset each time which overall reduces code size by
+         * about a kilobyte and improves performance.
+         */
+        XXH_COMPILER_GUARD(xsecret);
+#endif
+        /* Scalar lanes use the normal scalarRound routine */
+        for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
+            XXH3_scalarRound(acc, input, secret, i);
+        }
+        i = 0;
+        /* 4 NEON lanes at a time. */
+        for (; i+1 < XXH3_NEON_LANES / 2; i+=2) {
+            /* data_vec = xinput[i]; */
+            uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput  + (i * 16));
+            uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput  + ((i+1) * 16));
+            /* key_vec  = xsecret[i];  */
+            uint64x2_t key_vec_1  = XXH_vld1q_u64(xsecret + (i * 16));
+            uint64x2_t key_vec_2  = XXH_vld1q_u64(xsecret + ((i+1) * 16));
+            /* data_swap = swap(data_vec) */
+            uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
+            uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
+            /* data_key = data_vec ^ key_vec; */
+            uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
+            uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
+
+            /*
+             * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a
+             * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to
+             * get one vector with the low 32 bits of each lane, and one vector
+             * with the high 32 bits of each lane.
+             *
+             * The intrinsic returns a double vector because the original ARMv7-a
+             * instruction modified both arguments in place. AArch64 and SIMD128 emit
+             * two instructions from this intrinsic.
+             *
+             *  [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ]
+             *  [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ]
+             */
+            uint32x4x2_t unzipped = vuzpq_u32(
+                vreinterpretq_u32_u64(data_key_1),
+                vreinterpretq_u32_u64(data_key_2)
+            );
+            /* data_key_lo = data_key & 0xFFFFFFFF */
+            uint32x4_t data_key_lo = unzipped.val[0];
+            /* data_key_hi = data_key >> 32 */
+            uint32x4_t data_key_hi = unzipped.val[1];
+            /*
+             * Then, we can split the vectors horizontally and multiply which, as for most
+             * widening intrinsics, have a variant that works on both high half vectors
+             * for free on AArch64. A similar instruction is available on SIMD128.
+             *
+             * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi
+             */
+            uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi);
+            uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi);
+            /*
+             * Clang reorders
+             *    a += b * c;     // umlal   swap.2d, dkl.2s, dkh.2s
+             *    c += a;         // add     acc.2d, acc.2d, swap.2d
+             * to
+             *    c += a;         // add     acc.2d, acc.2d, swap.2d
+             *    c += b * c;     // umlal   acc.2d, dkl.2s, dkh.2s
+             *
+             * While it would make sense in theory since the addition is faster,
+             * for reasons likely related to umlal being limited to certain NEON
+             * pipelines, this is worse. A compiler guard fixes this.
+             */
+            XXH_COMPILER_GUARD_CLANG_NEON(sum_1);
+            XXH_COMPILER_GUARD_CLANG_NEON(sum_2);
+            /* xacc[i] = acc_vec + sum; */
+            xacc[i]   = vaddq_u64(xacc[i], sum_1);
+            xacc[i+1] = vaddq_u64(xacc[i+1], sum_2);
+        }
+        /* Operate on the remaining NEON lanes 2 at a time. */
+        for (; i < XXH3_NEON_LANES / 2; i++) {
+            /* data_vec = xinput[i]; */
+            uint64x2_t data_vec = XXH_vld1q_u64(xinput  + (i * 16));
+            /* key_vec  = xsecret[i];  */
+            uint64x2_t key_vec  = XXH_vld1q_u64(xsecret + (i * 16));
+            /* acc_vec_2 = swap(data_vec) */
+            uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1);
+            /* data_key = data_vec ^ key_vec; */
+            uint64x2_t data_key = veorq_u64(data_vec, key_vec);
+            /* For two lanes, just use VMOVN and VSHRN. */
+            /* data_key_lo = data_key & 0xFFFFFFFF; */
+            uint32x2_t data_key_lo = vmovn_u64(data_key);
+            /* data_key_hi = data_key >> 32; */
+            uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32);
+            /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */
+            uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi);
+            /* Same Clang workaround as before */
+            XXH_COMPILER_GUARD_CLANG_NEON(sum);
+            /* xacc[i] = acc_vec + sum; */
+            xacc[i] = vaddq_u64 (xacc[i], sum);
+        }
+    }
+}
+XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon)
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+    {   xxh_aliasing_uint64x2_t* xacc       = (xxh_aliasing_uint64x2_t*) acc;
+        uint8_t const* xsecret = (uint8_t const*) secret;
+
+        size_t i;
+        /* WASM uses operator overloads and doesn't need these. */
+#ifndef __wasm_simd128__
+        /* { prime32_1, prime32_1 } */
+        uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1);
+        /* { 0, prime32_1, 0, prime32_1 } */
+        uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32));
+#endif
+
+        /* AArch64 uses both scalar and neon at the same time */
+        for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
+            XXH3_scalarScrambleRound(acc, secret, i);
+        }
+        for (i=0; i < XXH3_NEON_LANES / 2; i++) {
+            /* xacc[i] ^= (xacc[i] >> 47); */
+            uint64x2_t acc_vec  = xacc[i];
+            uint64x2_t shifted  = vshrq_n_u64(acc_vec, 47);
+            uint64x2_t data_vec = veorq_u64(acc_vec, shifted);
+
+            /* xacc[i] ^= xsecret[i]; */
+            uint64x2_t key_vec  = XXH_vld1q_u64(xsecret + (i * 16));
+            uint64x2_t data_key = veorq_u64(data_vec, key_vec);
+            /* xacc[i] *= XXH_PRIME32_1 */
+#ifdef __wasm_simd128__
+            /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */
+            xacc[i] = data_key * XXH_PRIME32_1;
+#else
+            /*
+             * Expanded version with portable NEON intrinsics
+             *
+             *    lo(x) * lo(y) + (hi(x) * lo(y) << 32)
+             *
+             * prod_hi = hi(data_key) * lo(prime) << 32
+             *
+             * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector
+             * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits
+             * and avoid the shift.
+             */
+            uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi);
+            /* Extract low bits for vmlal_u32  */
+            uint32x2_t data_key_lo = vmovn_u64(data_key);
+            /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */
+            xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo);
+#endif
+        }
+    }
+}
+#endif
+
+#if (XXH_VECTOR == XXH_VSX)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_vsx(  void* XXH_RESTRICT acc,
+                    const void* XXH_RESTRICT input,
+                    const void* XXH_RESTRICT secret)
+{
+    /* presumed aligned */
+    xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
+    xxh_u8 const* const xinput   = (xxh_u8 const*) input;   /* no alignment restriction */
+    xxh_u8 const* const xsecret  = (xxh_u8 const*) secret;    /* no alignment restriction */
+    xxh_u64x2 const v32 = { 32, 32 };
+    size_t i;
+    for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+        /* data_vec = xinput[i]; */
+        xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i);
+        /* key_vec = xsecret[i]; */
+        xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + 16*i);
+        xxh_u64x2 const data_key = data_vec ^ key_vec;
+        /* shuffled = (data_key << 32) | (data_key >> 32); */
+        xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
+        /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
+        xxh_u64x2 const product  = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
+        /* acc_vec = xacc[i]; */
+        xxh_u64x2 acc_vec        = xacc[i];
+        acc_vec += product;
+
+        /* swap high and low halves */
+#ifdef __s390x__
+        acc_vec += vec_permi(data_vec, data_vec, 2);
+#else
+        acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
+#endif
+        xacc[i] = acc_vec;
+    }
+}
+XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx)
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+    {   xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
+        const xxh_u8* const xsecret = (const xxh_u8*) secret;
+        /* constants */
+        xxh_u64x2 const v32  = { 32, 32 };
+        xxh_u64x2 const v47 = { 47, 47 };
+        xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
+        size_t i;
+        for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+            /* xacc[i] ^= (xacc[i] >> 47); */
+            xxh_u64x2 const acc_vec  = xacc[i];
+            xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
+
+            /* xacc[i] ^= xsecret[i]; */
+            xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + 16*i);
+            xxh_u64x2 const data_key = data_vec ^ key_vec;
+
+            /* xacc[i] *= XXH_PRIME32_1 */
+            /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF);  */
+            xxh_u64x2 const prod_even  = XXH_vec_mule((xxh_u32x4)data_key, prime);
+            /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32);  */
+            xxh_u64x2 const prod_odd  = XXH_vec_mulo((xxh_u32x4)data_key, prime);
+            xacc[i] = prod_odd + (prod_even << v32);
+    }   }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_SVE)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_sve( void* XXH_RESTRICT acc,
+                   const void* XXH_RESTRICT input,
+                   const void* XXH_RESTRICT secret)
+{
+    uint64_t *xacc = (uint64_t *)acc;
+    const uint64_t *xinput = (const uint64_t *)(const void *)input;
+    const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
+    svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
+    uint64_t element_count = svcntd();
+    if (element_count >= 8) {
+        svbool_t mask = svptrue_pat_b64(SV_VL8);
+        svuint64_t vacc = svld1_u64(mask, xacc);
+        ACCRND(vacc, 0);
+        svst1_u64(mask, xacc, vacc);
+    } else if (element_count == 2) {   /* sve128 */
+        svbool_t mask = svptrue_pat_b64(SV_VL2);
+        svuint64_t acc0 = svld1_u64(mask, xacc + 0);
+        svuint64_t acc1 = svld1_u64(mask, xacc + 2);
+        svuint64_t acc2 = svld1_u64(mask, xacc + 4);
+        svuint64_t acc3 = svld1_u64(mask, xacc + 6);
+        ACCRND(acc0, 0);
+        ACCRND(acc1, 2);
+        ACCRND(acc2, 4);
+        ACCRND(acc3, 6);
+        svst1_u64(mask, xacc + 0, acc0);
+        svst1_u64(mask, xacc + 2, acc1);
+        svst1_u64(mask, xacc + 4, acc2);
+        svst1_u64(mask, xacc + 6, acc3);
+    } else {
+        svbool_t mask = svptrue_pat_b64(SV_VL4);
+        svuint64_t acc0 = svld1_u64(mask, xacc + 0);
+        svuint64_t acc1 = svld1_u64(mask, xacc + 4);
+        ACCRND(acc0, 0);
+        ACCRND(acc1, 4);
+        svst1_u64(mask, xacc + 0, acc0);
+        svst1_u64(mask, xacc + 4, acc1);
+    }
+}
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc,
+               const xxh_u8* XXH_RESTRICT input,
+               const xxh_u8* XXH_RESTRICT secret,
+               size_t nbStripes)
+{
+    if (nbStripes != 0) {
+        uint64_t *xacc = (uint64_t *)acc;
+        const uint64_t *xinput = (const uint64_t *)(const void *)input;
+        const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
+        svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
+        uint64_t element_count = svcntd();
+        if (element_count >= 8) {
+            svbool_t mask = svptrue_pat_b64(SV_VL8);
+            svuint64_t vacc = svld1_u64(mask, xacc + 0);
+            do {
+                /* svprfd(svbool_t, void *, enum svfprop); */
+                svprfd(mask, xinput + 128, SV_PLDL1STRM);
+                ACCRND(vacc, 0);
+                xinput += 8;
+                xsecret += 1;
+                nbStripes--;
+           } while (nbStripes != 0);
+
+           svst1_u64(mask, xacc + 0, vacc);
+        } else if (element_count == 2) { /* sve128 */
+            svbool_t mask = svptrue_pat_b64(SV_VL2);
+            svuint64_t acc0 = svld1_u64(mask, xacc + 0);
+            svuint64_t acc1 = svld1_u64(mask, xacc + 2);
+            svuint64_t acc2 = svld1_u64(mask, xacc + 4);
+            svuint64_t acc3 = svld1_u64(mask, xacc + 6);
+            do {
+                svprfd(mask, xinput + 128, SV_PLDL1STRM);
+                ACCRND(acc0, 0);
+                ACCRND(acc1, 2);
+                ACCRND(acc2, 4);
+                ACCRND(acc3, 6);
+                xinput += 8;
+                xsecret += 1;
+                nbStripes--;
+           } while (nbStripes != 0);
+
+           svst1_u64(mask, xacc + 0, acc0);
+           svst1_u64(mask, xacc + 2, acc1);
+           svst1_u64(mask, xacc + 4, acc2);
+           svst1_u64(mask, xacc + 6, acc3);
+        } else {
+            svbool_t mask = svptrue_pat_b64(SV_VL4);
+            svuint64_t acc0 = svld1_u64(mask, xacc + 0);
+            svuint64_t acc1 = svld1_u64(mask, xacc + 4);
+            do {
+                svprfd(mask, xinput + 128, SV_PLDL1STRM);
+                ACCRND(acc0, 0);
+                ACCRND(acc1, 4);
+                xinput += 8;
+                xsecret += 1;
+                nbStripes--;
+           } while (nbStripes != 0);
+
+           svst1_u64(mask, xacc + 0, acc0);
+           svst1_u64(mask, xacc + 4, acc1);
+       }
+    }
+}
+
+#endif
+
+/* scalar variants - universal */
+
+#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__))
+/*
+ * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they
+ * emit an excess mask and a full 64-bit multiply-add (MADD X-form).
+ *
+ * While this might not seem like much, as AArch64 is a 64-bit architecture, only
+ * big Cortex designs have a full 64-bit multiplier.
+ *
+ * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit
+ * multiplies expand to 2-3 multiplies in microcode. This has a major penalty
+ * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline.
+ *
+ * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does
+ * not have this penalty and does the mask automatically.
+ */
+XXH_FORCE_INLINE xxh_u64
+XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
+{
+    xxh_u64 ret;
+    /* note: %x = 64-bit register, %w = 32-bit register */
+    __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc));
+    return ret;
+}
+#else
+XXH_FORCE_INLINE xxh_u64
+XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
+{
+    return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc;
+}
+#endif
+
+/*!
+ * @internal
+ * @brief Scalar round for @ref XXH3_accumulate_512_scalar().
+ *
+ * This is extracted to its own function because the NEON path uses a combination
+ * of NEON and scalar.
+ */
+XXH_FORCE_INLINE void
+XXH3_scalarRound(void* XXH_RESTRICT acc,
+                 void const* XXH_RESTRICT input,
+                 void const* XXH_RESTRICT secret,
+                 size_t lane)
+{
+    xxh_u64* xacc = (xxh_u64*) acc;
+    xxh_u8 const* xinput  = (xxh_u8 const*) input;
+    xxh_u8 const* xsecret = (xxh_u8 const*) secret;
+    XXH_ASSERT(lane < XXH_ACC_NB);
+    XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
+    {
+        xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8);
+        xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8);
+        xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
+        xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]);
+    }
+}
+
+/*!
+ * @internal
+ * @brief Processes a 64 byte block of data using the scalar path.
+ */
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
+                     const void* XXH_RESTRICT input,
+                     const void* XXH_RESTRICT secret)
+{
+    size_t i;
+    /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */
+#if defined(__GNUC__) && !defined(__clang__) \
+  && (defined(__arm__) || defined(__thumb2__)) \
+  && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \
+  && XXH_SIZE_OPT <= 0
+#  pragma GCC unroll 8
+#endif
+    for (i=0; i < XXH_ACC_NB; i++) {
+        XXH3_scalarRound(acc, input, secret, i);
+    }
+}
+XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar)
+
+/*!
+ * @internal
+ * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar().
+ *
+ * This is extracted to its own function because the NEON path uses a combination
+ * of NEON and scalar.
+ */
+XXH_FORCE_INLINE void
+XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
+                         void const* XXH_RESTRICT secret,
+                         size_t lane)
+{
+    xxh_u64* const xacc = (xxh_u64*) acc;   /* presumed aligned */
+    const xxh_u8* const xsecret = (const xxh_u8*) secret;   /* no alignment restriction */
+    XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
+    XXH_ASSERT(lane < XXH_ACC_NB);
+    {
+        xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8);
+        xxh_u64 acc64 = xacc[lane];
+        acc64 = XXH_xorshift64(acc64, 47);
+        acc64 ^= key64;
+        acc64 *= XXH_PRIME32_1;
+        xacc[lane] = acc64;
+    }
+}
+
+/*!
+ * @internal
+ * @brief Scrambles the accumulators after a large chunk has been read
+ */
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+    size_t i;
+    for (i=0; i < XXH_ACC_NB; i++) {
+        XXH3_scalarScrambleRound(acc, secret, i);
+    }
+}
+
+XXH_FORCE_INLINE void
+XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+    /*
+     * We need a separate pointer for the hack below,
+     * which requires a non-const pointer.
+     * Any decent compiler will optimize this out otherwise.
+     */
+    const xxh_u8* kSecretPtr = XXH3_kSecret;
+    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+
+#if defined(__GNUC__) && defined(__aarch64__)
+    /*
+     * UGLY HACK:
+     * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are
+     * placed sequentially, in order, at the top of the unrolled loop.
+     *
+     * While MOVK is great for generating constants (2 cycles for a 64-bit
+     * constant compared to 4 cycles for LDR), it fights for bandwidth with
+     * the arithmetic instructions.
+     *
+     *   I   L   S
+     * MOVK
+     * MOVK
+     * MOVK
+     * MOVK
+     * ADD
+     * SUB      STR
+     *          STR
+     * By forcing loads from memory (as the asm line causes the compiler to assume
+     * that XXH3_kSecretPtr has been changed), the pipelines are used more
+     * efficiently:
+     *   I   L   S
+     *      LDR
+     *  ADD LDR
+     *  SUB     STR
+     *          STR
+     *
+     * See XXH3_NEON_LANES for details on the pipsline.
+     *
+     * XXH3_64bits_withSeed, len == 256, Snapdragon 835
+     *   without hack: 2654.4 MB/s
+     *   with hack:    3202.9 MB/s
+     */
+    XXH_COMPILER_GUARD(kSecretPtr);
+#endif
+    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
+        int i;
+        for (i=0; i < nbRounds; i++) {
+            /*
+             * The asm hack causes the compiler to assume that kSecretPtr aliases with
+             * customSecret, and on aarch64, this prevented LDP from merging two
+             * loads together for free. Putting the loads together before the stores
+             * properly generates LDP.
+             */
+            xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i)     + seed64;
+            xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
+            XXH_writeLE64((xxh_u8*)customSecret + 16*i,     lo);
+            XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
+    }   }
+}
+
+
+typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t);
+typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
+typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
+
+
+#if (XXH_VECTOR == XXH_AVX512)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
+#define XXH3_accumulate     XXH3_accumulate_avx512
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_avx512
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
+
+#elif (XXH_VECTOR == XXH_AVX2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
+#define XXH3_accumulate     XXH3_accumulate_avx2
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_avx2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
+
+#elif (XXH_VECTOR == XXH_SSE2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
+#define XXH3_accumulate     XXH3_accumulate_sse2
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_sse2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
+
+#elif (XXH_VECTOR == XXH_NEON)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_neon
+#define XXH3_accumulate     XXH3_accumulate_neon
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_neon
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#elif (XXH_VECTOR == XXH_VSX)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
+#define XXH3_accumulate     XXH3_accumulate_vsx
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_vsx
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#elif (XXH_VECTOR == XXH_SVE)
+#define XXH3_accumulate_512 XXH3_accumulate_512_sve
+#define XXH3_accumulate     XXH3_accumulate_sve
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_scalar
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#else /* scalar */
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
+#define XXH3_accumulate     XXH3_accumulate_scalar
+#define XXH3_scrambleAcc    XXH3_scrambleAcc_scalar
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#endif
+
+#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */
+#  undef XXH3_initCustomSecret
+#  define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+#endif
+
+XXH_FORCE_INLINE void
+XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
+                      const xxh_u8* XXH_RESTRICT input, size_t len,
+                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                            XXH3_f_accumulate f_acc,
+                            XXH3_f_scrambleAcc f_scramble)
+{
+    size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
+    size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
+    size_t const nb_blocks = (len - 1) / block_len;
+
+    size_t n;
+
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+
+    for (n = 0; n < nb_blocks; n++) {
+        f_acc(acc, input + n*block_len, secret, nbStripesPerBlock);
+        f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
+    }
+
+    /* last partial block */
+    XXH_ASSERT(len > XXH_STRIPE_LEN);
+    {   size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
+        XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
+        f_acc(acc, input + nb_blocks*block_len, secret, nbStripes);
+
+        /* last stripe */
+        {   const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
+#define XXH_SECRET_LASTACC_START 7  /* not aligned on 8, last secret is different from acc & scrambler */
+            XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
+    }   }
+}
+
+XXH_FORCE_INLINE xxh_u64
+XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
+{
+    return XXH3_mul128_fold64(
+               acc[0] ^ XXH_readLE64(secret),
+               acc[1] ^ XXH_readLE64(secret+8) );
+}
+
+static XXH64_hash_t
+XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
+{
+    xxh_u64 result64 = start;
+    size_t i = 0;
+
+    for (i = 0; i < 4; i++) {
+        result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
+#if defined(__clang__)                                /* Clang */ \
+    && (defined(__arm__) || defined(__thumb__))       /* ARMv7 */ \
+    && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */  \
+    && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
+        /*
+         * UGLY HACK:
+         * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
+         * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
+         * XXH3_64bits, len == 256, Snapdragon 835:
+         *   without hack: 2063.7 MB/s
+         *   with hack:    2560.7 MB/s
+         */
+        XXH_COMPILER_GUARD(result64);
+#endif
+    }
+
+    return XXH3_avalanche(result64);
+}
+
+#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
+                        XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
+                           const void* XXH_RESTRICT secret, size_t secretSize,
+                           XXH3_f_accumulate f_acc,
+                           XXH3_f_scrambleAcc f_scramble)
+{
+    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+    XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble);
+
+    /* converge into final hash */
+    XXH_STATIC_ASSERT(sizeof(acc) == 64);
+    /* do not align on 8, so that the secret is different from the accumulator */
+#define XXH_SECRET_MERGEACCS_START 11
+    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+    return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
+}
+
+/*
+ * It's important for performance to transmit secret's size (when it's static)
+ * so that the compiler can properly optimize the vectorized loop.
+ * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
+ * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
+ * breaks -Og, this is XXH_NO_INLINE.
+ */
+XXH3_WITH_SECRET_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
+                             XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64;
+    return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc);
+}
+
+/*
+ * It's preferable for performance that XXH3_hashLong is not inlined,
+ * as it results in a smaller function for small data, easier to the instruction cache.
+ * Note that inside this no_inline function, we do inline the internal loop,
+ * and provide a statically defined secret size to allow optimization of vector loop.
+ */
+XXH_NO_INLINE XXH_PUREF XXH64_hash_t
+XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
+                          XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64; (void)secret; (void)secretLen;
+    return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc);
+}
+
+/*
+ * XXH3_hashLong_64b_withSeed():
+ * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
+ * and then use this key for long mode hashing.
+ *
+ * This operation is decently fast but nonetheless costs a little bit of time.
+ * Try to avoid it whenever possible (typically when seed==0).
+ *
+ * It's important for performance that XXH3_hashLong is not inlined. Not sure
+ * why (uop cache maybe?), but the difference is large and easily measurable.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
+                                    XXH64_hash_t seed,
+                                    XXH3_f_accumulate f_acc,
+                                    XXH3_f_scrambleAcc f_scramble,
+                                    XXH3_f_initCustomSecret f_initSec)
+{
+#if XXH_SIZE_OPT <= 0
+    if (seed == 0)
+        return XXH3_hashLong_64b_internal(input, len,
+                                          XXH3_kSecret, sizeof(XXH3_kSecret),
+                                          f_acc, f_scramble);
+#endif
+    {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+        f_initSec(secret, seed);
+        return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
+                                          f_acc, f_scramble);
+    }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len,
+                           XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)secret; (void)secretLen;
+    return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
+                XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+
+typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
+                                          XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
+                     XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+                     XXH3_hashLong64_f f_hashLong)
+{
+    XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+    /*
+     * If an action is to be taken if `secretLen` condition is not respected,
+     * it should be done here.
+     * For now, it's a contract pre-condition.
+     * Adding a check and a branch here would cost performance at every hash.
+     * Also, note that function signature doesn't offer room to return an error.
+     */
+    if (len <= 16)
+        return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+    if (len <= 128)
+        return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    if (len <= XXH3_MIDSIZE_MAX)
+        return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
+}
+
+
+/* ===   Public entry point   === */
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length)
+{
+    return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize)
+{
+    return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed)
+{
+    return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
+}
+
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+    if (length <= XXH3_MIDSIZE_MAX)
+        return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
+    return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize);
+}
+
+
+/* ===   XXH3 streaming   === */
+#ifndef XXH_NO_STREAM
+/*
+ * Malloc's a pointer that is always aligned to align.
+ *
+ * This must be freed with `XXH_alignedFree()`.
+ *
+ * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
+ * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
+ * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
+ *
+ * This underalignment previously caused a rather obvious crash which went
+ * completely unnoticed due to XXH3_createState() not actually being tested.
+ * Credit to RedSpah for noticing this bug.
+ *
+ * The alignment is done manually: Functions like posix_memalign or _mm_malloc
+ * are avoided: To maintain portability, we would have to write a fallback
+ * like this anyways, and besides, testing for the existence of library
+ * functions without relying on external build tools is impossible.
+ *
+ * The method is simple: Overallocate, manually align, and store the offset
+ * to the original behind the returned pointer.
+ *
+ * Align must be a power of 2 and 8 <= align <= 128.
+ */
+static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align)
+{
+    XXH_ASSERT(align <= 128 && align >= 8); /* range check */
+    XXH_ASSERT((align & (align-1)) == 0);   /* power of 2 */
+    XXH_ASSERT(s != 0 && s < (s + align));  /* empty/overflow */
+    {   /* Overallocate to make room for manual realignment and an offset byte */
+        xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
+        if (base != NULL) {
+            /*
+             * Get the offset needed to align this pointer.
+             *
+             * Even if the returned pointer is aligned, there will always be
+             * at least one byte to store the offset to the original pointer.
+             */
+            size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
+            /* Add the offset for the now-aligned pointer */
+            xxh_u8* ptr = base + offset;
+
+            XXH_ASSERT((size_t)ptr % align == 0);
+
+            /* Store the offset immediately before the returned pointer. */
+            ptr[-1] = (xxh_u8)offset;
+            return ptr;
+        }
+        return NULL;
+    }
+}
+/*
+ * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
+ * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
+ */
+static void XXH_alignedFree(void* p)
+{
+    if (p != NULL) {
+        xxh_u8* ptr = (xxh_u8*)p;
+        /* Get the offset byte we added in XXH_malloc. */
+        xxh_u8 offset = ptr[-1];
+        /* Free the original malloc'd pointer */
+        xxh_u8* base = ptr - offset;
+        XXH_free(base);
+    }
+}
+/*! @ingroup XXH3_family */
+/*!
+ * @brief Allocate an @ref XXH3_state_t.
+ *
+ * @return An allocated pointer of @ref XXH3_state_t on success.
+ * @return `NULL` on failure.
+ *
+ * @note Must be freed with XXH3_freeState().
+ */
+XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
+{
+    XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
+    if (state==NULL) return NULL;
+    XXH3_INITSTATE(state);
+    return state;
+}
+
+/*! @ingroup XXH3_family */
+/*!
+ * @brief Frees an @ref XXH3_state_t.
+ *
+ * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
+ *
+ * @return @ref XXH_OK.
+ *
+ * @note Must be allocated with XXH3_createState().
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
+{
+    XXH_alignedFree(statePtr);
+    return XXH_OK;
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API void
+XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state)
+{
+    XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
+}
+
+static void
+XXH3_reset_internal(XXH3_state_t* statePtr,
+                    XXH64_hash_t seed,
+                    const void* secret, size_t secretSize)
+{
+    size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
+    size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
+    XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
+    XXH_ASSERT(statePtr != NULL);
+    /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
+    memset((char*)statePtr + initStart, 0, initLength);
+    statePtr->acc[0] = XXH_PRIME32_3;
+    statePtr->acc[1] = XXH_PRIME64_1;
+    statePtr->acc[2] = XXH_PRIME64_2;
+    statePtr->acc[3] = XXH_PRIME64_3;
+    statePtr->acc[4] = XXH_PRIME64_4;
+    statePtr->acc[5] = XXH_PRIME32_2;
+    statePtr->acc[6] = XXH_PRIME64_5;
+    statePtr->acc[7] = XXH_PRIME32_1;
+    statePtr->seed = seed;
+    statePtr->useSeed = (seed != 0);
+    statePtr->extSecret = (const unsigned char*)secret;
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+    statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
+    statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
+    return XXH_OK;
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    XXH3_reset_internal(statePtr, 0, secret, secretSize);
+    if (secret == NULL) return XXH_ERROR;
+    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+    return XXH_OK;
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    if (seed==0) return XXH3_64bits_reset(statePtr);
+    if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
+        XXH3_initCustomSecret(statePtr->customSecret, seed);
+    XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
+    return XXH_OK;
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64)
+{
+    if (statePtr == NULL) return XXH_ERROR;
+    if (secret == NULL) return XXH_ERROR;
+    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+    XXH3_reset_internal(statePtr, seed64, secret, secretSize);
+    statePtr->useSeed = 1; /* always, even if seed64==0 */
+    return XXH_OK;
+}
+
+/*!
+ * @internal
+ * @brief Processes a large input for XXH3_update() and XXH3_digest_long().
+ *
+ * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block.
+ *
+ * @param acc                Pointer to the 8 accumulator lanes
+ * @param nbStripesSoFarPtr  In/out pointer to the number of leftover stripes in the block*
+ * @param nbStripesPerBlock  Number of stripes in a block
+ * @param input              Input pointer
+ * @param nbStripes          Number of stripes to process
+ * @param secret             Secret pointer
+ * @param secretLimit        Offset of the last block in @p secret
+ * @param f_acc              Pointer to an XXH3_accumulate implementation
+ * @param f_scramble         Pointer to an XXH3_scrambleAcc implementation
+ * @return                   Pointer past the end of @p input after processing
+ */
+XXH_FORCE_INLINE const xxh_u8 *
+XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
+                    size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
+                    const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
+                    const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
+                    XXH3_f_accumulate f_acc,
+                    XXH3_f_scrambleAcc f_scramble)
+{
+    const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE;
+    /* Process full blocks */
+    if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) {
+        /* Process the initial partial block... */
+        size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr;
+
+        do {
+            /* Accumulate and scramble */
+            f_acc(acc, input, initialSecret, nbStripesThisIter);
+            f_scramble(acc, secret + secretLimit);
+            input += nbStripesThisIter * XXH_STRIPE_LEN;
+            nbStripes -= nbStripesThisIter;
+            /* Then continue the loop with the full block size */
+            nbStripesThisIter = nbStripesPerBlock;
+            initialSecret = secret;
+        } while (nbStripes >= nbStripesPerBlock);
+        *nbStripesSoFarPtr = 0;
+    }
+    /* Process a partial block */
+    if (nbStripes > 0) {
+        f_acc(acc, input, initialSecret, nbStripes);
+        input += nbStripes * XXH_STRIPE_LEN;
+        *nbStripesSoFarPtr += nbStripes;
+    }
+    /* Return end pointer */
+    return input;
+}
+
+#ifndef XXH3_STREAM_USE_STACK
+# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */
+#   define XXH3_STREAM_USE_STACK 1
+# endif
+#endif
+/*
+ * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
+ */
+XXH_FORCE_INLINE XXH_errorcode
+XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
+            const xxh_u8* XXH_RESTRICT input, size_t len,
+            XXH3_f_accumulate f_acc,
+            XXH3_f_scrambleAcc f_scramble)
+{
+    if (input==NULL) {
+        XXH_ASSERT(len == 0);
+        return XXH_OK;
+    }
+
+    XXH_ASSERT(state != NULL);
+    {   const xxh_u8* const bEnd = input + len;
+        const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
+        /* For some reason, gcc and MSVC seem to suffer greatly
+         * when operating accumulators directly into state.
+         * Operating into stack space seems to enable proper optimization.
+         * clang, on the other hand, doesn't seem to need this trick */
+        XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8];
+        XXH_memcpy(acc, state->acc, sizeof(acc));
+#else
+        xxh_u64* XXH_RESTRICT const acc = state->acc;
+#endif
+        state->totalLen += len;
+        XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
+
+        /* small input : just fill in tmp buffer */
+        if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) {
+            XXH_memcpy(state->buffer + state->bufferedSize, input, len);
+            state->bufferedSize += (XXH32_hash_t)len;
+            return XXH_OK;
+        }
+
+        /* total input is now > XXH3_INTERNALBUFFER_SIZE */
+        #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
+        XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0);   /* clean multiple */
+
+        /*
+         * Internal buffer is partially filled (always, except at beginning)
+         * Complete it, then consume it.
+         */
+        if (state->bufferedSize) {
+            size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
+            XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
+            input += loadSize;
+            XXH3_consumeStripes(acc,
+                               &state->nbStripesSoFar, state->nbStripesPerBlock,
+                                state->buffer, XXH3_INTERNALBUFFER_STRIPES,
+                                secret, state->secretLimit,
+                                f_acc, f_scramble);
+            state->bufferedSize = 0;
+        }
+        XXH_ASSERT(input < bEnd);
+        if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
+            size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
+            input = XXH3_consumeStripes(acc,
+                                       &state->nbStripesSoFar, state->nbStripesPerBlock,
+                                       input, nbStripes,
+                                       secret, state->secretLimit,
+                                       f_acc, f_scramble);
+            XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
+
+        }
+        /* Some remaining input (always) : buffer it */
+        XXH_ASSERT(input < bEnd);
+        XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
+        XXH_ASSERT(state->bufferedSize == 0);
+        XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
+        state->bufferedSize = (XXH32_hash_t)(bEnd-input);
+#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
+        /* save stack accumulators into state */
+        XXH_memcpy(state->acc, acc, sizeof(acc));
+#endif
+    }
+
+    return XXH_OK;
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
+{
+    return XXH3_update(state, (const xxh_u8*)input, len,
+                       XXH3_accumulate, XXH3_scrambleAcc);
+}
+
+
+XXH_FORCE_INLINE void
+XXH3_digest_long (XXH64_hash_t* acc,
+                  const XXH3_state_t* state,
+                  const unsigned char* secret)
+{
+    xxh_u8 lastStripe[XXH_STRIPE_LEN];
+    const xxh_u8* lastStripePtr;
+
+    /*
+     * Digest on a local copy. This way, the state remains unaltered, and it can
+     * continue ingesting more input afterwards.
+     */
+    XXH_memcpy(acc, state->acc, sizeof(state->acc));
+    if (state->bufferedSize >= XXH_STRIPE_LEN) {
+        /* Consume remaining stripes then point to remaining data in buffer */
+        size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
+        size_t nbStripesSoFar = state->nbStripesSoFar;
+        XXH3_consumeStripes(acc,
+                           &nbStripesSoFar, state->nbStripesPerBlock,
+                            state->buffer, nbStripes,
+                            secret, state->secretLimit,
+                            XXH3_accumulate, XXH3_scrambleAcc);
+        lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN;
+    } else {  /* bufferedSize < XXH_STRIPE_LEN */
+        /* Copy to temp buffer */
+        size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
+        XXH_ASSERT(state->bufferedSize > 0);  /* there is always some input buffered */
+        XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
+        XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
+        lastStripePtr = lastStripe;
+    }
+    /* Last stripe */
+    XXH3_accumulate_512(acc,
+                        lastStripePtr,
+                        secret + state->secretLimit - XXH_SECRET_LASTACC_START);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
+{
+    const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+    if (state->totalLen > XXH3_MIDSIZE_MAX) {
+        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+        XXH3_digest_long(acc, state, secret);
+        return XXH3_mergeAccs(acc,
+                              secret + XXH_SECRET_MERGEACCS_START,
+                              (xxh_u64)state->totalLen * XXH_PRIME64_1);
+    }
+    /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
+    if (state->useSeed)
+        return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+    return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
+                                  secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+#endif /* !XXH_NO_STREAM */
+
+
+/* ==========================================
+ * XXH3 128 bits (a.k.a XXH128)
+ * ==========================================
+ * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
+ * even without counting the significantly larger output size.
+ *
+ * For example, extra steps are taken to avoid the seed-dependent collisions
+ * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
+ *
+ * This strength naturally comes at the cost of some speed, especially on short
+ * lengths. Note that longer hashes are about as fast as the 64-bit version
+ * due to it using only a slight modification of the 64-bit loop.
+ *
+ * XXH128 is also more oriented towards 64-bit machines. It is still extremely
+ * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
+ */
+
+XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
+XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    /* A doubled version of 1to3_64b with different constants. */
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(1 <= len && len <= 3);
+    XXH_ASSERT(secret != NULL);
+    /*
+     * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
+     * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
+     * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
+     */
+    {   xxh_u8 const c1 = input[0];
+        xxh_u8 const c2 = input[len >> 1];
+        xxh_u8 const c3 = input[len - 1];
+        xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
+                                | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
+        xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
+        xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+        xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
+        xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
+        xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
+        XXH128_hash_t h128;
+        h128.low64  = XXH64_avalanche(keyed_lo);
+        h128.high64 = XXH64_avalanche(keyed_hi);
+        return h128;
+    }
+}
+
+XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
+XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(4 <= len && len <= 8);
+    seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+    {   xxh_u32 const input_lo = XXH_readLE32(input);
+        xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
+        xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
+        xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
+        xxh_u64 const keyed = input_64 ^ bitflip;
+
+        /* Shift len to the left to ensure it is even, this avoids even multiplies. */
+        XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
+
+        m128.high64 += (m128.low64 << 1);
+        m128.low64  ^= (m128.high64 >> 3);
+
+        m128.low64   = XXH_xorshift64(m128.low64, 35);
+        m128.low64  *= PRIME_MX2;
+        m128.low64   = XXH_xorshift64(m128.low64, 28);
+        m128.high64  = XXH3_avalanche(m128.high64);
+        return m128;
+    }
+}
+
+XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
+XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(input != NULL);
+    XXH_ASSERT(secret != NULL);
+    XXH_ASSERT(9 <= len && len <= 16);
+    {   xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
+        xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
+        xxh_u64 const input_lo = XXH_readLE64(input);
+        xxh_u64       input_hi = XXH_readLE64(input + len - 8);
+        XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
+        /*
+         * Put len in the middle of m128 to ensure that the length gets mixed to
+         * both the low and high bits in the 128x64 multiply below.
+         */
+        m128.low64 += (xxh_u64)(len - 1) << 54;
+        input_hi   ^= bitfliph;
+        /*
+         * Add the high 32 bits of input_hi to the high 32 bits of m128, then
+         * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
+         * the high 64 bits of m128.
+         *
+         * The best approach to this operation is different on 32-bit and 64-bit.
+         */
+        if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
+            /*
+             * 32-bit optimized version, which is more readable.
+             *
+             * On 32-bit, it removes an ADC and delays a dependency between the two
+             * halves of m128.high64, but it generates an extra mask on 64-bit.
+             */
+            m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
+        } else {
+            /*
+             * 64-bit optimized (albeit more confusing) version.
+             *
+             * Uses some properties of addition and multiplication to remove the mask:
+             *
+             * Let:
+             *    a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
+             *    b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
+             *    c = XXH_PRIME32_2
+             *
+             *    a + (b * c)
+             * Inverse Property: x + y - x == y
+             *    a + (b * (1 + c - 1))
+             * Distributive Property: x * (y + z) == (x * y) + (x * z)
+             *    a + (b * 1) + (b * (c - 1))
+             * Identity Property: x * 1 == x
+             *    a + b + (b * (c - 1))
+             *
+             * Substitute a, b, and c:
+             *    input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+             *
+             * Since input_hi.hi + input_hi.lo == input_hi, we get this:
+             *    input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+             */
+            m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
+        }
+        /* m128 ^= XXH_swap64(m128 >> 64); */
+        m128.low64  ^= XXH_swap64(m128.high64);
+
+        {   /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
+            XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
+            h128.high64 += m128.high64 * XXH_PRIME64_2;
+
+            h128.low64   = XXH3_avalanche(h128.low64);
+            h128.high64  = XXH3_avalanche(h128.high64);
+            return h128;
+    }   }
+}
+
+/*
+ * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
+ */
+XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
+XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+    XXH_ASSERT(len <= 16);
+    {   if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
+        if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
+        if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
+        {   XXH128_hash_t h128;
+            xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
+            xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
+            h128.low64 = XXH64_avalanche(seed ^ bitflipl);
+            h128.high64 = XXH64_avalanche( seed ^ bitfliph);
+            return h128;
+    }   }
+}
+
+/*
+ * A bit slower than XXH3_mix16B, but handles multiply by zero better.
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
+              const xxh_u8* secret, XXH64_hash_t seed)
+{
+    acc.low64  += XXH3_mix16B (input_1, secret+0, seed);
+    acc.low64  ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
+    acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
+    acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
+    return acc;
+}
+
+
+XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
+XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                      XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(16 < len && len <= 128);
+
+    {   XXH128_hash_t acc;
+        acc.low64 = len * XXH_PRIME64_1;
+        acc.high64 = 0;
+
+#if XXH_SIZE_OPT >= 1
+        {
+            /* Smaller, but slightly slower. */
+            unsigned int i = (unsigned int)(len - 1) / 32;
+            do {
+                acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed);
+            } while (i-- != 0);
+        }
+#else
+        if (len > 32) {
+            if (len > 64) {
+                if (len > 96) {
+                    acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
+                }
+                acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
+            }
+            acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
+        }
+        acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
+#endif
+        {   XXH128_hash_t h128;
+            h128.low64  = acc.low64 + acc.high64;
+            h128.high64 = (acc.low64    * XXH_PRIME64_1)
+                        + (acc.high64   * XXH_PRIME64_4)
+                        + ((len - seed) * XXH_PRIME64_2);
+            h128.low64  = XXH3_avalanche(h128.low64);
+            h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+            return h128;
+        }
+    }
+}
+
+XXH_NO_INLINE XXH_PUREF XXH128_hash_t
+XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+                       const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                       XXH64_hash_t seed)
+{
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+    {   XXH128_hash_t acc;
+        unsigned i;
+        acc.low64 = len * XXH_PRIME64_1;
+        acc.high64 = 0;
+        /*
+         *  We set as `i` as offset + 32. We do this so that unchanged
+         * `len` can be used as upper bound. This reaches a sweet spot
+         * where both x86 and aarch64 get simple agen and good codegen
+         * for the loop.
+         */
+        for (i = 32; i < 160; i += 32) {
+            acc = XXH128_mix32B(acc,
+                                input  + i - 32,
+                                input  + i - 16,
+                                secret + i - 32,
+                                seed);
+        }
+        acc.low64 = XXH3_avalanche(acc.low64);
+        acc.high64 = XXH3_avalanche(acc.high64);
+        /*
+         * NB: `i <= len` will duplicate the last 32-bytes if
+         * len % 32 was zero. This is an unfortunate necessity to keep
+         * the hash result stable.
+         */
+        for (i=160; i <= len; i += 32) {
+            acc = XXH128_mix32B(acc,
+                                input + i - 32,
+                                input + i - 16,
+                                secret + XXH3_MIDSIZE_STARTOFFSET + i - 160,
+                                seed);
+        }
+        /* last bytes */
+        acc = XXH128_mix32B(acc,
+                            input + len - 16,
+                            input + len - 32,
+                            secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
+                            (XXH64_hash_t)0 - seed);
+
+        {   XXH128_hash_t h128;
+            h128.low64  = acc.low64 + acc.high64;
+            h128.high64 = (acc.low64    * XXH_PRIME64_1)
+                        + (acc.high64   * XXH_PRIME64_4)
+                        + ((len - seed) * XXH_PRIME64_2);
+            h128.low64  = XXH3_avalanche(h128.low64);
+            h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+            return h128;
+        }
+    }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
+                            const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+                            XXH3_f_accumulate f_acc,
+                            XXH3_f_scrambleAcc f_scramble)
+{
+    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+    XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble);
+
+    /* converge into final hash */
+    XXH_STATIC_ASSERT(sizeof(acc) == 64);
+    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+    {   XXH128_hash_t h128;
+        h128.low64  = XXH3_mergeAccs(acc,
+                                     secret + XXH_SECRET_MERGEACCS_START,
+                                     (xxh_u64)len * XXH_PRIME64_1);
+        h128.high64 = XXH3_mergeAccs(acc,
+                                     secret + secretSize
+                                            - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+                                     ~((xxh_u64)len * XXH_PRIME64_2));
+        return h128;
+    }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong() is not inlined.
+ */
+XXH_NO_INLINE XXH_PUREF XXH128_hash_t
+XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
+                           XXH64_hash_t seed64,
+                           const void* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64; (void)secret; (void)secretLen;
+    return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
+                                       XXH3_accumulate, XXH3_scrambleAcc);
+}
+
+/*
+ * It's important for performance to pass @p secretLen (when it's static)
+ * to the compiler, so that it can properly optimize the vectorized loop.
+ *
+ * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
+ * breaks -Og, this is XXH_NO_INLINE.
+ */
+XXH3_WITH_SECRET_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
+                              XXH64_hash_t seed64,
+                              const void* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64;
+    return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
+                                       XXH3_accumulate, XXH3_scrambleAcc);
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
+                                XXH64_hash_t seed64,
+                                XXH3_f_accumulate f_acc,
+                                XXH3_f_scrambleAcc f_scramble,
+                                XXH3_f_initCustomSecret f_initSec)
+{
+    if (seed64 == 0)
+        return XXH3_hashLong_128b_internal(input, len,
+                                           XXH3_kSecret, sizeof(XXH3_kSecret),
+                                           f_acc, f_scramble);
+    {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+        f_initSec(secret, seed64);
+        return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
+                                           f_acc, f_scramble);
+    }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed(const void* input, size_t len,
+                            XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)secret; (void)secretLen;
+    return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
+                XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
+                                            XXH64_hash_t, const void* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_128bits_internal(const void* input, size_t len,
+                      XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+                      XXH3_hashLong128_f f_hl128)
+{
+    XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+    /*
+     * If an action is to be taken if `secret` conditions are not respected,
+     * it should be done here.
+     * For now, it's a contract pre-condition.
+     * Adding a check and a branch here would cost performance at every hash.
+     */
+    if (len <= 16)
+        return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+    if (len <= 128)
+        return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    if (len <= XXH3_MIDSIZE_MAX)
+        return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+    return f_hl128(input, len, seed64, secret, secretLen);
+}
+
+
+/* ===   Public XXH128 API   === */
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len)
+{
+    return XXH3_128bits_internal(input, len, 0,
+                                 XXH3_kSecret, sizeof(XXH3_kSecret),
+                                 XXH3_hashLong_128b_default);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize)
+{
+    return XXH3_128bits_internal(input, len, 0,
+                                 (const xxh_u8*)secret, secretSize,
+                                 XXH3_hashLong_128b_withSecret);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
+{
+    return XXH3_128bits_internal(input, len, seed,
+                                 XXH3_kSecret, sizeof(XXH3_kSecret),
+                                 XXH3_hashLong_128b_withSeed);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+    if (len <= XXH3_MIDSIZE_MAX)
+        return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
+    return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
+{
+    return XXH3_128bits_withSeed(input, len, seed);
+}
+
+
+/* ===   XXH3 128-bit streaming   === */
+#ifndef XXH_NO_STREAM
+/*
+ * All initialization and update functions are identical to 64-bit streaming variant.
+ * The only difference is the finalization routine.
+ */
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
+{
+    return XXH3_64bits_reset(statePtr);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
+{
+    return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+    return XXH3_64bits_reset_withSeed(statePtr, seed);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+    return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
+{
+    return XXH3_64bits_update(state, input, len);
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
+{
+    const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+    if (state->totalLen > XXH3_MIDSIZE_MAX) {
+        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+        XXH3_digest_long(acc, state, secret);
+        XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+        {   XXH128_hash_t h128;
+            h128.low64  = XXH3_mergeAccs(acc,
+                                         secret + XXH_SECRET_MERGEACCS_START,
+                                         (xxh_u64)state->totalLen * XXH_PRIME64_1);
+            h128.high64 = XXH3_mergeAccs(acc,
+                                         secret + state->secretLimit + XXH_STRIPE_LEN
+                                                - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+                                         ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
+            return h128;
+        }
+    }
+    /* len <= XXH3_MIDSIZE_MAX : short code */
+    if (state->seed)
+        return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+    return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
+                                   secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+#endif /* !XXH_NO_STREAM */
+/* 128-bit utility functions */
+
+#include <string.h>   /* memcmp, memcpy */
+
+/* return : 1 is equal, 0 if different */
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
+{
+    /* note : XXH128_hash_t is compact, it has no padding byte */
+    return !(memcmp(&h1, &h2, sizeof(h1)));
+}
+
+/* This prototype is compatible with stdlib's qsort().
+ * @return : >0 if *h128_1  > *h128_2
+ *           <0 if *h128_1  < *h128_2
+ *           =0 if *h128_1 == *h128_2  */
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2)
+{
+    XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
+    XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
+    int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
+    /* note : bets that, in most cases, hash values are different */
+    if (hcmp) return hcmp;
+    return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
+}
+
+
+/*======   Canonical representation   ======*/
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API void
+XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) {
+        hash.high64 = XXH_swap64(hash.high64);
+        hash.low64  = XXH_swap64(hash.low64);
+    }
+    XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
+    XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src)
+{
+    XXH128_hash_t h;
+    h.high64 = XXH_readBE64(src);
+    h.low64  = XXH_readBE64(src->digest + 8);
+    return h;
+}
+
+
+
+/* ==========================================
+ * Secret generators
+ * ==========================================
+ */
+#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
+
+XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128)
+{
+    XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
+    XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize)
+{
+#if (XXH_DEBUGLEVEL >= 1)
+    XXH_ASSERT(secretBuffer != NULL);
+    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+#else
+    /* production mode, assert() are disabled */
+    if (secretBuffer == NULL) return XXH_ERROR;
+    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+#endif
+
+    if (customSeedSize == 0) {
+        customSeed = XXH3_kSecret;
+        customSeedSize = XXH_SECRET_DEFAULT_SIZE;
+    }
+#if (XXH_DEBUGLEVEL >= 1)
+    XXH_ASSERT(customSeed != NULL);
+#else
+    if (customSeed == NULL) return XXH_ERROR;
+#endif
+
+    /* Fill secretBuffer with a copy of customSeed - repeat as needed */
+    {   size_t pos = 0;
+        while (pos < secretSize) {
+            size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
+            memcpy((char*)secretBuffer + pos, customSeed, toCopy);
+            pos += toCopy;
+    }   }
+
+    {   size_t const nbSeg16 = secretSize / 16;
+        size_t n;
+        XXH128_canonical_t scrambler;
+        XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
+        for (n=0; n<nbSeg16; n++) {
+            XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
+            XXH3_combine16((char*)secretBuffer + n*16, h128);
+        }
+        /* last segment */
+        XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
+    }
+    return XXH_OK;
+}
+
+/*! @ingroup XXH3_family */
+XXH_PUBLIC_API void
+XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
+{
+    XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+    XXH3_initCustomSecret(secret, seed);
+    XXH_ASSERT(secretBuffer != NULL);
+    memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
+}
+
+
+
+/* Pop our optimization override from above */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+  && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+  && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
+#  pragma GCC pop_options
+#endif
+
+#endif  /* XXH_NO_LONG_LONG */
+
+#endif  /* XXH_NO_XXH3 */
+
+/*!
+ * @}
+ */
+#endif  /* XXH_IMPLEMENTATION */
+
+
+#if defined (__cplusplus)
+} /* extern "C" */
+#endif
+/**** ended inlining xxhash.h ****/
+#ifndef ZSTD_NO_TRACE
+/**** start inlining zstd_trace.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_TRACE_H
+#define ZSTD_TRACE_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include <stddef.h>
+
+/* weak symbol support
+ * For now, enable conservatively:
+ * - Only GNUC
+ * - Only ELF
+ * - Only x86-64, i386, aarch64 and risc-v.
+ * Also, explicitly disable on platforms known not to work so they aren't
+ * forgotten in the future.
+ */
+#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
+    defined(__GNUC__) && defined(__ELF__) && \
+    (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
+     defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \
+    !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
+    !defined(__CYGWIN__) && !defined(_AIX)
+#  define ZSTD_HAVE_WEAK_SYMBOLS 1
+#else
+#  define ZSTD_HAVE_WEAK_SYMBOLS 0
+#endif
+#if ZSTD_HAVE_WEAK_SYMBOLS
+#  define ZSTD_WEAK_ATTR __attribute__((__weak__))
+#else
+#  define ZSTD_WEAK_ATTR
+#endif
+
+/* Only enable tracing when weak symbols are available. */
+#ifndef ZSTD_TRACE
+#  define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
+#endif
+
+#if ZSTD_TRACE
+
+struct ZSTD_CCtx_s;
+struct ZSTD_DCtx_s;
+struct ZSTD_CCtx_params_s;
+
+typedef struct {
+    /**
+     * ZSTD_VERSION_NUMBER
+     *
+     * This is guaranteed to be the first member of ZSTD_trace.
+     * Otherwise, this struct is not stable between versions. If
+     * the version number does not match your expectation, you
+     * should not interpret the rest of the struct.
+     */
+    unsigned version;
+    /**
+     * Non-zero if streaming (de)compression is used.
+     */
+    unsigned streaming;
+    /**
+     * The dictionary ID.
+     */
+    unsigned dictionaryID;
+    /**
+     * Is the dictionary cold?
+     * Only set on decompression.
+     */
+    unsigned dictionaryIsCold;
+    /**
+     * The dictionary size or zero if no dictionary.
+     */
+    size_t dictionarySize;
+    /**
+     * The uncompressed size of the data.
+     */
+    size_t uncompressedSize;
+    /**
+     * The compressed size of the data.
+     */
+    size_t compressedSize;
+    /**
+     * The fully resolved CCtx parameters (NULL on decompression).
+     */
+    struct ZSTD_CCtx_params_s const* params;
+    /**
+     * The ZSTD_CCtx pointer (NULL on decompression).
+     */
+    struct ZSTD_CCtx_s const* cctx;
+    /**
+     * The ZSTD_DCtx pointer (NULL on compression).
+     */
+    struct ZSTD_DCtx_s const* dctx;
+} ZSTD_Trace;
+
+/**
+ * A tracing context. It must be 0 when tracing is disabled.
+ * Otherwise, any non-zero value returned by a tracing begin()
+ * function is presented to any subsequent calls to end().
+ *
+ * Any non-zero value is treated as tracing is enabled and not
+ * interpreted by the library.
+ *
+ * Two possible uses are:
+ * * A timestamp for when the begin() function was called.
+ * * A unique key identifying the (de)compression, like the
+ *   address of the [dc]ctx pointer if you need to track
+ *   more information than just a timestamp.
+ */
+typedef unsigned long long ZSTD_TraceCtx;
+
+/**
+ * Trace the beginning of a compression call.
+ * @param cctx The dctx pointer for the compression.
+ *             It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled. The return value is
+ *          passed to ZSTD_trace_compress_end().
+ */
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
+    struct ZSTD_CCtx_s const* cctx);
+
+/**
+ * Trace the end of a compression call.
+ * @param ctx The return value of ZSTD_trace_compress_begin().
+ * @param trace The zstd tracing info.
+ */
+ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
+    ZSTD_TraceCtx ctx,
+    ZSTD_Trace const* trace);
+
+/**
+ * Trace the beginning of a decompression call.
+ * @param dctx The dctx pointer for the decompression.
+ *             It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled. The return value is
+ *          passed to ZSTD_trace_compress_end().
+ */
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
+    struct ZSTD_DCtx_s const* dctx);
+
+/**
+ * Trace the end of a decompression call.
+ * @param ctx The return value of ZSTD_trace_decompress_begin().
+ * @param trace The zstd tracing info.
+ */
+ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
+    ZSTD_TraceCtx ctx,
+    ZSTD_Trace const* trace);
+
+#endif /* ZSTD_TRACE */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_TRACE_H */
+/**** ended inlining zstd_trace.h ****/
+#else
+#  define ZSTD_TRACE 0
+#endif
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* ---- static assert (debug) --- */
+#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
+#define ZSTD_isError ERR_isError   /* for inlining */
+#define FSE_isError  ERR_isError
+#define HUF_isError  ERR_isError
+
+
+/*-*************************************
+*  shared macros
+***************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+#define BOUNDED(min,val,max) (MAX(min,MIN(val,max)))
+
+
+/*-*************************************
+*  Common constants
+***************************************/
+#define ZSTD_OPT_NUM    (1<<12)
+
+#define ZSTD_REP_NUM      3                 /* number of repcodes */
+static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define BIT7 128
+#define BIT6  64
+#define BIT5  32
+#define BIT4  16
+#define BIT1   2
+#define BIT0   1
+
+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
+static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
+static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
+
+#define ZSTD_FRAMEIDSIZE 4   /* magic number size */
+
+#define ZSTD_BLOCKHEADERSIZE 3   /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
+static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
+typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
+
+#define ZSTD_FRAMECHECKSUMSIZE 4
+
+#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
+#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */)   /* for a non-null block */
+#define MIN_LITERALS_FOR_4_STREAMS 6
+
+typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
+
+#define LONGNBSEQ 0x7F00
+
+#define MINMATCH 3
+
+#define Litbits  8
+#define LitHufLog 11
+#define MaxLit ((1<<Litbits) - 1)
+#define MaxML   52
+#define MaxLL   35
+#define DefaultMaxOff 28
+#define MaxOff  31
+#define MaxSeq MAX(MaxLL, MaxML)   /* Assumption : MaxOff < MaxLL,MaxML */
+#define MLFSELog    9
+#define LLFSELog    9
+#define OffFSELog   8
+#define MaxFSELog  MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
+#define MaxMLBits 16
+#define MaxLLBits 16
+
+#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
+/* Each table cannot take more than #symbols * FSELog bits */
+#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8)
+
+static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = {
+     0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0,
+     1, 1, 1, 1, 2, 2, 3, 3,
+     4, 6, 7, 8, 9,10,11,12,
+    13,14,15,16
+};
+static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = {
+     4, 3, 2, 2, 2, 2, 2, 2,
+     2, 2, 2, 2, 2, 1, 1, 1,
+     2, 2, 2, 2, 2, 2, 2, 2,
+     2, 3, 2, 1, 1, 1, 1, 1,
+    -1,-1,-1,-1
+};
+#define LL_DEFAULTNORMLOG 6  /* for static allocation */
+static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
+
+static UNUSED_ATTR const U8 ML_bits[MaxML+1] = {
+     0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0,
+     1, 1, 1, 1, 2, 2, 3, 3,
+     4, 4, 5, 7, 8, 9,10,11,
+    12,13,14,15,16
+};
+static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = {
+     1, 4, 3, 2, 2, 2, 2, 2,
+     2, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1,-1,-1,
+    -1,-1,-1,-1,-1
+};
+#define ML_DEFAULTNORMLOG 6  /* for static allocation */
+static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
+
+static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = {
+     1, 1, 1, 1, 1, 1, 2, 2,
+     2, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1,
+    -1,-1,-1,-1,-1
+};
+#define OF_DEFAULTNORMLOG 5  /* for static allocation */
+static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
+
+
+/*-*******************************************
+*  Shared functions to include for inlining
+*********************************************/
+static void ZSTD_copy8(void* dst, const void* src) {
+#if defined(ZSTD_ARCH_ARM_NEON)
+    vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
+#else
+    ZSTD_memcpy(dst, src, 8);
+#endif
+}
+#define COPY8(d,s) do { ZSTD_copy8(d,s); d+=8; s+=8; } while (0)
+
+/* Need to use memmove here since the literal buffer can now be located within
+   the dst buffer. In circumstances where the op "catches up" to where the
+   literal buffer is, there can be partial overlaps in this call on the final
+   copy if the literal is being shifted by less than 16 bytes. */
+static void ZSTD_copy16(void* dst, const void* src) {
+#if defined(ZSTD_ARCH_ARM_NEON)
+    vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
+#elif defined(ZSTD_ARCH_X86_SSE2)
+    _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
+#elif defined(__clang__)
+    ZSTD_memmove(dst, src, 16);
+#else
+    /* ZSTD_memmove is not inlined properly by gcc */
+    BYTE copy16_buf[16];
+    ZSTD_memcpy(copy16_buf, src, 16);
+    ZSTD_memcpy(dst, copy16_buf, 16);
+#endif
+}
+#define COPY16(d,s) do { ZSTD_copy16(d,s); d+=16; s+=16; } while (0)
+
+#define WILDCOPY_OVERLENGTH 32
+#define WILDCOPY_VECLEN 16
+
+typedef enum {
+    ZSTD_no_overlap,
+    ZSTD_overlap_src_before_dst
+    /*  ZSTD_overlap_dst_before_src, */
+} ZSTD_overlap_e;
+
+/*! ZSTD_wildcopy() :
+ *  Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
+ *  @param ovtype controls the overlap detection
+ *         - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
+ *         - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
+ *           The src buffer must be before the dst buffer.
+ */
+MEM_STATIC FORCE_INLINE_ATTR
+void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
+{
+    ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
+    const BYTE* ip = (const BYTE*)src;
+    BYTE* op = (BYTE*)dst;
+    BYTE* const oend = op + length;
+
+    if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
+        /* Handle short offset copies. */
+        do {
+            COPY8(op, ip);
+        } while (op < oend);
+    } else {
+        assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
+        /* Separate out the first COPY16() call because the copy length is
+         * almost certain to be short, so the branches have different
+         * probabilities. Since it is almost certain to be short, only do
+         * one COPY16() in the first call. Then, do two calls per loop since
+         * at that point it is more likely to have a high trip count.
+         */
+        ZSTD_copy16(op, ip);
+        if (16 >= length) return;
+        op += 16;
+        ip += 16;
+        do {
+            COPY16(op, ip);
+            COPY16(op, ip);
+        }
+        while (op < oend);
+    }
+}
+
+MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+    size_t const length = MIN(dstCapacity, srcSize);
+    if (length > 0) {
+        ZSTD_memcpy(dst, src, length);
+    }
+    return length;
+}
+
+/* define "workspace is too large" as this number of times larger than needed */
+#define ZSTD_WORKSPACETOOLARGE_FACTOR 3
+
+/* when workspace is continuously too large
+ * during at least this number of times,
+ * context's memory usage is considered wasteful,
+ * because it's sized to handle a worst case scenario which rarely happens.
+ * In which case, resize it down to free some memory */
+#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
+
+/* Controls whether the input/output buffer is buffered or stable. */
+typedef enum {
+    ZSTD_bm_buffered = 0,  /* Buffer the input/output */
+    ZSTD_bm_stable = 1     /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
+} ZSTD_bufferMode_e;
+
+
+/*-*******************************************
+*  Private declarations
+*********************************************/
+typedef struct seqDef_s {
+    U32 offBase;   /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
+    U16 litLength;
+    U16 mlBase;    /* mlBase == matchLength - MINMATCH */
+} seqDef;
+
+/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
+typedef enum {
+    ZSTD_llt_none = 0,             /* no longLengthType */
+    ZSTD_llt_literalLength = 1,    /* represents a long literal */
+    ZSTD_llt_matchLength = 2       /* represents a long match */
+} ZSTD_longLengthType_e;
+
+typedef struct {
+    seqDef* sequencesStart;
+    seqDef* sequences;      /* ptr to end of sequences */
+    BYTE*  litStart;
+    BYTE*  lit;             /* ptr to end of literals */
+    BYTE*  llCode;
+    BYTE*  mlCode;
+    BYTE*  ofCode;
+    size_t maxNbSeq;
+    size_t maxNbLit;
+
+    /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
+     * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
+     * the existing value of the litLength or matchLength by 0x10000.
+     */
+    ZSTD_longLengthType_e longLengthType;
+    U32                   longLengthPos;  /* Index of the sequence to apply long length modification to */
+} seqStore_t;
+
+typedef struct {
+    U32 litLength;
+    U32 matchLength;
+} ZSTD_sequenceLength;
+
+/**
+ * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
+ * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
+ */
+MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
+{
+    ZSTD_sequenceLength seqLen;
+    seqLen.litLength = seq->litLength;
+    seqLen.matchLength = seq->mlBase + MINMATCH;
+    if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
+        if (seqStore->longLengthType == ZSTD_llt_literalLength) {
+            seqLen.litLength += 0x10000;
+        }
+        if (seqStore->longLengthType == ZSTD_llt_matchLength) {
+            seqLen.matchLength += 0x10000;
+        }
+    }
+    return seqLen;
+}
+
+/**
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
+ * Note: before using `compressedSize`, check for errors using ZSTD_isError().
+ *       similarly, before using `decompressedBound`, check for errors using:
+ *          `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
+ */
+typedef struct {
+    size_t nbBlocks;
+    size_t compressedSize;
+    unsigned long long decompressedBound;
+} ZSTD_frameSizeInfo;   /* decompress & legacy */
+
+const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
+
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ *        do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);   /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
+
+
+typedef struct {
+    blockType_e blockType;
+    U32 lastBlock;
+    U32 origSize;
+} blockProperties_t;   /* declared here for decompress and fullbench */
+
+/*! ZSTD_getcBlockSize() :
+ *  Provides the size of compressed block from block header `src` */
+/*  Used by: decompress, fullbench */
+size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
+                          blockProperties_t* bpPtr);
+
+/*! ZSTD_decodeSeqHeaders() :
+ *  decode sequence header from src */
+/*  Used by: zstd_decompress_block, fullbench */
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+                       const void* src, size_t srcSize);
+
+/**
+ * @returns true iff the CPU supports dynamic BMI2 dispatch.
+ */
+MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
+{
+    ZSTD_cpuid_t cpuid = ZSTD_cpuid();
+    return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif   /* ZSTD_CCOMMON_H_MODULE */
+/**** ended inlining zstd_internal.h ****/
+
+
+/*-****************************************
+*  Version
+******************************************/
+unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
+
+const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
+
+
+/*-****************************************
+*  ZSTD Error Management
+******************************************/
+#undef ZSTD_isError   /* defined within zstd_internal.h */
+/*! ZSTD_isError() :
+ *  tells if a return value is an error code
+ *  symbol is required for external callers */
+unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
+
+/*! ZSTD_getErrorName() :
+ *  provides error code string from function result (useful for debugging) */
+const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+/*! ZSTD_getError() :
+ *  convert a `size_t` function result into a proper ZSTD_errorCode enum */
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
+
+/*! ZSTD_getErrorString() :
+ *  provides error code string from enum */
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
+/**** ended inlining common/zstd_common.c ****/
+
+/**** start inlining decompress/huf_decompress.c ****/
+/* ******************************************************************
+ * huff0 huffman decoder,
+ * part of Finite State Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ *  You can contact the author at :
+ *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* **************************************************************
+*  Dependencies
+****************************************************************/
+/**** skipping file: ../common/zstd_deps.h ****/
+/**** skipping file: ../common/compiler.h ****/
+/**** skipping file: ../common/bitstream.h ****/
+/**** skipping file: ../common/fse.h ****/
+/**** skipping file: ../common/huf.h ****/
+/**** skipping file: ../common/error_private.h ****/
+/**** skipping file: ../common/zstd_internal.h ****/
+/**** skipping file: ../common/bits.h ****/
+
+/* **************************************************************
+*  Constants
+****************************************************************/
+
+#define HUF_DECODER_FAST_TABLELOG 11
+
+/* **************************************************************
+*  Macros
+****************************************************************/
+
+#ifdef HUF_DISABLE_FAST_DECODE
+# define HUF_ENABLE_FAST_DECODE 0
+#else
+# define HUF_ENABLE_FAST_DECODE 1
+#endif
+
+/* These two optional macros force the use one way or another of the two
+ * Huffman decompression implementations. You can't force in both directions
+ * at the same time.
+ */
+#if defined(HUF_FORCE_DECOMPRESS_X1) && \
+    defined(HUF_FORCE_DECOMPRESS_X2)
+#error "Cannot force the use of the X1 and X2 decoders at the same time!"
+#endif
+
+/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is
+ * supported at runtime, so we can add the BMI2 target attribute.
+ * When it is disabled, we will still get BMI2 if it is enabled statically.
+ */
+#if DYNAMIC_BMI2
+# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
+#else
+# define HUF_FAST_BMI2_ATTRS
+#endif
+
+#ifdef __cplusplus
+# define HUF_EXTERN_C extern "C"
+#else
+# define HUF_EXTERN_C
+#endif
+#define HUF_ASM_DECL HUF_EXTERN_C
+
+#if DYNAMIC_BMI2
+# define HUF_NEED_BMI2_FUNCTION 1
+#else
+# define HUF_NEED_BMI2_FUNCTION 0
+#endif
+
+/* **************************************************************
+*  Error Management
+****************************************************************/
+#define HUF_isError ERR_isError
+
+
+/* **************************************************************
+*  Byte alignment for workSpace management
+****************************************************************/
+#define HUF_ALIGN(x, a)         HUF_ALIGN_MASK((x), (a) - 1)
+#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
+
+
+/* **************************************************************
+*  BMI2 Variant Wrappers
+****************************************************************/
+typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize,
+                                              const void *cSrc,
+                                              size_t cSrcSize,
+                                              const HUF_DTable *DTable);
+
+#if DYNAMIC_BMI2
+
+#define HUF_DGEN(fn)                                                        \
+                                                                            \
+    static size_t fn##_default(                                             \
+                  void* dst,  size_t dstSize,                               \
+            const void* cSrc, size_t cSrcSize,                              \
+            const HUF_DTable* DTable)                                       \
+    {                                                                       \
+        return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
+    }                                                                       \
+                                                                            \
+    static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2(                          \
+                  void* dst,  size_t dstSize,                               \
+            const void* cSrc, size_t cSrcSize,                              \
+            const HUF_DTable* DTable)                                       \
+    {                                                                       \
+        return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
+    }                                                                       \
+                                                                            \
+    static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
+    {                                                                       \
+        if (flags & HUF_flags_bmi2) {                                       \
+            return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);         \
+        }                                                                   \
+        return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable);          \
+    }
+
+#else
+
+#define HUF_DGEN(fn)                                                        \
+    static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
+    {                                                                       \
+        (void)flags;                                                        \
+        return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
+    }
+
+#endif
+
+
+/*-***************************/
+/*  generic DTableDesc       */
+/*-***************************/
+typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
+
+static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
+{
+    DTableDesc dtd;
+    ZSTD_memcpy(&dtd, table, sizeof(dtd));
+    return dtd;
+}
+
+static size_t HUF_initFastDStream(BYTE const* ip) {
+    BYTE const lastByte = ip[7];
+    size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
+    size_t const value = MEM_readLEST(ip) | 1;
+    assert(bitsConsumed <= 8);
+    assert(sizeof(size_t) == 8);
+    return value << bitsConsumed;
+}
+
+
+/**
+ * The input/output arguments to the Huffman fast decoding loop:
+ *
+ * ip [in/out] - The input pointers, must be updated to reflect what is consumed.
+ * op [in/out] - The output pointers, must be updated to reflect what is written.
+ * bits [in/out] - The bitstream containers, must be updated to reflect the current state.
+ * dt [in] - The decoding table.
+ * ilowest [in] - The beginning of the valid range of the input. Decoders may read
+ *                down to this pointer. It may be below iend[0].
+ * oend [in] - The end of the output stream. op[3] must not cross oend.
+ * iend [in] - The end of each input stream. ip[i] may cross iend[i],
+ *             as long as it is above ilowest, but that indicates corruption.
+ */
+typedef struct {
+    BYTE const* ip[4];
+    BYTE* op[4];
+    U64 bits[4];
+    void const* dt;
+    BYTE const* ilowest;
+    BYTE* oend;
+    BYTE const* iend[4];
+} HUF_DecompressFastArgs;
+
+typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*);
+
+/**
+ * Initializes args for the fast decoding loop.
+ * @returns 1 on success
+ *          0 if the fallback implementation should be used.
+ *          Or an error code on failure.
+ */
+static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
+{
+    void const* dt = DTable + 1;
+    U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
+
+    const BYTE* const istart = (const BYTE*)src;
+
+    BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize);
+
+    /* The fast decoding loop assumes 64-bit little-endian.
+     * This condition is false on x32.
+     */
+    if (!MEM_isLittleEndian() || MEM_32bits())
+        return 0;
+
+    /* Avoid nullptr addition */
+    if (dstSize == 0)
+        return 0;
+    assert(dst != NULL);
+
+    /* strict minimum : jump table + 1 byte per stream */
+    if (srcSize < 10)
+        return ERROR(corruption_detected);
+
+    /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers.
+     * If table log is not correct at this point, fallback to the old decoder.
+     * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
+     */
+    if (dtLog != HUF_DECODER_FAST_TABLELOG)
+        return 0;
+
+    /* Read the jump table. */
+    {
+        size_t const length1 = MEM_readLE16(istart);
+        size_t const length2 = MEM_readLE16(istart+2);
+        size_t const length3 = MEM_readLE16(istart+4);
+        size_t const length4 = srcSize - (length1 + length2 + length3 + 6);
+        args->iend[0] = istart + 6;  /* jumpTable */
+        args->iend[1] = args->iend[0] + length1;
+        args->iend[2] = args->iend[1] + length2;
+        args->iend[3] = args->iend[2] + length3;
+
+        /* HUF_initFastDStream() requires this, and this small of an input
+         * won't benefit from the ASM loop anyways.
+         */
+        if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8)
+            return 0;
+        if (length4 > srcSize) return ERROR(corruption_detected);   /* overflow */
+    }
+    /* ip[] contains the position that is currently loaded into bits[]. */
+    args->ip[0] = args->iend[1] - sizeof(U64);
+    args->ip[1] = args->iend[2] - sizeof(U64);
+    args->ip[2] = args->iend[3] - sizeof(U64);
+    args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64);
+
+    /* op[] contains the output pointers. */
+    args->op[0] = (BYTE*)dst;
+    args->op[1] = args->op[0] + (dstSize+3)/4;
+    args->op[2] = args->op[1] + (dstSize+3)/4;
+    args->op[3] = args->op[2] + (dstSize+3)/4;
+
+    /* No point to call the ASM loop for tiny outputs. */
+    if (args->op[3] >= oend)
+        return 0;
+
+    /* bits[] is the bit container.
+        * It is read from the MSB down to the LSB.
+        * It is shifted left as it is read, and zeros are
+        * shifted in. After the lowest valid bit a 1 is
+        * set, so that CountTrailingZeros(bits[]) can be used
+        * to count how many bits we've consumed.
+        */
+    args->bits[0] = HUF_initFastDStream(args->ip[0]);
+    args->bits[1] = HUF_initFastDStream(args->ip[1]);
+    args->bits[2] = HUF_initFastDStream(args->ip[2]);
+    args->bits[3] = HUF_initFastDStream(args->ip[3]);
+
+    /* The decoders must be sure to never read beyond ilowest.
+     * This is lower than iend[0], but allowing decoders to read
+     * down to ilowest can allow an extra iteration or two in the
+     * fast loop.
+     */
+    args->ilowest = istart;
+
+    args->oend = oend;
+    args->dt = dt;
+
+    return 1;
+}
+
+static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd)
+{
+    /* Validate that we haven't overwritten. */
+    if (args->op[stream] > segmentEnd)
+        return ERROR(corruption_detected);
+    /* Validate that we haven't read beyond iend[].
+        * Note that ip[] may be < iend[] because the MSB is
+        * the next bit to read, and we may have consumed 100%
+        * of the stream, so down to iend[i] - 8 is valid.
+        */
+    if (args->ip[stream] < args->iend[stream] - 8)
+        return ERROR(corruption_detected);
+
+    /* Construct the BIT_DStream_t. */
+    assert(sizeof(size_t) == 8);
+    bit->bitContainer = MEM_readLEST(args->ip[stream]);
+    bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]);
+    bit->start = (const char*)args->ilowest;
+    bit->limitPtr = bit->start + sizeof(size_t);
+    bit->ptr = (const char*)args->ip[stream];
+
+    return 0;
+}
+
+/* Calls X(N) for each stream 0, 1, 2, 3. */
+#define HUF_4X_FOR_EACH_STREAM(X) \
+    do {                          \
+        X(0);                     \
+        X(1);                     \
+        X(2);                     \
+        X(3);                     \
+    } while (0)
+
+/* Calls X(N, var) for each stream 0, 1, 2, 3. */
+#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \
+    do {                                        \
+        X(0, (var));                            \
+        X(1, (var));                            \
+        X(2, (var));                            \
+        X(3, (var));                            \
+    } while (0)
+
+
+#ifndef HUF_FORCE_DECOMPRESS_X2
+
+/*-***************************/
+/*  single-symbol decoding   */
+/*-***************************/
+typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1;   /* single-symbol decoding */
+
+/**
+ * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at
+ * a time.
+ */
+static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
+    U64 D4;
+    if (MEM_isLittleEndian()) {
+        D4 = (U64)((symbol << 8) + nbBits);
+    } else {
+        D4 = (U64)(symbol + (nbBits << 8));
+    }
+    assert(D4 < (1U << 16));
+    D4 *= 0x0001000100010001ULL;
+    return D4;
+}
+
+/**
+ * Increase the tableLog to targetTableLog and rescales the stats.
+ * If tableLog > targetTableLog this is a no-op.
+ * @returns New tableLog
+ */
+static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog)
+{
+    if (tableLog > targetTableLog)
+        return tableLog;
+    if (tableLog < targetTableLog) {
+        U32 const scale = targetTableLog - tableLog;
+        U32 s;
+        /* Increase the weight for all non-zero probability symbols by scale. */
+        for (s = 0; s < nbSymbols; ++s) {
+            huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale);
+        }
+        /* Update rankVal to reflect the new weights.
+         * All weights except 0 get moved to weight + scale.
+         * Weights [1, scale] are empty.
+         */
+        for (s = targetTableLog; s > scale; --s) {
+            rankVal[s] = rankVal[s - scale];
+        }
+        for (s = scale; s > 0; --s) {
+            rankVal[s] = 0;
+        }
+    }
+    return targetTableLog;
+}
+
+typedef struct {
+        U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
+        U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1];
+        U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+        BYTE symbols[HUF_SYMBOLVALUE_MAX + 1];
+        BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
+} HUF_ReadDTableX1_Workspace;
+
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags)
+{
+    U32 tableLog = 0;
+    U32 nbSymbols = 0;
+    size_t iSize;
+    void* const dtPtr = DTable + 1;
+    HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr;
+    HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace;
+
+    DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp));
+    if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge);
+
+    DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
+    /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */   /* is not necessary, even though some analyzer complain ... */
+
+    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags);
+    if (HUF_isError(iSize)) return iSize;
+
+
+    /* Table header */
+    {   DTableDesc dtd = HUF_getDTableDesc(DTable);
+        U32 const maxTableLog = dtd.maxTableLog + 1;
+        U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG);
+        tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog);
+        if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge);   /* DTable too small, Huffman tree cannot fit in */
+        dtd.tableType = 0;
+        dtd.tableLog = (BYTE)tableLog;
+        ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
+    }
+
+    /* Compute symbols and rankStart given rankVal:
+     *
+     * rankVal already contains the number of values of each weight.
+     *
+     * symbols contains the symbols ordered by weight. First are the rankVal[0]
+     * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on.
+     * symbols[0] is filled (but unused) to avoid a branch.
+     *
+     * rankStart contains the offset where each rank belongs in the DTable.
+     * rankStart[0] is not filled because there are no entries in the table for
+     * weight 0.
+     */
+    {   int n;
+        U32 nextRankStart = 0;
+        int const unroll = 4;
+        int const nLimit = (int)nbSymbols - unroll + 1;
+        for (n=0; n<(int)tableLog+1; n++) {
+            U32 const curr = nextRankStart;
+            nextRankStart += wksp->rankVal[n];
+            wksp->rankStart[n] = curr;
+        }
+        for (n=0; n < nLimit; n += unroll) {
+            int u;
+            for (u=0; u < unroll; ++u) {
+                size_t const w = wksp->huffWeight[n+u];
+                wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u);
+            }
+        }
+        for (; n < (int)nbSymbols; ++n) {
+            size_t const w = wksp->huffWeight[n];
+            wksp->symbols[wksp->rankStart[w]++] = (BYTE)n;
+        }
+    }
+
+    /* fill DTable
+     * We fill all entries of each weight in order.
+     * That way length is a constant for each iteration of the outer loop.
+     * We can switch based on the length to a different inner loop which is
+     * optimized for that particular case.
+     */
+    {   U32 w;
+        int symbol = wksp->rankVal[0];
+        int rankStart = 0;
+        for (w=1; w<tableLog+1; ++w) {
+            int const symbolCount = wksp->rankVal[w];
+            int const length = (1 << w) >> 1;
+            int uStart = rankStart;
+            BYTE const nbBits = (BYTE)(tableLog + 1 - w);
+            int s;
+            int u;
+            switch (length) {
+            case 1:
+                for (s=0; s<symbolCount; ++s) {
+                    HUF_DEltX1 D;
+                    D.byte = wksp->symbols[symbol + s];
+                    D.nbBits = nbBits;
+                    dt[uStart] = D;
+                    uStart += 1;
+                }
+                break;
+            case 2:
+                for (s=0; s<symbolCount; ++s) {
+                    HUF_DEltX1 D;
+                    D.byte = wksp->symbols[symbol + s];
+                    D.nbBits = nbBits;
+                    dt[uStart+0] = D;
+                    dt[uStart+1] = D;
+                    uStart += 2;
+                }
+                break;
+            case 4:
+                for (s=0; s<symbolCount; ++s) {
+                    U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+                    MEM_write64(dt + uStart, D4);
+                    uStart += 4;
+                }
+                break;
+            case 8:
+                for (s=0; s<symbolCount; ++s) {
+                    U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+                    MEM_write64(dt + uStart, D4);
+                    MEM_write64(dt + uStart + 4, D4);
+                    uStart += 8;
+                }
+                break;
+            default:
+                for (s=0; s<symbolCount; ++s) {
+                    U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+                    for (u=0; u < length; u += 16) {
+                        MEM_write64(dt + uStart + u + 0, D4);
+                        MEM_write64(dt + uStart + u + 4, D4);
+                        MEM_write64(dt + uStart + u + 8, D4);
+                        MEM_write64(dt + uStart + u + 12, D4);
+                    }
+                    assert(u == length);
+                    uStart += length;
+                }
+                break;
+            }
+            symbol += symbolCount;
+            rankStart += symbolCount * length;
+        }
+    }
+    return iSize;
+}
+
+FORCE_INLINE_TEMPLATE BYTE
+HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog)
+{
+    size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
+    BYTE const c = dt[val].byte;
+    BIT_skipBits(Dstream, dt[val].nbBits);
+    return c;
+}
+
+#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \
+    do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0)
+
+#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr)      \
+    do {                                            \
+        if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
+            HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \
+    } while (0)
+
+#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr)      \
+    do {                                            \
+        if (MEM_64bits())                           \
+            HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \
+    } while (0)
+
+HINT_INLINE size_t
+HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog)
+{
+    BYTE* const pStart = p;
+
+    /* up to 4 symbols at a time */
+    if ((pEnd - p) > 3) {
+        while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) {
+            HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
+            HUF_DECODE_SYMBOLX1_1(p, bitDPtr);
+            HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
+            HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+        }
+    } else {
+        BIT_reloadDStream(bitDPtr);
+    }
+
+    /* [0-3] symbols remaining */
+    if (MEM_32bits())
+        while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd))
+            HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+
+    /* no more data to retrieve from bitstream, no need to reload */
+    while (p < pEnd)
+        HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+
+    return (size_t)(pEnd-pStart);
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress1X1_usingDTable_internal_body(
+          void* dst,  size_t dstSize,
+    const void* cSrc, size_t cSrcSize,
+    const HUF_DTable* DTable)
+{
+    BYTE* op = (BYTE*)dst;
+    BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize);
+    const void* dtPtr = DTable + 1;
+    const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
+    BIT_DStream_t bitD;
+    DTableDesc const dtd = HUF_getDTableDesc(DTable);
+    U32 const dtLog = dtd.tableLog;
+
+    CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
+
+    HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog);
+
+    if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
+
+    return dstSize;
+}
+
+/* HUF_decompress4X1_usingDTable_internal_body():
+ * Conditions :
+ * @dstSize >= 6
+ */
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress4X1_usingDTable_internal_body(
+          void* dst,  size_t dstSize,
+    const void* cSrc, size_t cSrcSize,
+    const HUF_DTable* DTable)
+{
+    /* Check */
+    if (cSrcSize < 10) return ERROR(corruption_detected);  /* strict minimum : jump table + 1 byte per stream */
+    if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
+
+    {   const BYTE* const istart = (const BYTE*) cSrc;
+        BYTE* const ostart = (BYTE*) dst;
+        BYTE* const oend = ostart + dstSize;
+        BYTE* const olimit = oend - 3;
+        const void* const dtPtr = DTable + 1;
+        const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
+
+        /* Init */
+        BIT_DStream_t bitD1;
+        BIT_DStream_t bitD2;
+        BIT_DStream_t bitD3;
+        BIT_DStream_t bitD4;
+        size_t const length1 = MEM_readLE16(istart);
+        size_t const length2 = MEM_readLE16(istart+2);
+        size_t const length3 = MEM_readLE16(istart+4);
+        size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+        const BYTE* const istart1 = istart + 6;  /* jumpTable */
+        const BYTE* const istart2 = istart1 + length1;
+        const BYTE* const istart3 = istart2 + length2;
+        const BYTE* const istart4 = istart3 + length3;
+        const size_t segmentSize = (dstSize+3) / 4;
+        BYTE* const opStart2 = ostart + segmentSize;
+        BYTE* const opStart3 = opStart2 + segmentSize;
+        BYTE* const opStart4 = opStart3 + segmentSize;
+        BYTE* op1 = ostart;
+        BYTE* op2 = opStart2;
+        BYTE* op3 = opStart3;
+        BYTE* op4 = opStart4;
+        DTableDesc const dtd = HUF_getDTableDesc(DTable);
+        U32 const dtLog = dtd.tableLog;
+        U32 endSignal = 1;
+
+        if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
+        if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        assert(dstSize >= 6); /* validated above */
+        CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
+        CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
+        CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
+        CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
+
+        /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */
+        if ((size_t)(oend - op4) >= sizeof(size_t)) {
+            for ( ; (endSignal) & (op4 < olimit) ; ) {
+                HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
+                HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
+                HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
+                HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
+                HUF_DECODE_SYMBOLX1_1(op1, &bitD1);
+                HUF_DECODE_SYMBOLX1_1(op2, &bitD2);
+                HUF_DECODE_SYMBOLX1_1(op3, &bitD3);
+                HUF_DECODE_SYMBOLX1_1(op4, &bitD4);
+                HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
+                HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
+                HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
+                HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
+                HUF_DECODE_SYMBOLX1_0(op1, &bitD1);
+                HUF_DECODE_SYMBOLX1_0(op2, &bitD2);
+                HUF_DECODE_SYMBOLX1_0(op3, &bitD3);
+                HUF_DECODE_SYMBOLX1_0(op4, &bitD4);
+                endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
+                endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
+                endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
+                endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
+            }
+        }
+
+        /* check corruption */
+        /* note : should not be necessary : op# advance in lock step, and we control op4.
+         *        but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */
+        if (op1 > opStart2) return ERROR(corruption_detected);
+        if (op2 > opStart3) return ERROR(corruption_detected);
+        if (op3 > opStart4) return ERROR(corruption_detected);
+        /* note : op4 supposed already verified within main loop */
+
+        /* finish bitStreams one by one */
+        HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog);
+        HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog);
+        HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog);
+        HUF_decodeStreamX1(op4, &bitD4, oend,     dt, dtLog);
+
+        /* check */
+        { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+          if (!endCheck) return ERROR(corruption_detected); }
+
+        /* decoded size */
+        return dstSize;
+    }
+}
+
+#if HUF_NEED_BMI2_FUNCTION
+static BMI2_TARGET_ATTRIBUTE
+size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
+                    size_t cSrcSize, HUF_DTable const* DTable) {
+    return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+#endif
+
+static
+size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
+                    size_t cSrcSize, HUF_DTable const* DTable) {
+    return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2
+
+HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    U16 const* const dtable = (U16 const*)args->dt;
+    BYTE* const oend = args->oend;
+    BYTE const* const ilowest = args->ilowest;
+
+    /* Copy the arguments to local variables */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= (stream == 3 ? oend : op[stream + 1]));
+            assert(ip[stream] >= ilowest);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each iteration produces 5 output symbols per stream */
+            size_t const oiters = (size_t)(oend - op[3]) / 5;
+            /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes
+             * per stream.
+             */
+            size_t const iiters = (size_t)(ip[0] - ilowest) / 7;
+            /* We can safely run iters iterations before running bounds checks */
+            size_t const iters = MIN(oiters, iiters);
+            size_t const symbols = iters * 5;
+
+            /* We can simply check that op[3] < olimit, instead of checking all
+             * of our bounds, since we can't hit the other bounds until we've run
+             * iters iterations, which only happens when op[3] == olimit.
+             */
+            olimit = op[3] + symbols;
+
+            /* Exit fast decoding loop once we reach the end. */
+            if (op[3] == olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    goto _out;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol)                 \
+    do {                                                        \
+        int const index = (int)(bits[(_stream)] >> 53);         \
+        int const entry = (int)dtable[index];                   \
+        bits[(_stream)] <<= (entry & 0x3F);                     \
+        op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \
+    } while (0)
+
+#define HUF_4X1_RELOAD_STREAM(_stream)                              \
+    do {                                                            \
+        int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
+        int const nbBits = ctz & 7;                                 \
+        int const nbBytes = ctz >> 3;                               \
+        op[(_stream)] += 5;                                         \
+        ip[(_stream)] -= nbBytes;                                   \
+        bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1;            \
+        bits[(_stream)] <<= nbBits;                                 \
+    } while (0)
+
+        /* Manually unroll the loop because compilers don't consistently
+         * unroll the inner loops, which destroys performance.
+         */
+        do {
+            /* Decode 5 symbols in each of the 4 streams */
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4);
+
+            /* Reload each of the 4 the bitstreams */
+            HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM);
+        } while (op[3] < olimit);
+
+#undef HUF_4X1_DECODE_SYMBOL
+#undef HUF_4X1_RELOAD_STREAM
+    }
+
+_out:
+
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+/**
+ * @returns @p dstSize on success (>= 6)
+ *          0 if the fallback implementation should be used
+ *          An error if an error occurred
+ */
+static HUF_FAST_BMI2_ATTRS
+size_t
+HUF_decompress4X1_usingDTable_internal_fast(
+          void* dst,  size_t dstSize,
+    const void* cSrc, size_t cSrcSize,
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn)
+{
+    void const* dt = DTable + 1;
+    BYTE const* const ilowest = (BYTE const*)cSrc;
+    BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize);
+    HUF_DecompressFastArgs args;
+    {   size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
+        if (ret == 0)
+            return 0;
+    }
+
+    assert(args.ip[0] >= args.ilowest);
+    loopFn(&args);
+
+    /* Our loop guarantees that ip[] >= ilowest and that we haven't
+    * overwritten any op[].
+    */
+    assert(args.ip[0] >= ilowest);
+    assert(args.ip[0] >= ilowest);
+    assert(args.ip[1] >= ilowest);
+    assert(args.ip[2] >= ilowest);
+    assert(args.ip[3] >= ilowest);
+    assert(args.op[3] <= oend);
+
+    assert(ilowest == args.ilowest);
+    assert(ilowest + 6 == args.iend[0]);
+    (void)ilowest;
+
+    /* finish bit streams one by one. */
+    {   size_t const segmentSize = (dstSize+3) / 4;
+        BYTE* segmentEnd = (BYTE*)dst;
+        int i;
+        for (i = 0; i < 4; ++i) {
+            BIT_DStream_t bit;
+            if (segmentSize <= (size_t)(oend - segmentEnd))
+                segmentEnd += segmentSize;
+            else
+                segmentEnd = oend;
+            FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
+            /* Decompress and validate that we've produced exactly the expected length. */
+            args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG);
+            if (args.op[i] != segmentEnd) return ERROR(corruption_detected);
+        }
+    }
+
+    /* decoded size */
+    assert(dstSize != 0);
+    return dstSize;
+}
+
+HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
+
+static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
+{
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop;
+
+#if DYNAMIC_BMI2
+    if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2;
+# if ZSTD_ENABLE_ASM_X86_64_BMI2
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+        }
+# endif
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+    }
+#endif
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+    }
+#endif
+
+    if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+                                   const void* cSrc, size_t cSrcSize,
+                                   void* workSpace, size_t wkspSize, int flags)
+{
+    const BYTE* ip = (const BYTE*) cSrc;
+
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
+    if (HUF_isError(hSize)) return hSize;
+    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+    ip += hSize; cSrcSize -= hSize;
+
+    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+
+#endif /* HUF_FORCE_DECOMPRESS_X2 */
+
+
+#ifndef HUF_FORCE_DECOMPRESS_X1
+
+/* *************************/
+/* double-symbols decoding */
+/* *************************/
+
+typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2;  /* double-symbols decoding */
+typedef struct { BYTE symbol; } sortedSymbol_t;
+typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
+typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
+
+/**
+ * Constructs a HUF_DEltX2 in a U32.
+ */
+static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level)
+{
+    U32 seq;
+    DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0);
+    DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2);
+    DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3);
+    DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32));
+    if (MEM_isLittleEndian()) {
+        seq = level == 1 ? symbol : (baseSeq + (symbol << 8));
+        return seq + (nbBits << 16) + ((U32)level << 24);
+    } else {
+        seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol);
+        return (seq << 16) + (nbBits << 8) + (U32)level;
+    }
+}
+
+/**
+ * Constructs a HUF_DEltX2.
+ */
+static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level)
+{
+    HUF_DEltX2 DElt;
+    U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
+    DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val));
+    ZSTD_memcpy(&DElt, &val, sizeof(val));
+    return DElt;
+}
+
+/**
+ * Constructs 2 HUF_DEltX2s and packs them into a U64.
+ */
+static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level)
+{
+    U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
+    return (U64)DElt + ((U64)DElt << 32);
+}
+
+/**
+ * Fills the DTable rank with all the symbols from [begin, end) that are each
+ * nbBits long.
+ *
+ * @param DTableRank The start of the rank in the DTable.
+ * @param begin The first symbol to fill (inclusive).
+ * @param end The last symbol to fill (exclusive).
+ * @param nbBits Each symbol is nbBits long.
+ * @param tableLog The table log.
+ * @param baseSeq If level == 1 { 0 } else { the first level symbol }
+ * @param level The level in the table. Must be 1 or 2.
+ */
+static void HUF_fillDTableX2ForWeight(
+    HUF_DEltX2* DTableRank,
+    sortedSymbol_t const* begin, sortedSymbol_t const* end,
+    U32 nbBits, U32 tableLog,
+    U16 baseSeq, int const level)
+{
+    U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */);
+    const sortedSymbol_t* ptr;
+    assert(level >= 1 && level <= 2);
+    switch (length) {
+    case 1:
+        for (ptr = begin; ptr != end; ++ptr) {
+            HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
+            *DTableRank++ = DElt;
+        }
+        break;
+    case 2:
+        for (ptr = begin; ptr != end; ++ptr) {
+            HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
+            DTableRank[0] = DElt;
+            DTableRank[1] = DElt;
+            DTableRank += 2;
+        }
+        break;
+    case 4:
+        for (ptr = begin; ptr != end; ++ptr) {
+            U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+            ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+            ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+            DTableRank += 4;
+        }
+        break;
+    case 8:
+        for (ptr = begin; ptr != end; ++ptr) {
+            U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+            ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+            ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+            ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
+            ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
+            DTableRank += 8;
+        }
+        break;
+    default:
+        for (ptr = begin; ptr != end; ++ptr) {
+            U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+            HUF_DEltX2* const DTableRankEnd = DTableRank + length;
+            for (; DTableRank != DTableRankEnd; DTableRank += 8) {
+                ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+                ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+                ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
+                ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
+            }
+        }
+        break;
+    }
+}
+
+/* HUF_fillDTableX2Level2() :
+ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
+static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits,
+                           const U32* rankVal, const int minWeight, const int maxWeight1,
+                           const sortedSymbol_t* sortedSymbols, U32 const* rankStart,
+                           U32 nbBitsBaseline, U16 baseSeq)
+{
+    /* Fill skipped values (all positions up to rankVal[minWeight]).
+     * These are positions only get a single symbol because the combined weight
+     * is too large.
+     */
+    if (minWeight>1) {
+        U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */);
+        U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1);
+        int const skipSize = rankVal[minWeight];
+        assert(length > 1);
+        assert((U32)skipSize < length);
+        switch (length) {
+        case 2:
+            assert(skipSize == 1);
+            ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2));
+            break;
+        case 4:
+            assert(skipSize <= 4);
+            ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2));
+            ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2));
+            break;
+        default:
+            {
+                int i;
+                for (i = 0; i < skipSize; i += 8) {
+                    ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2));
+                    ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2));
+                    ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2));
+                    ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2));
+                }
+            }
+        }
+    }
+
+    /* Fill each of the second level symbols by weight. */
+    {
+        int w;
+        for (w = minWeight; w < maxWeight1; ++w) {
+            int const begin = rankStart[w];
+            int const end = rankStart[w+1];
+            U32 const nbBits = nbBitsBaseline - w;
+            U32 const totalBits = nbBits + consumedBits;
+            HUF_fillDTableX2ForWeight(
+                DTable + rankVal[w],
+                sortedSymbols + begin, sortedSymbols + end,
+                totalBits, targetLog,
+                baseSeq, /* level */ 2);
+        }
+    }
+}
+
+static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
+                           const sortedSymbol_t* sortedList,
+                           const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight,
+                           const U32 nbBitsBaseline)
+{
+    U32* const rankVal = rankValOrigin[0];
+    const int scaleLog = nbBitsBaseline - targetLog;   /* note : targetLog >= srcLog, hence scaleLog <= 1 */
+    const U32 minBits  = nbBitsBaseline - maxWeight;
+    int w;
+    int const wEnd = (int)maxWeight + 1;
+
+    /* Fill DTable in order of weight. */
+    for (w = 1; w < wEnd; ++w) {
+        int const begin = (int)rankStart[w];
+        int const end = (int)rankStart[w+1];
+        U32 const nbBits = nbBitsBaseline - w;
+
+        if (targetLog-nbBits >= minBits) {
+            /* Enough room for a second symbol. */
+            int start = rankVal[w];
+            U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */);
+            int minWeight = nbBits + scaleLog;
+            int s;
+            if (minWeight < 1) minWeight = 1;
+            /* Fill the DTable for every symbol of weight w.
+             * These symbols get at least 1 second symbol.
+             */
+            for (s = begin; s != end; ++s) {
+                HUF_fillDTableX2Level2(
+                    DTable + start, targetLog, nbBits,
+                    rankValOrigin[nbBits], minWeight, wEnd,
+                    sortedList, rankStart,
+                    nbBitsBaseline, sortedList[s].symbol);
+                start += length;
+            }
+        } else {
+            /* Only a single symbol. */
+            HUF_fillDTableX2ForWeight(
+                DTable + rankVal[w],
+                sortedList + begin, sortedList + end,
+                nbBits, targetLog,
+                /* baseSeq */ 0, /* level */ 1);
+        }
+    }
+}
+
+typedef struct {
+    rankValCol_t rankVal[HUF_TABLELOG_MAX];
+    U32 rankStats[HUF_TABLELOG_MAX + 1];
+    U32 rankStart0[HUF_TABLELOG_MAX + 3];
+    sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
+    BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
+    U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+} HUF_ReadDTableX2_Workspace;
+
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
+                       const void* src, size_t srcSize,
+                             void* workSpace, size_t wkspSize, int flags)
+{
+    U32 tableLog, maxW, nbSymbols;
+    DTableDesc dtd = HUF_getDTableDesc(DTable);
+    U32 maxTableLog = dtd.maxTableLog;
+    size_t iSize;
+    void* dtPtr = DTable+1;   /* force compiler to avoid strict-aliasing */
+    HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
+    U32 *rankStart;
+
+    HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace;
+
+    if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC);
+
+    rankStart = wksp->rankStart0 + 1;
+    ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats));
+    ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0));
+
+    DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable));   /* if compiler fails here, assertion is wrong */
+    if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
+    /* ZSTD_memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
+
+    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags);
+    if (HUF_isError(iSize)) return iSize;
+
+    /* check result */
+    if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge);   /* DTable can't fit code depth */
+    if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG;
+
+    /* find maxWeight */
+    for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {}  /* necessarily finds a solution before 0 */
+
+    /* Get start index of each weight */
+    {   U32 w, nextRankStart = 0;
+        for (w=1; w<maxW+1; w++) {
+            U32 curr = nextRankStart;
+            nextRankStart += wksp->rankStats[w];
+            rankStart[w] = curr;
+        }
+        rankStart[0] = nextRankStart;   /* put all 0w symbols at the end of sorted list*/
+        rankStart[maxW+1] = nextRankStart;
+    }
+
+    /* sort symbols by weight */
+    {   U32 s;
+        for (s=0; s<nbSymbols; s++) {
+            U32 const w = wksp->weightList[s];
+            U32 const r = rankStart[w]++;
+            wksp->sortedSymbol[r].symbol = (BYTE)s;
+        }
+        rankStart[0] = 0;   /* forget 0w symbols; this is beginning of weight(1) */
+    }
+
+    /* Build rankVal */
+    {   U32* const rankVal0 = wksp->rankVal[0];
+        {   int const rescale = (maxTableLog-tableLog) - 1;   /* tableLog <= maxTableLog */
+            U32 nextRankVal = 0;
+            U32 w;
+            for (w=1; w<maxW+1; w++) {
+                U32 curr = nextRankVal;
+                nextRankVal += wksp->rankStats[w] << (w+rescale);
+                rankVal0[w] = curr;
+        }   }
+        {   U32 const minBits = tableLog+1 - maxW;
+            U32 consumed;
+            for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
+                U32* const rankValPtr = wksp->rankVal[consumed];
+                U32 w;
+                for (w = 1; w < maxW+1; w++) {
+                    rankValPtr[w] = rankVal0[w] >> consumed;
+    }   }   }   }
+
+    HUF_fillDTableX2(dt, maxTableLog,
+                   wksp->sortedSymbol,
+                   wksp->rankStart0, wksp->rankVal, maxW,
+                   tableLog+1);
+
+    dtd.tableLog = (BYTE)maxTableLog;
+    dtd.tableType = 1;
+    ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
+    return iSize;
+}
+
+
+FORCE_INLINE_TEMPLATE U32
+HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
+{
+    size_t const val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
+    ZSTD_memcpy(op, &dt[val].sequence, 2);
+    BIT_skipBits(DStream, dt[val].nbBits);
+    return dt[val].length;
+}
+
+FORCE_INLINE_TEMPLATE U32
+HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
+{
+    size_t const val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
+    ZSTD_memcpy(op, &dt[val].sequence, 1);
+    if (dt[val].length==1) {
+        BIT_skipBits(DStream, dt[val].nbBits);
+    } else {
+        if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
+            BIT_skipBits(DStream, dt[val].nbBits);
+            if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
+                /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
+                DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
+        }
+    }
+    return 1;
+}
+
+#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
+    do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0)
+
+#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr)                     \
+    do {                                                           \
+        if (MEM_64bits() || (HUF_TABLELOG_MAX<=12))                \
+            ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \
+    } while (0)
+
+#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr)                     \
+    do {                                                           \
+        if (MEM_64bits())                                          \
+            ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \
+    } while (0)
+
+HINT_INLINE size_t
+HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd,
+                const HUF_DEltX2* const dt, const U32 dtLog)
+{
+    BYTE* const pStart = p;
+
+    /* up to 8 symbols at a time */
+    if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) {
+        if (dtLog <= 11 && MEM_64bits()) {
+            /* up to 10 symbols at a time */
+            while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) {
+                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+            }
+        } else {
+            /* up to 8 symbols at a time */
+            while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
+                HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+                HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
+                HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+            }
+        }
+    } else {
+        BIT_reloadDStream(bitDPtr);
+    }
+
+    /* closer to end : up to 2 symbols at a time */
+    if ((size_t)(pEnd - p) >= 2) {
+        while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
+            HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+
+        while (p <= pEnd-2)
+            HUF_DECODE_SYMBOLX2_0(p, bitDPtr);   /* no need to reload : reached the end of DStream */
+    }
+
+    if (p < pEnd)
+        p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog);
+
+    return p-pStart;
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress1X2_usingDTable_internal_body(
+          void* dst,  size_t dstSize,
+    const void* cSrc, size_t cSrcSize,
+    const HUF_DTable* DTable)
+{
+    BIT_DStream_t bitD;
+
+    /* Init */
+    CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
+
+    /* decode */
+    {   BYTE* const ostart = (BYTE*) dst;
+        BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize);
+        const void* const dtPtr = DTable+1;   /* force compiler to not use strict-aliasing */
+        const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
+        DTableDesc const dtd = HUF_getDTableDesc(DTable);
+        HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog);
+    }
+
+    /* check */
+    if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
+
+    /* decoded size */
+    return dstSize;
+}
+
+/* HUF_decompress4X2_usingDTable_internal_body():
+ * Conditions:
+ * @dstSize >= 6
+ */
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress4X2_usingDTable_internal_body(
+          void* dst,  size_t dstSize,
+    const void* cSrc, size_t cSrcSize,
+    const HUF_DTable* DTable)
+{
+    if (cSrcSize < 10) return ERROR(corruption_detected);   /* strict minimum : jump table + 1 byte per stream */
+    if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
+
+    {   const BYTE* const istart = (const BYTE*) cSrc;
+        BYTE* const ostart = (BYTE*) dst;
+        BYTE* const oend = ostart + dstSize;
+        BYTE* const olimit = oend - (sizeof(size_t)-1);
+        const void* const dtPtr = DTable+1;
+        const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
+
+        /* Init */
+        BIT_DStream_t bitD1;
+        BIT_DStream_t bitD2;
+        BIT_DStream_t bitD3;
+        BIT_DStream_t bitD4;
+        size_t const length1 = MEM_readLE16(istart);
+        size_t const length2 = MEM_readLE16(istart+2);
+        size_t const length3 = MEM_readLE16(istart+4);
+        size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+        const BYTE* const istart1 = istart + 6;  /* jumpTable */
+        const BYTE* const istart2 = istart1 + length1;
+        const BYTE* const istart3 = istart2 + length2;
+        const BYTE* const istart4 = istart3 + length3;
+        size_t const segmentSize = (dstSize+3) / 4;
+        BYTE* const opStart2 = ostart + segmentSize;
+        BYTE* const opStart3 = opStart2 + segmentSize;
+        BYTE* const opStart4 = opStart3 + segmentSize;
+        BYTE* op1 = ostart;
+        BYTE* op2 = opStart2;
+        BYTE* op3 = opStart3;
+        BYTE* op4 = opStart4;
+        U32 endSignal = 1;
+        DTableDesc const dtd = HUF_getDTableDesc(DTable);
+        U32 const dtLog = dtd.tableLog;
+
+        if (length4 > cSrcSize) return ERROR(corruption_detected);  /* overflow */
+        if (opStart4 > oend) return ERROR(corruption_detected);     /* overflow */
+        assert(dstSize >= 6 /* validated above */);
+        CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
+        CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
+        CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
+        CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
+
+        /* 16-32 symbols per loop (4-8 symbols per stream) */
+        if ((size_t)(oend - op4) >= sizeof(size_t)) {
+            for ( ; (endSignal) & (op4 < olimit); ) {
+#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
+                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+                HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
+                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+                HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
+                endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
+                endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
+                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+                HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
+                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+                HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
+                endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
+                endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
+#else
+                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+                HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
+                HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
+                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+                HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
+                HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
+                HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
+                HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
+                endSignal = (U32)LIKELY((U32)
+                            (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished)
+                        & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished)
+                        & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished)
+                        & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished));
+#endif
+            }
+        }
+
+        /* check corruption */
+        if (op1 > opStart2) return ERROR(corruption_detected);
+        if (op2 > opStart3) return ERROR(corruption_detected);
+        if (op3 > opStart4) return ERROR(corruption_detected);
+        /* note : op4 already verified within main loop */
+
+        /* finish bitStreams one by one */
+        HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
+        HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
+        HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
+        HUF_decodeStreamX2(op4, &bitD4, oend,     dt, dtLog);
+
+        /* check */
+        { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+          if (!endCheck) return ERROR(corruption_detected); }
+
+        /* decoded size */
+        return dstSize;
+    }
+}
+
+#if HUF_NEED_BMI2_FUNCTION
+static BMI2_TARGET_ATTRIBUTE
+size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
+                    size_t cSrcSize, HUF_DTable const* DTable) {
+    return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+#endif
+
+static
+size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
+                    size_t cSrcSize, HUF_DTable const* DTable) {
+    return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2
+
+HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    BYTE* oend[4];
+    HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt;
+    BYTE const* const ilowest = args->ilowest;
+
+    /* Copy the arguments to local registers. */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    oend[0] = op[1];
+    oend[1] = op[2];
+    oend[2] = op[3];
+    oend[3] = args->oend;
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= oend[stream]);
+            assert(ip[stream] >= ilowest);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each loop does 5 table lookups for each of the 4 streams.
+             * Each table lookup consumes up to 11 bits of input, and produces
+             * up to 2 bytes of output.
+             */
+            /* We can consume up to 7 bytes of input per iteration per stream.
+             * We also know that each input pointer is >= ip[0]. So we can run
+             * iters loops before running out of input.
+             */
+            size_t iters = (size_t)(ip[0] - ilowest) / 7;
+            /* Each iteration can produce up to 10 bytes of output per stream.
+             * Each output stream my advance at different rates. So take the
+             * minimum number of safe iterations among all the output streams.
+             */
+            for (stream = 0; stream < 4; ++stream) {
+                size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10;
+                iters = MIN(iters, oiters);
+            }
+
+            /* Each iteration produces at least 5 output symbols. So until
+             * op[3] crosses olimit, we know we haven't executed iters
+             * iterations yet. This saves us maintaining an iters counter,
+             * at the expense of computing the remaining # of iterations
+             * more frequently.
+             */
+            olimit = op[3] + (iters * 5);
+
+            /* Exit the fast decoding loop once we reach the end. */
+            if (op[3] == olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    goto _out;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3)                      \
+    do {                                                              \
+        if ((_decode3) || (_stream) != 3) {                           \
+            int const index = (int)(bits[(_stream)] >> 53);           \
+            HUF_DEltX2 const entry = dtable[index];                   \
+            MEM_write16(op[(_stream)], entry.sequence); \
+            bits[(_stream)] <<= (entry.nbBits) & 0x3F;                \
+            op[(_stream)] += (entry.length);                          \
+        }                                                             \
+    } while (0)
+
+#define HUF_4X2_RELOAD_STREAM(_stream)                                  \
+    do {                                                                \
+        HUF_4X2_DECODE_SYMBOL(3, 1);                                    \
+        {                                                               \
+            int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
+            int const nbBits = ctz & 7;                                 \
+            int const nbBytes = ctz >> 3;                               \
+            ip[(_stream)] -= nbBytes;                                   \
+            bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1;            \
+            bits[(_stream)] <<= nbBits;                                 \
+        }                                                               \
+    } while (0)
+
+        /* Manually unroll the loop because compilers don't consistently
+         * unroll the inner loops, which destroys performance.
+         */
+        do {
+            /* Decode 5 symbols from each of the first 3 streams.
+             * The final stream will be decoded during the reload phase
+             * to reduce register pressure.
+             */
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
+
+            /* Decode one symbol from the final stream */
+            HUF_4X2_DECODE_SYMBOL(3, 1);
+
+            /* Decode 4 symbols from the final stream & reload bitstreams.
+             * The final stream is reloaded last, meaning that all 5 symbols
+             * are decoded from the final stream before it is reloaded.
+             */
+            HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM);
+        } while (op[3] < olimit);
+    }
+
+#undef HUF_4X2_DECODE_SYMBOL
+#undef HUF_4X2_RELOAD_STREAM
+
+_out:
+
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+
+static HUF_FAST_BMI2_ATTRS size_t
+HUF_decompress4X2_usingDTable_internal_fast(
+          void* dst,  size_t dstSize,
+    const void* cSrc, size_t cSrcSize,
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn) {
+    void const* dt = DTable + 1;
+    const BYTE* const ilowest = (const BYTE*)cSrc;
+    BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize);
+    HUF_DecompressFastArgs args;
+    {
+        size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        FORWARD_IF_ERROR(ret, "Failed to init asm args");
+        if (ret == 0)
+            return 0;
+    }
+
+    assert(args.ip[0] >= args.ilowest);
+    loopFn(&args);
+
+    /* note : op4 already verified within main loop */
+    assert(args.ip[0] >= ilowest);
+    assert(args.ip[1] >= ilowest);
+    assert(args.ip[2] >= ilowest);
+    assert(args.ip[3] >= ilowest);
+    assert(args.op[3] <= oend);
+
+    assert(ilowest == args.ilowest);
+    assert(ilowest + 6 == args.iend[0]);
+    (void)ilowest;
+
+    /* finish bitStreams one by one */
+    {
+        size_t const segmentSize = (dstSize+3) / 4;
+        BYTE* segmentEnd = (BYTE*)dst;
+        int i;
+        for (i = 0; i < 4; ++i) {
+            BIT_DStream_t bit;
+            if (segmentSize <= (size_t)(oend - segmentEnd))
+                segmentEnd += segmentSize;
+            else
+                segmentEnd = oend;
+            FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
+            args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG);
+            if (args.op[i] != segmentEnd)
+                return ERROR(corruption_detected);
+        }
+    }
+
+    /* decoded size */
+    return dstSize;
+}
+
+static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
+{
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop;
+
+#if DYNAMIC_BMI2
+    if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2;
+# if ZSTD_ENABLE_ASM_X86_64_BMI2
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+        }
+# endif
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+    }
+#endif
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+    }
+#endif
+
+    if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
+
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
+                                   const void* cSrc, size_t cSrcSize,
+                                   void* workSpace, size_t wkspSize, int flags)
+{
+    const BYTE* ip = (const BYTE*) cSrc;
+
+    size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
+                                               workSpace, wkspSize, flags);
+    if (HUF_isError(hSize)) return hSize;
+    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+    ip += hSize; cSrcSize -= hSize;
+
+    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags);
+}
+
+static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+                                   const void* cSrc, size_t cSrcSize,
+                                   void* workSpace, size_t wkspSize, int flags)
+{
+    const BYTE* ip = (const BYTE*) cSrc;
+
+    size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
+                                         workSpace, wkspSize, flags);
+    if (HUF_isError(hSize)) return hSize;
+    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+    ip += hSize; cSrcSize -= hSize;
+
+    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+
+#endif /* HUF_FORCE_DECOMPRESS_X1 */
+
+
+/* ***********************************/
+/* Universal decompression selectors */
+/* ***********************************/
+
+
+#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
+typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
+static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] =
+{
+    /* single, double, quad */
+    {{0,0}, {1,1}},  /* Q==0 : impossible */
+    {{0,0}, {1,1}},  /* Q==1 : impossible */
+    {{ 150,216}, { 381,119}},   /* Q == 2 : 12-18% */
+    {{ 170,205}, { 514,112}},   /* Q == 3 : 18-25% */
+    {{ 177,199}, { 539,110}},   /* Q == 4 : 25-32% */
+    {{ 197,194}, { 644,107}},   /* Q == 5 : 32-38% */
+    {{ 221,192}, { 735,107}},   /* Q == 6 : 38-44% */
+    {{ 256,189}, { 881,106}},   /* Q == 7 : 44-50% */
+    {{ 359,188}, {1167,109}},   /* Q == 8 : 50-56% */
+    {{ 582,187}, {1570,114}},   /* Q == 9 : 56-62% */
+    {{ 688,187}, {1712,122}},   /* Q ==10 : 62-69% */
+    {{ 825,186}, {1965,136}},   /* Q ==11 : 69-75% */
+    {{ 976,185}, {2131,150}},   /* Q ==12 : 75-81% */
+    {{1180,186}, {2070,175}},   /* Q ==13 : 81-87% */
+    {{1377,185}, {1731,202}},   /* Q ==14 : 87-93% */
+    {{1412,185}, {1695,202}},   /* Q ==15 : 93-99% */
+};
+#endif
+
+/** HUF_selectDecoder() :
+ *  Tells which decoder is likely to decode faster,
+ *  based on a set of pre-computed metrics.
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
+ *  Assumption : 0 < dstSize <= 128 KB */
+U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
+{
+    assert(dstSize > 0);
+    assert(dstSize <= 128*1024);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+    (void)dstSize;
+    (void)cSrcSize;
+    return 0;
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+    (void)dstSize;
+    (void)cSrcSize;
+    return 1;
+#else
+    /* decoder timing evaluation */
+    {   U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize);   /* Q < 16 */
+        U32 const D256 = (U32)(dstSize >> 8);
+        U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
+        U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
+        DTime1 += DTime1 >> 5;  /* small advantage to algorithm using less memory, to reduce cache eviction */
+        return DTime1 < DTime0;
+    }
+#endif
+}
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+                                  const void* cSrc, size_t cSrcSize,
+                                  void* workSpace, size_t wkspSize, int flags)
+{
+    /* validation checks */
+    if (dstSize == 0) return ERROR(dstSize_tooSmall);
+    if (cSrcSize > dstSize) return ERROR(corruption_detected);   /* invalid */
+    if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; }   /* not compressed */
+    if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; }   /* RLE */
+
+    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+        (void)algoNb;
+        assert(algoNb == 0);
+        return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+        (void)algoNb;
+        assert(algoNb == 1);
+        return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags);
+#else
+        return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags):
+                        HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+                                cSrcSize, workSpace, wkspSize, flags);
+#endif
+    }
+}
+
+
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
+{
+    DTableDesc const dtd = HUF_getDTableDesc(DTable);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+    (void)dtd;
+    assert(dtd.tableType == 0);
+    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+    (void)dtd;
+    assert(dtd.tableType == 1);
+    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#else
+    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#endif
+}
+
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
+{
+    const BYTE* ip = (const BYTE*) cSrc;
+
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
+    if (HUF_isError(hSize)) return hSize;
+    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+    ip += hSize; cSrcSize -= hSize;
+
+    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+#endif
+
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
+{
+    DTableDesc const dtd = HUF_getDTableDesc(DTable);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+    (void)dtd;
+    assert(dtd.tableType == 0);
+    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+    (void)dtd;
+    assert(dtd.tableType == 1);
+    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#else
+    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#endif
+}
+
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
+{
+    /* validation checks */
+    if (dstSize == 0) return ERROR(dstSize_tooSmall);
+    if (cSrcSize == 0) return ERROR(corruption_detected);
+
+    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+        (void)algoNb;
+        assert(algoNb == 0);
+        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+        (void)algoNb;
+        assert(algoNb == 1);
+        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#else
+        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) :
+                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#endif
+    }
+}
+/**** ended inlining decompress/huf_decompress.c ****/
+/**** start inlining decompress/zstd_ddict.c ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* zstd_ddict.c :
+ * concentrates all logic that needs to know the internals of ZSTD_DDict object */
+
+/*-*******************************************************
+*  Dependencies
+*********************************************************/
+/**** start inlining ../common/allocations.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides custom allocation primitives
+ */
+
+#define ZSTD_DEPS_NEED_MALLOC
+/**** skipping file: zstd_deps.h ****/
+
+/**** skipping file: compiler.h ****/
+#define ZSTD_STATIC_LINKING_ONLY
+/**** skipping file: ../zstd.h ****/
+
+#ifndef ZSTD_ALLOCATIONS_H
+#define ZSTD_ALLOCATIONS_H
+
+/* custom memory allocation functions */
+
+MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
+{
+    if (customMem.customAlloc)
+        return customMem.customAlloc(customMem.opaque, size);
+    return ZSTD_malloc(size);
+}
+
+MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
+{
+    if (customMem.customAlloc) {
+        /* calloc implemented as malloc+memset;
+         * not as efficient as calloc, but next best guess for custom malloc */
+        void* const ptr = customMem.customAlloc(customMem.opaque, size);
+        ZSTD_memset(ptr, 0, size);
+        return ptr;
+    }
+    return ZSTD_calloc(1, size);
+}
+
+MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
+{
+    if (ptr!=NULL) {
+        if (customMem.customFree)
+            customMem.customFree(customMem.opaque, ptr);
+        else
+            ZSTD_free(ptr);
+    }
+}
+
+#endif /* ZSTD_ALLOCATIONS_H */
+/**** ended inlining ../common/allocations.h ****/
+/**** skipping file: ../common/zstd_deps.h ****/
+/**** skipping file: ../common/cpu.h ****/
+/**** skipping file: ../common/mem.h ****/
+#define FSE_STATIC_LINKING_ONLY
+/**** skipping file: ../common/fse.h ****/
+/**** skipping file: ../common/huf.h ****/
+/**** start inlining zstd_decompress_internal.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* zstd_decompress_internal:
+ * objects and definitions shared within lib/decompress modules */
+
+ #ifndef ZSTD_DECOMPRESS_INTERNAL_H
+ #define ZSTD_DECOMPRESS_INTERNAL_H
+
+
+/*-*******************************************************
+ *  Dependencies
+ *********************************************************/
+/**** skipping file: ../common/mem.h ****/
+/**** skipping file: ../common/zstd_internal.h ****/
+
+
+
+/*-*******************************************************
+ *  Constants
+ *********************************************************/
+static UNUSED_ATTR const U32 LL_base[MaxLL+1] = {
+                 0,    1,    2,     3,     4,     5,     6,      7,
+                 8,    9,   10,    11,    12,    13,    14,     15,
+                16,   18,   20,    22,    24,    28,    32,     40,
+                48,   64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
+                0x2000, 0x4000, 0x8000, 0x10000 };
+
+static UNUSED_ATTR const U32 OF_base[MaxOff+1] = {
+                 0,        1,       1,       5,     0xD,     0x1D,     0x3D,     0x7D,
+                 0xFD,   0x1FD,   0x3FD,   0x7FD,   0xFFD,   0x1FFD,   0x3FFD,   0x7FFD,
+                 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
+                 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
+
+static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = {
+                     0,  1,  2,  3,  4,  5,  6,  7,
+                     8,  9, 10, 11, 12, 13, 14, 15,
+                    16, 17, 18, 19, 20, 21, 22, 23,
+                    24, 25, 26, 27, 28, 29, 30, 31 };
+
+static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
+                     3,  4,  5,    6,     7,     8,     9,    10,
+                    11, 12, 13,   14,    15,    16,    17,    18,
+                    19, 20, 21,   22,    23,    24,    25,    26,
+                    27, 28, 29,   30,    31,    32,    33,    34,
+                    35, 37, 39,   41,    43,    47,    51,    59,
+                    67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
+                    0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
+
+
+/*-*******************************************************
+ *  Decompression types
+ *********************************************************/
+ typedef struct {
+     U32 fastMode;
+     U32 tableLog;
+ } ZSTD_seqSymbol_header;
+
+ typedef struct {
+     U16  nextState;
+     BYTE nbAdditionalBits;
+     BYTE nbBits;
+     U32  baseValue;
+ } ZSTD_seqSymbol;
+
+ #define SEQSYMBOL_TABLE_SIZE(log)   (1 + (1 << (log)))
+
+#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
+#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
+#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
+
+typedef struct {
+    ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)];    /* Note : Space reserved for FSE Tables */
+    ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)];   /* is also used as temporary workspace while building hufTable during DDict creation */
+    ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)];    /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
+    HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)];  /* can accommodate HUF_decompress4X */
+    U32 rep[ZSTD_REP_NUM];
+    U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
+} ZSTD_entropyDTables_t;
+
+typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
+               ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
+               ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
+               ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
+
+typedef enum { zdss_init=0, zdss_loadHeader,
+               zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
+
+typedef enum {
+    ZSTD_use_indefinitely = -1,  /* Use the dictionary indefinitely */
+    ZSTD_dont_use = 0,           /* Do not use the dictionary (if one exists free it) */
+    ZSTD_use_once = 1            /* Use the dictionary once and set to ZSTD_dont_use */
+} ZSTD_dictUses_e;
+
+/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
+typedef struct {
+    const ZSTD_DDict** ddictPtrTable;
+    size_t ddictPtrTableSize;
+    size_t ddictPtrCount;
+} ZSTD_DDictHashSet;
+
+#ifndef ZSTD_DECODER_INTERNAL_BUFFER
+#  define ZSTD_DECODER_INTERNAL_BUFFER  (1 << 16)
+#endif
+
+#define ZSTD_LBMIN 64
+#define ZSTD_LBMAX (128 << 10)
+
+/* extra buffer, compensates when dst is not large enough to store litBuffer */
+#define ZSTD_LITBUFFEREXTRASIZE  BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX)
+
+typedef enum {
+    ZSTD_not_in_dst = 0,  /* Stored entirely within litExtraBuffer */
+    ZSTD_in_dst = 1,           /* Stored entirely within dst (in memory after current output write) */
+    ZSTD_split = 2            /* Split between litExtraBuffer and dst */
+} ZSTD_litLocation_e;
+
+struct ZSTD_DCtx_s
+{
+    const ZSTD_seqSymbol* LLTptr;
+    const ZSTD_seqSymbol* MLTptr;
+    const ZSTD_seqSymbol* OFTptr;
+    const HUF_DTable* HUFptr;
+    ZSTD_entropyDTables_t entropy;
+    U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];   /* space needed when building huffman tables */
+    const void* previousDstEnd;   /* detect continuity */
+    const void* prefixStart;      /* start of current segment */
+    const void* virtualStart;     /* virtual start of previous segment if it was just before current one */
+    const void* dictEnd;          /* end of previous segment */
+    size_t expected;
+    ZSTD_frameHeader fParams;
+    U64 processedCSize;
+    U64 decodedSize;
+    blockType_e bType;            /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
+    ZSTD_dStage stage;
+    U32 litEntropy;
+    U32 fseEntropy;
+    XXH64_state_t xxhState;
+    size_t headerSize;
+    ZSTD_format_e format;
+    ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum;   /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */
+    U32 validateChecksum;         /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */
+    const BYTE* litPtr;
+    ZSTD_customMem customMem;
+    size_t litSize;
+    size_t rleSize;
+    size_t staticSize;
+    int isFrameDecompression;
+#if DYNAMIC_BMI2 != 0
+    int bmi2;                     /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
+#endif
+
+    /* dictionary */
+    ZSTD_DDict* ddictLocal;
+    const ZSTD_DDict* ddict;     /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
+    U32 dictID;
+    int ddictIsCold;             /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
+    ZSTD_dictUses_e dictUses;
+    ZSTD_DDictHashSet* ddictSet;                    /* Hash set for multiple ddicts */
+    ZSTD_refMultipleDDicts_e refMultipleDDicts;     /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
+    int disableHufAsm;
+    int maxBlockSizeParam;
+
+    /* streaming */
+    ZSTD_dStreamStage streamStage;
+    char*  inBuff;
+    size_t inBuffSize;
+    size_t inPos;
+    size_t maxWindowSize;
+    char*  outBuff;
+    size_t outBuffSize;
+    size_t outStart;
+    size_t outEnd;
+    size_t lhSize;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+    void* legacyContext;
+    U32 previousLegacyVersion;
+    U32 legacyVersion;
+#endif
+    U32 hostageByte;
+    int noForwardProgress;
+    ZSTD_bufferMode_e outBufferMode;
+    ZSTD_outBuffer expectedOutBuffer;
+
+    /* workspace */
+    BYTE* litBuffer;
+    const BYTE* litBufferEnd;
+    ZSTD_litLocation_e litBufferLocation;
+    BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
+    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+
+    size_t oversizedDuration;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    void const* dictContentBeginForFuzzing;
+    void const* dictContentEndForFuzzing;
+#endif
+
+    /* Tracing */
+#if ZSTD_TRACE
+    ZSTD_TraceCtx traceCtx;
+#endif
+};  /* typedef'd to ZSTD_DCtx within "zstd.h" */
+
+MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
+#if DYNAMIC_BMI2 != 0
+	return dctx->bmi2;
+#else
+    (void)dctx;
+	return 0;
+#endif
+}
+
+/*-*******************************************************
+ *  Shared internal functions
+ *********************************************************/
+
+/*! ZSTD_loadDEntropy() :
+ *  dict : must point at beginning of a valid zstd dictionary.
+ * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
+size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
+                   const void* const dict, size_t const dictSize);
+
+/*! ZSTD_checkContinuity() :
+ *  check if next `dst` follows previous position, where decompression ended.
+ *  If yes, do nothing (continue on current segment).
+ *  If not, classify previous segment as "external dictionary", and start a new segment.
+ *  This function cannot fail. */
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
+
+
+#endif /* ZSTD_DECOMPRESS_INTERNAL_H */
+/**** ended inlining zstd_decompress_internal.h ****/
+/**** start inlining zstd_ddict.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_DDICT_H
+#define ZSTD_DDICT_H
+
+/*-*******************************************************
+ *  Dependencies
+ *********************************************************/
+/**** skipping file: ../common/zstd_deps.h ****/
+/**** skipping file: ../zstd.h ****/
+
+
+/*-*******************************************************
+ *  Interface
+ *********************************************************/
+
+/* note: several prototypes are already published in `zstd.h` :
+ * ZSTD_createDDict()
+ * ZSTD_createDDict_byReference()
+ * ZSTD_createDDict_advanced()
+ * ZSTD_freeDDict()
+ * ZSTD_initStaticDDict()
+ * ZSTD_sizeof_DDict()
+ * ZSTD_estimateDDictSize()
+ * ZSTD_getDictID_fromDict()
+ */
+
+const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
+size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
+
+void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+
+
+#endif /* ZSTD_DDICT_H */
+/**** ended inlining zstd_ddict.h ****/
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix)
+#endif
+
+
+
+/*-*******************************************************
+*  Types
+*********************************************************/
+struct ZSTD_DDict_s {
+    void* dictBuffer;
+    const void* dictContent;
+    size_t dictSize;
+    ZSTD_entropyDTables_t entropy;
+    U32 dictID;
+    U32 entropyPresent;
+    ZSTD_customMem cMem;
+};  /* typedef'd to ZSTD_DDict within "zstd.h" */
+
+const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
+{
+    assert(ddict != NULL);
+    return ddict->dictContent;
+}
+
+size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
+{
+    assert(ddict != NULL);
+    return ddict->dictSize;
+}
+
+void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+    DEBUGLOG(4, "ZSTD_copyDDictParameters");
+    assert(dctx != NULL);
+    assert(ddict != NULL);
+    dctx->dictID = ddict->dictID;
+    dctx->prefixStart = ddict->dictContent;
+    dctx->virtualStart = ddict->dictContent;
+    dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
+    dctx->previousDstEnd = dctx->dictEnd;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    dctx->dictContentBeginForFuzzing = dctx->prefixStart;
+    dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
+#endif
+    if (ddict->entropyPresent) {
+        dctx->litEntropy = 1;
+        dctx->fseEntropy = 1;
+        dctx->LLTptr = ddict->entropy.LLTable;
+        dctx->MLTptr = ddict->entropy.MLTable;
+        dctx->OFTptr = ddict->entropy.OFTable;
+        dctx->HUFptr = ddict->entropy.hufTable;
+        dctx->entropy.rep[0] = ddict->entropy.rep[0];
+        dctx->entropy.rep[1] = ddict->entropy.rep[1];
+        dctx->entropy.rep[2] = ddict->entropy.rep[2];
+    } else {
+        dctx->litEntropy = 0;
+        dctx->fseEntropy = 0;
+    }
+}
+
+
+static size_t
+ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
+                           ZSTD_dictContentType_e dictContentType)
+{
+    ddict->dictID = 0;
+    ddict->entropyPresent = 0;
+    if (dictContentType == ZSTD_dct_rawContent) return 0;
+
+    if (ddict->dictSize < 8) {
+        if (dictContentType == ZSTD_dct_fullDict)
+            return ERROR(dictionary_corrupted);   /* only accept specified dictionaries */
+        return 0;   /* pure content mode */
+    }
+    {   U32 const magic = MEM_readLE32(ddict->dictContent);
+        if (magic != ZSTD_MAGIC_DICTIONARY) {
+            if (dictContentType == ZSTD_dct_fullDict)
+                return ERROR(dictionary_corrupted);   /* only accept specified dictionaries */
+            return 0;   /* pure content mode */
+        }
+    }
+    ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
+
+    /* load entropy tables */
+    RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
+            &ddict->entropy, ddict->dictContent, ddict->dictSize)),
+        dictionary_corrupted, "");
+    ddict->entropyPresent = 1;
+    return 0;
+}
+
+
+static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
+                                      const void* dict, size_t dictSize,
+                                      ZSTD_dictLoadMethod_e dictLoadMethod,
+                                      ZSTD_dictContentType_e dictContentType)
+{
+    if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
+        ddict->dictBuffer = NULL;
+        ddict->dictContent = dict;
+        if (!dict) dictSize = 0;
+    } else {
+        void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
+        ddict->dictBuffer = internalBuffer;
+        ddict->dictContent = internalBuffer;
+        if (!internalBuffer) return ERROR(memory_allocation);
+        ZSTD_memcpy(internalBuffer, dict, dictSize);
+    }
+    ddict->dictSize = dictSize;
+    ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001);  /* cover both little and big endian */
+
+    /* parse dictionary content */
+    FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
+
+    return 0;
+}
+
+ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+                                      ZSTD_dictLoadMethod_e dictLoadMethod,
+                                      ZSTD_dictContentType_e dictContentType,
+                                      ZSTD_customMem customMem)
+{
+    if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+    {   ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
+        if (ddict == NULL) return NULL;
+        ddict->cMem = customMem;
+        {   size_t const initResult = ZSTD_initDDict_internal(ddict,
+                                            dict, dictSize,
+                                            dictLoadMethod, dictContentType);
+            if (ZSTD_isError(initResult)) {
+                ZSTD_freeDDict(ddict);
+                return NULL;
+        }   }
+        return ddict;
+    }
+}
+
+/*! ZSTD_createDDict() :
+*   Create a digested dictionary, to start decompression without startup delay.
+*   `dict` content is copied inside DDict.
+*   Consequently, `dict` can be released after `ZSTD_DDict` creation */
+ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
+{
+    ZSTD_customMem const allocator = { NULL, NULL, NULL };
+    return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
+}
+
+/*! ZSTD_createDDict_byReference() :
+ *  Create a digested dictionary, to start decompression without startup delay.
+ *  Dictionary content is simply referenced, it will be accessed during decompression.
+ *  Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
+ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
+{
+    ZSTD_customMem const allocator = { NULL, NULL, NULL };
+    return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
+}
+
+
+const ZSTD_DDict* ZSTD_initStaticDDict(
+                                void* sBuffer, size_t sBufferSize,
+                                const void* dict, size_t dictSize,
+                                ZSTD_dictLoadMethod_e dictLoadMethod,
+                                ZSTD_dictContentType_e dictContentType)
+{
+    size_t const neededSpace = sizeof(ZSTD_DDict)
+                             + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
+    ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
+    assert(sBuffer != NULL);
+    assert(dict != NULL);
+    if ((size_t)sBuffer & 7) return NULL;   /* 8-aligned */
+    if (sBufferSize < neededSpace) return NULL;
+    if (dictLoadMethod == ZSTD_dlm_byCopy) {
+        ZSTD_memcpy(ddict+1, dict, dictSize);  /* local copy */
+        dict = ddict+1;
+    }
+    if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
+                                              dict, dictSize,
+                                              ZSTD_dlm_byRef, dictContentType) ))
+        return NULL;
+    return ddict;
+}
+
+
+size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
+{
+    if (ddict==NULL) return 0;   /* support free on NULL */
+    {   ZSTD_customMem const cMem = ddict->cMem;
+        ZSTD_customFree(ddict->dictBuffer, cMem);
+        ZSTD_customFree(ddict, cMem);
+        return 0;
+    }
+}
+
+/*! ZSTD_estimateDDictSize() :
+ *  Estimate amount of memory that will be needed to create a dictionary for decompression.
+ *  Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
+size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
+{
+    return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
+}
+
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
+{
+    if (ddict==NULL) return 0;   /* support sizeof on NULL */
+    return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
+}
+
+/*! ZSTD_getDictID_fromDDict() :
+ *  Provides the dictID of the dictionary loaded into `ddict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
+{
+    if (ddict==NULL) return 0;
+    return ddict->dictID;
+}
+/**** ended inlining decompress/zstd_ddict.c ****/
+/**** start inlining decompress/zstd_decompress.c ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* ***************************************************************
+*  Tuning parameters
+*****************************************************************/
+/*!
+ * HEAPMODE :
+ * Select how default decompression function ZSTD_decompress() allocates its context,
+ * on stack (0), or into heap (1, default; requires malloc()).
+ * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
+ */
+#ifndef ZSTD_HEAPMODE
+#  define ZSTD_HEAPMODE 1
+#endif
+
+/*!
+*  LEGACY_SUPPORT :
+*  if set to 1+, ZSTD_decompress() can decode older formats (v0.1+)
+*/
+#ifndef ZSTD_LEGACY_SUPPORT
+#  define ZSTD_LEGACY_SUPPORT 0
+#endif
+
+/*!
+ *  MAXWINDOWSIZE_DEFAULT :
+ *  maximum window size accepted by DStream __by default__.
+ *  Frames requiring more memory will be rejected.
+ *  It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
+ */
+#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
+#  define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)
+#endif
+
+/*!
+ *  NO_FORWARD_PROGRESS_MAX :
+ *  maximum allowed nb of calls to ZSTD_decompressStream()
+ *  without any forward progress
+ *  (defined as: no byte read from input, and no byte flushed to output)
+ *  before triggering an error.
+ */
+#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX
+#  define ZSTD_NO_FORWARD_PROGRESS_MAX 16
+#endif
+
+
+/*-*******************************************************
+*  Dependencies
+*********************************************************/
+/**** skipping file: ../common/zstd_deps.h ****/
+/**** skipping file: ../common/allocations.h ****/
+/**** skipping file: ../common/error_private.h ****/
+/**** skipping file: ../common/zstd_internal.h ****/
+/**** skipping file: ../common/mem.h ****/
+/**** skipping file: ../common/bits.h ****/
+#define FSE_STATIC_LINKING_ONLY
+/**** skipping file: ../common/fse.h ****/
+/**** skipping file: ../common/huf.h ****/
+/**** skipping file: ../common/xxhash.h ****/
+/**** skipping file: zstd_decompress_internal.h ****/
+/**** skipping file: zstd_ddict.h ****/
+/**** start inlining zstd_decompress_block.h ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_DEC_BLOCK_H
+#define ZSTD_DEC_BLOCK_H
+
+/*-*******************************************************
+ *  Dependencies
+ *********************************************************/
+/**** skipping file: ../common/zstd_deps.h ****/
+/**** skipping file: ../zstd.h ****/
+/**** skipping file: ../common/zstd_internal.h ****/
+/**** skipping file: zstd_decompress_internal.h ****/
+
+
+/* ===   Prototypes   === */
+
+/* note: prototypes already published within `zstd.h` :
+ * ZSTD_decompressBlock()
+ */
+
+/* note: prototypes already published within `zstd_internal.h` :
+ * ZSTD_getcBlockSize()
+ * ZSTD_decodeSeqHeaders()
+ */
+
+
+ /* Streaming state is used to inform allocation of the literal buffer */
+typedef enum {
+    not_streaming = 0,
+    is_streaming = 1
+} streaming_operation;
+
+/* ZSTD_decompressBlock_internal() :
+ * decompress block, starting at `src`,
+ * into destination buffer `dst`.
+ * @return : decompressed block size,
+ *           or an error code (which can be tested using ZSTD_isError())
+ */
+size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize, const streaming_operation streaming);
+
+/* ZSTD_buildFSETable() :
+ * generate FSE decoding table for one symbol (ll, ml or off)
+ * this function must be called with valid parameters only
+ * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
+ * in which case it cannot fail.
+ * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is
+ * defined in zstd_decompress_internal.h.
+ * Internal use only.
+ */
+void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
+             const short* normalizedCounter, unsigned maxSymbolValue,
+             const U32* baseValue, const U8* nbAdditionalBits,
+                   unsigned tableLog, void* wksp, size_t wkspSize,
+                   int bmi2);
+
+/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */
+size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity,
+                      const void* src, size_t srcSize);
+
+
+#endif /* ZSTD_DEC_BLOCK_H */
+/**** ended inlining zstd_decompress_block.h ****/
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix)
+#endif
+
+
+
+/*************************************
+ * Multiple DDicts Hashset internals *
+ *************************************/
+
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3  /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
+                                                    * Currently, that means a 0.75 load factor.
+                                                    * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
+                                                    * the load factor of the ddict hash set.
+                                                    */
+
+#define DDICT_HASHSET_TABLE_BASE_SIZE 64
+#define DDICT_HASHSET_RESIZE_FACTOR 2
+
+/* Hash function to determine starting position of dict insertion within the table
+ * Returns an index between [0, hashSet->ddictPtrTableSize]
+ */
+static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
+    const U64 hash = XXH64(&dictID, sizeof(U32), 0);
+    /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
+    return hash & (hashSet->ddictPtrTableSize - 1);
+}
+
+/* Adds DDict to a hashset without resizing it.
+ * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
+ * Returns 0 if successful, or a zstd error code if something went wrong.
+ */
+static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
+    const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
+    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
+    RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
+    DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
+    while (hashSet->ddictPtrTable[idx] != NULL) {
+        /* Replace existing ddict if inserting ddict with same dictID */
+        if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
+            DEBUGLOG(4, "DictID already exists, replacing rather than adding");
+            hashSet->ddictPtrTable[idx] = ddict;
+            return 0;
+        }
+        idx &= idxRangeMask;
+        idx++;
+    }
+    DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
+    hashSet->ddictPtrTable[idx] = ddict;
+    hashSet->ddictPtrCount++;
+    return 0;
+}
+
+/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
+ * rehashes all values, allocates new table, frees old table.
+ * Returns 0 on success, otherwise a zstd error code.
+ */
+static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+    size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
+    const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
+    const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
+    size_t oldTableSize = hashSet->ddictPtrTableSize;
+    size_t i;
+
+    DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
+    RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
+    hashSet->ddictPtrTable = newTable;
+    hashSet->ddictPtrTableSize = newTableSize;
+    hashSet->ddictPtrCount = 0;
+    for (i = 0; i < oldTableSize; ++i) {
+        if (oldTable[i] != NULL) {
+            FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
+        }
+    }
+    ZSTD_customFree((void*)oldTable, customMem);
+    DEBUGLOG(4, "Finished re-hash");
+    return 0;
+}
+
+/* Fetches a DDict with the given dictID
+ * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
+ */
+static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
+    size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+    const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
+    DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
+    for (;;) {
+        size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
+        if (currDictID == dictID || currDictID == 0) {
+            /* currDictID == 0 implies a NULL ddict entry */
+            break;
+        } else {
+            idx &= idxRangeMask;    /* Goes to start of table when we reach the end */
+            idx++;
+        }
+    }
+    DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
+    return hashSet->ddictPtrTable[idx];
+}
+
+/* Allocates space for and returns a ddict hash set
+ * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
+ * Returns NULL if allocation failed.
+ */
+static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
+    ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
+    DEBUGLOG(4, "Allocating new hash set");
+    if (!ret)
+        return NULL;
+    ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
+    if (!ret->ddictPtrTable) {
+        ZSTD_customFree(ret, customMem);
+        return NULL;
+    }
+    ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
+    ret->ddictPtrCount = 0;
+    return ret;
+}
+
+/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
+ * Note: The ZSTD_DDict* within the table are NOT freed.
+ */
+static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+    DEBUGLOG(4, "Freeing ddict hash set");
+    if (hashSet && hashSet->ddictPtrTable) {
+        ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
+    }
+    if (hashSet) {
+        ZSTD_customFree(hashSet, customMem);
+    }
+}
+
+/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
+ * Returns 0 on success, or a ZSTD error.
+ */
+static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
+    DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
+    if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
+        FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
+    }
+    FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
+    return 0;
+}
+
+/*-*************************************************************
+*   Context management
+***************************************************************/
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
+{
+    if (dctx==NULL) return 0;   /* support sizeof NULL */
+    return sizeof(*dctx)
+           + ZSTD_sizeof_DDict(dctx->ddictLocal)
+           + dctx->inBuffSize + dctx->outBuffSize;
+}
+
+size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
+
+
+static size_t ZSTD_startingInputLength(ZSTD_format_e format)
+{
+    size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
+    /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
+    assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
+    return startingInputLength;
+}
+
+static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
+{
+    assert(dctx->streamStage == zdss_init);
+    dctx->format = ZSTD_f_zstd1;
+    dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
+    dctx->outBufferMode = ZSTD_bm_buffered;
+    dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
+    dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
+    dctx->disableHufAsm = 0;
+    dctx->maxBlockSizeParam = 0;
+}
+
+static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
+{
+    dctx->staticSize  = 0;
+    dctx->ddict       = NULL;
+    dctx->ddictLocal  = NULL;
+    dctx->dictEnd     = NULL;
+    dctx->ddictIsCold = 0;
+    dctx->dictUses = ZSTD_dont_use;
+    dctx->inBuff      = NULL;
+    dctx->inBuffSize  = 0;
+    dctx->outBuffSize = 0;
+    dctx->streamStage = zdss_init;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+    dctx->legacyContext = NULL;
+    dctx->previousLegacyVersion = 0;
+#endif
+    dctx->noForwardProgress = 0;
+    dctx->oversizedDuration = 0;
+    dctx->isFrameDecompression = 1;
+#if DYNAMIC_BMI2
+    dctx->bmi2 = ZSTD_cpuSupportsBmi2();
+#endif
+    dctx->ddictSet = NULL;
+    ZSTD_DCtx_resetParameters(dctx);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    dctx->dictContentEndForFuzzing = NULL;
+#endif
+}
+
+ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
+{
+    ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace;
+
+    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
+    if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL;  /* minimum size */
+
+    ZSTD_initDCtx_internal(dctx);
+    dctx->staticSize = workspaceSize;
+    dctx->inBuff = (char*)(dctx+1);
+    return dctx;
+}
+
+static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) {
+    if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+    {   ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem);
+        if (!dctx) return NULL;
+        dctx->customMem = customMem;
+        ZSTD_initDCtx_internal(dctx);
+        return dctx;
+    }
+}
+
+ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
+{
+    return ZSTD_createDCtx_internal(customMem);
+}
+
+ZSTD_DCtx* ZSTD_createDCtx(void)
+{
+    DEBUGLOG(3, "ZSTD_createDCtx");
+    return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+}
+
+static void ZSTD_clearDict(ZSTD_DCtx* dctx)
+{
+    ZSTD_freeDDict(dctx->ddictLocal);
+    dctx->ddictLocal = NULL;
+    dctx->ddict = NULL;
+    dctx->dictUses = ZSTD_dont_use;
+}
+
+size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
+{
+    if (dctx==NULL) return 0;   /* support free on NULL */
+    RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
+    {   ZSTD_customMem const cMem = dctx->customMem;
+        ZSTD_clearDict(dctx);
+        ZSTD_customFree(dctx->inBuff, cMem);
+        dctx->inBuff = NULL;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+        if (dctx->legacyContext)
+            ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
+#endif
+        if (dctx->ddictSet) {
+            ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
+            dctx->ddictSet = NULL;
+        }
+        ZSTD_customFree(dctx, cMem);
+        return 0;
+    }
+}
+
+/* no longer useful */
+void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
+{
+    size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
+    ZSTD_memcpy(dstDCtx, srcDCtx, toCopy);  /* no need to copy workspace */
+}
+
+/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
+ * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
+ * accordingly sets the ddict to be used to decompress the frame.
+ *
+ * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
+ *
+ * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
+ */
+static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
+    assert(dctx->refMultipleDDicts && dctx->ddictSet);
+    DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
+    if (dctx->ddict) {
+        const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
+        if (frameDDict) {
+            DEBUGLOG(4, "DDict found!");
+            ZSTD_clearDict(dctx);
+            dctx->dictID = dctx->fParams.dictID;
+            dctx->ddict = frameDDict;
+            dctx->dictUses = ZSTD_use_indefinitely;
+        }
+    }
+}
+
+
+/*-*************************************************************
+ *   Frame header decoding
+ ***************************************************************/
+
+/*! ZSTD_isFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier.
+ *  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ *  Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ *  Note 3 : Skippable Frame Identifiers are considered valid. */
+unsigned ZSTD_isFrame(const void* buffer, size_t size)
+{
+    if (size < ZSTD_FRAMEIDSIZE) return 0;
+    {   U32 const magic = MEM_readLE32(buffer);
+        if (magic == ZSTD_MAGICNUMBER) return 1;
+        if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+    }
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (ZSTD_isLegacy(buffer, size)) return 1;
+#endif
+    return 0;
+}
+
+/*! ZSTD_isSkippableFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ *  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ */
+unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
+{
+    if (size < ZSTD_FRAMEIDSIZE) return 0;
+    {   U32 const magic = MEM_readLE32(buffer);
+        if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+    }
+    return 0;
+}
+
+/** ZSTD_frameHeaderSize_internal() :
+ *  srcSize must be large enough to reach header size fields.
+ *  note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
+ * @return : size of the Frame Header
+ *           or an error code, which can be tested with ZSTD_isError() */
+static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
+{
+    size_t const minInputSize = ZSTD_startingInputLength(format);
+    RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");
+
+    {   BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
+        U32 const dictID= fhd & 3;
+        U32 const singleSegment = (fhd >> 5) & 1;
+        U32 const fcsId = fhd >> 6;
+        return minInputSize + !singleSegment
+             + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+             + (singleSegment && !fcsId);
+    }
+}
+
+/** ZSTD_frameHeaderSize() :
+ *  srcSize must be >= ZSTD_frameHeaderSize_prefix.
+ * @return : size of the Frame Header,
+ *           or an error code (if srcSize is too small) */
+size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
+{
+    return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);
+}
+
+
+/** ZSTD_getFrameHeader_advanced() :
+ *  decode Frame Header, or require larger `srcSize`.
+ *  note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
+ * @return : 0, `zfhPtr` is correctly filled,
+ *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+**           or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
+{
+    const BYTE* ip = (const BYTE*)src;
+    size_t const minInputSize = ZSTD_startingInputLength(format);
+
+    DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize);
+
+    if (srcSize > 0) {
+        /* note : technically could be considered an assert(), since it's an invalid entry */
+        RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0");
+    }
+    if (srcSize < minInputSize) {
+        if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) {
+            /* when receiving less than @minInputSize bytes,
+             * control these bytes at least correspond to a supported magic number
+             * in order to error out early if they don't.
+            **/
+            size_t const toCopy = MIN(4, srcSize);
+            unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER);
+            assert(src != NULL);
+            ZSTD_memcpy(hbuf, src, toCopy);
+            if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) {
+                /* not a zstd frame : let's check if it's a skippable frame */
+                MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START);
+                ZSTD_memcpy(hbuf, src, toCopy);
+                if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) {
+                    RETURN_ERROR(prefix_unknown,
+                                "first bytes don't correspond to any supported magic number");
+        }   }   }
+        return minInputSize;
+    }
+
+    ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));   /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */
+    if ( (format != ZSTD_f_zstd1_magicless)
+      && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
+        if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+            /* skippable frame */
+            if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
+                return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
+            ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
+            zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
+            zfhPtr->frameType = ZSTD_skippableFrame;
+            return 0;
+        }
+        RETURN_ERROR(prefix_unknown, "");
+    }
+
+    /* ensure there is enough `srcSize` to fully read/decode frame header */
+    {   size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format);
+        if (srcSize < fhsize) return fhsize;
+        zfhPtr->headerSize = (U32)fhsize;
+    }
+
+    {   BYTE const fhdByte = ip[minInputSize-1];
+        size_t pos = minInputSize;
+        U32 const dictIDSizeCode = fhdByte&3;
+        U32 const checksumFlag = (fhdByte>>2)&1;
+        U32 const singleSegment = (fhdByte>>5)&1;
+        U32 const fcsID = fhdByte>>6;
+        U64 windowSize = 0;
+        U32 dictID = 0;
+        U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+        RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported,
+                        "reserved bits, must be zero");
+
+        if (!singleSegment) {
+            BYTE const wlByte = ip[pos++];
+            U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+            RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, "");
+            windowSize = (1ULL << windowLog);
+            windowSize += (windowSize >> 3) * (wlByte&7);
+        }
+        switch(dictIDSizeCode)
+        {
+            default:
+                assert(0);  /* impossible */
+                ZSTD_FALLTHROUGH;
+            case 0 : break;
+            case 1 : dictID = ip[pos]; pos++; break;
+            case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
+            case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
+        }
+        switch(fcsID)
+        {
+            default:
+                assert(0);  /* impossible */
+                ZSTD_FALLTHROUGH;
+            case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
+            case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
+            case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
+            case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
+        }
+        if (singleSegment) windowSize = frameContentSize;
+
+        zfhPtr->frameType = ZSTD_frame;
+        zfhPtr->frameContentSize = frameContentSize;
+        zfhPtr->windowSize = windowSize;
+        zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+        zfhPtr->dictID = dictID;
+        zfhPtr->checksumFlag = checksumFlag;
+    }
+    return 0;
+}
+
+/** ZSTD_getFrameHeader() :
+ *  decode Frame Header, or require larger `srcSize`.
+ *  note : this function does not consume input, it only reads it.
+ * @return : 0, `zfhPtr` is correctly filled,
+ *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ *           or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
+{
+    return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
+}
+
+/** ZSTD_getFrameContentSize() :
+ *  compatible with legacy mode
+ * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
+ *         - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ *         - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
+{
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (ZSTD_isLegacy(src, srcSize)) {
+        unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
+        return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
+    }
+#endif
+    {   ZSTD_frameHeader zfh;
+        if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
+            return ZSTD_CONTENTSIZE_ERROR;
+        if (zfh.frameType == ZSTD_skippableFrame) {
+            return 0;
+        } else {
+            return zfh.frameContentSize;
+    }   }
+}
+
+static size_t readSkippableFrameSize(void const* src, size_t srcSize)
+{
+    size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
+    U32 sizeU32;
+
+    RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
+
+    sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
+    RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
+                    frameParameter_unsupported, "");
+    {   size_t const skippableSize = skippableHeaderSize + sizeU32;
+        RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
+        return skippableSize;
+    }
+}
+
+/*! ZSTD_readSkippableFrame() :
+ * Retrieves content of a skippable frame, and writes it to dst buffer.
+ *
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
+ * in the magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame.
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
+                               unsigned* magicVariant,  /* optional, can be NULL */
+                         const void* src, size_t srcSize)
+{
+    RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
+
+    {   U32 const magicNumber = MEM_readLE32(src);
+        size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
+        size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
+
+        /* check input validity */
+        RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
+        RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
+        RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
+
+        /* deliver payload */
+        if (skippableContentSize > 0  && dst != NULL)
+            ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
+        if (magicVariant != NULL)
+            *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
+        return skippableContentSize;
+    }
+}
+
+/** ZSTD_findDecompressedSize() :
+ *  `srcSize` must be the exact length of some number of ZSTD compressed and/or
+ *      skippable frames
+ *  note: compatible with legacy mode
+ * @return : decompressed size of the frames contained */
+unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
+{
+    unsigned long long totalDstSize = 0;
+
+    while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
+        U32 const magicNumber = MEM_readLE32(src);
+
+        if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+            size_t const skippableSize = readSkippableFrameSize(src, srcSize);
+            if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR;
+            assert(skippableSize <= srcSize);
+
+            src = (const BYTE *)src + skippableSize;
+            srcSize -= skippableSize;
+            continue;
+        }
+
+        {   unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize);
+            if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs;
+
+            if (totalDstSize + fcs < totalDstSize)
+                return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */
+            totalDstSize += fcs;
+        }
+        /* skip to next frame */
+        {   size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
+            if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR;
+            assert(frameSrcSize <= srcSize);
+
+            src = (const BYTE *)src + frameSrcSize;
+            srcSize -= frameSrcSize;
+        }
+    }  /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
+
+    if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
+
+    return totalDstSize;
+}
+
+/** ZSTD_getDecompressedSize() :
+ *  compatible with legacy mode
+ * @return : decompressed size if known, 0 otherwise
+             note : 0 can mean any of the following :
+                   - frame content is empty
+                   - decompressed size field is not present in frame header
+                   - frame header unknown / not supported
+                   - frame header not complete (`srcSize` too small) */
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
+{
+    unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);
+    return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;
+}
+
+
+/** ZSTD_decodeFrameHeader() :
+ * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
+ * If multiple DDict references are enabled, also will choose the correct DDict to use.
+ * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
+{
+    size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
+    if (ZSTD_isError(result)) return result;    /* invalid header */
+    RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
+
+    /* Reference DDict requested by frame if dctx references multiple ddicts */
+    if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
+        ZSTD_DCtx_selectFrameDDict(dctx);
+    }
+
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    /* Skip the dictID check in fuzzing mode, because it makes the search
+     * harder.
+     */
+    RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
+                    dictionary_wrong, "");
+#endif
+    dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
+    if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
+    dctx->processedCSize += headerSize;
+    return 0;
+}
+
+static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
+{
+    ZSTD_frameSizeInfo frameSizeInfo;
+    frameSizeInfo.compressedSize = ret;
+    frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
+    return frameSizeInfo;
+}
+
+static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format)
+{
+    ZSTD_frameSizeInfo frameSizeInfo;
+    ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize))
+        return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
+#endif
+
+    if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
+        && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+        frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
+        assert(ZSTD_isError(frameSizeInfo.compressedSize) ||
+               frameSizeInfo.compressedSize <= srcSize);
+        return frameSizeInfo;
+    } else {
+        const BYTE* ip = (const BYTE*)src;
+        const BYTE* const ipstart = ip;
+        size_t remainingSize = srcSize;
+        size_t nbBlocks = 0;
+        ZSTD_frameHeader zfh;
+
+        /* Extract Frame Header */
+        {   size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format);
+            if (ZSTD_isError(ret))
+                return ZSTD_errorFrameSizeInfo(ret);
+            if (ret > 0)
+                return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+        }
+
+        ip += zfh.headerSize;
+        remainingSize -= zfh.headerSize;
+
+        /* Iterate over each block */
+        while (1) {
+            blockProperties_t blockProperties;
+            size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+            if (ZSTD_isError(cBlockSize))
+                return ZSTD_errorFrameSizeInfo(cBlockSize);
+
+            if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
+                return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+
+            ip += ZSTD_blockHeaderSize + cBlockSize;
+            remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
+            nbBlocks++;
+
+            if (blockProperties.lastBlock) break;
+        }
+
+        /* Final frame content checksum */
+        if (zfh.checksumFlag) {
+            if (remainingSize < 4)
+                return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+            ip += 4;
+        }
+
+        frameSizeInfo.nbBlocks = nbBlocks;
+        frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
+        frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
+                                        ? zfh.frameContentSize
+                                        : (unsigned long long)nbBlocks * zfh.blockSizeMax;
+        return frameSizeInfo;
+    }
+}
+
+static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) {
+    ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format);
+    return frameSizeInfo.compressedSize;
+}
+
+/** ZSTD_findFrameCompressedSize() :
+ * See docs in zstd.h
+ * Note: compatible with legacy mode */
+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+    return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1);
+}
+
+/** ZSTD_decompressBound() :
+ *  compatible with legacy mode
+ *  `src` must point to the start of a ZSTD frame or a skippable frame
+ *  `srcSize` must be at least as large as the frame contained
+ *  @return : the maximum decompressed size of the compressed source
+ */
+unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
+{
+    unsigned long long bound = 0;
+    /* Iterate over each frame */
+    while (srcSize > 0) {
+        ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1);
+        size_t const compressedSize = frameSizeInfo.compressedSize;
+        unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+            return ZSTD_CONTENTSIZE_ERROR;
+        assert(srcSize >= compressedSize);
+        src = (const BYTE*)src + compressedSize;
+        srcSize -= compressedSize;
+        bound += decompressedBound;
+    }
+    return bound;
+}
+
+size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
+{
+    size_t margin = 0;
+    unsigned maxBlockSize = 0;
+
+    /* Iterate over each frame */
+    while (srcSize > 0) {
+        ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1);
+        size_t const compressedSize = frameSizeInfo.compressedSize;
+        unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+        ZSTD_frameHeader zfh;
+
+        FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
+        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+            return ERROR(corruption_detected);
+
+        if (zfh.frameType == ZSTD_frame) {
+            /* Add the frame header to our margin */
+            margin += zfh.headerSize;
+            /* Add the checksum to our margin */
+            margin += zfh.checksumFlag ? 4 : 0;
+            /* Add 3 bytes per block */
+            margin += 3 * frameSizeInfo.nbBlocks;
+
+            /* Compute the max block size */
+            maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax);
+        } else {
+            assert(zfh.frameType == ZSTD_skippableFrame);
+            /* Add the entire skippable frame size to our margin. */
+            margin += compressedSize;
+        }
+
+        assert(srcSize >= compressedSize);
+        src = (const BYTE*)src + compressedSize;
+        srcSize -= compressedSize;
+    }
+
+    /* Add the max block size back to the margin. */
+    margin += maxBlockSize;
+
+    return margin;
+}
+
+/*-*************************************************************
+ *   Frame decoding
+ ***************************************************************/
+
+/** ZSTD_insertBlock() :
+ *  insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
+size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
+{
+    DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
+    ZSTD_checkContinuity(dctx, blockStart, blockSize);
+    dctx->previousDstEnd = (const char*)blockStart + blockSize;
+    return blockSize;
+}
+
+
+static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
+                          const void* src, size_t srcSize)
+{
+    DEBUGLOG(5, "ZSTD_copyRawBlock");
+    RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
+    if (dst == NULL) {
+        if (srcSize == 0) return 0;
+        RETURN_ERROR(dstBuffer_null, "");
+    }
+    ZSTD_memmove(dst, src, srcSize);
+    return srcSize;
+}
+
+static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
+                               BYTE b,
+                               size_t regenSize)
+{
+    RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
+    if (dst == NULL) {
+        if (regenSize == 0) return 0;
+        RETURN_ERROR(dstBuffer_null, "");
+    }
+    ZSTD_memset(dst, b, regenSize);
+    return regenSize;
+}
+
+static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
+{
+#if ZSTD_TRACE
+    if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
+        ZSTD_Trace trace;
+        ZSTD_memset(&trace, 0, sizeof(trace));
+        trace.version = ZSTD_VERSION_NUMBER;
+        trace.streaming = streaming;
+        if (dctx->ddict) {
+            trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
+            trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
+            trace.dictionaryIsCold = dctx->ddictIsCold;
+        }
+        trace.uncompressedSize = (size_t)uncompressedSize;
+        trace.compressedSize = (size_t)compressedSize;
+        trace.dctx = dctx;
+        ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
+    }
+#else
+    (void)dctx;
+    (void)uncompressedSize;
+    (void)compressedSize;
+    (void)streaming;
+#endif
+}
+
+
+/*! ZSTD_decompressFrame() :
+ * @dctx must be properly initialized
+ *  will update *srcPtr and *srcSizePtr,
+ *  to make *srcPtr progress by one frame. */
+static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
+                                   void* dst, size_t dstCapacity,
+                             const void** srcPtr, size_t *srcSizePtr)
+{
+    const BYTE* const istart = (const BYTE*)(*srcPtr);
+    const BYTE* ip = istart;
+    BYTE* const ostart = (BYTE*)dst;
+    BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
+    BYTE* op = ostart;
+    size_t remainingSrcSize = *srcSizePtr;
+
+    DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr);
+
+    /* check */
+    RETURN_ERROR_IF(
+        remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
+        srcSize_wrong, "");
+
+    /* Frame Header */
+    {   size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
+                ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
+        if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
+        RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
+                        srcSize_wrong, "");
+        FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , "");
+        ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
+    }
+
+    /* Shrink the blockSizeMax if enabled */
+    if (dctx->maxBlockSizeParam != 0)
+        dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam);
+
+    /* Loop on each block */
+    while (1) {
+        BYTE* oBlockEnd = oend;
+        size_t decodedSize;
+        blockProperties_t blockProperties;
+        size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
+        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTD_blockHeaderSize;
+        remainingSrcSize -= ZSTD_blockHeaderSize;
+        RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
+
+        if (ip >= op && ip < oBlockEnd) {
+            /* We are decompressing in-place. Limit the output pointer so that we
+             * don't overwrite the block that we are currently reading. This will
+             * fail decompression if the input & output pointers aren't spaced
+             * far enough apart.
+             *
+             * This is important to set, even when the pointers are far enough
+             * apart, because ZSTD_decompressBlock_internal() can decide to store
+             * literals in the output buffer, after the block it is decompressing.
+             * Since we don't want anything to overwrite our input, we have to tell
+             * ZSTD_decompressBlock_internal to never write past ip.
+             *
+             * See ZSTD_allocateLiteralsBuffer() for reference.
+             */
+            oBlockEnd = op + (ip - op);
+        }
+
+        switch(blockProperties.blockType)
+        {
+        case bt_compressed:
+            assert(dctx->isFrameDecompression == 1);
+            decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming);
+            break;
+        case bt_raw :
+            /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */
+            decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
+            break;
+        case bt_rle :
+            decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize);
+            break;
+        case bt_reserved :
+        default:
+            RETURN_ERROR(corruption_detected, "invalid block type");
+        }
+        FORWARD_IF_ERROR(decodedSize, "Block decompression failure");
+        DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize);
+        if (dctx->validateChecksum) {
+            XXH64_update(&dctx->xxhState, op, decodedSize);
+        }
+        if (decodedSize) /* support dst = NULL,0 */ {
+            op += decodedSize;
+        }
+        assert(ip != NULL);
+        ip += cBlockSize;
+        remainingSrcSize -= cBlockSize;
+        if (blockProperties.lastBlock) break;
+    }
+
+    if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+        RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
+                        corruption_detected, "");
+    }
+    if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
+        RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
+        if (!dctx->forceIgnoreChecksum) {
+            U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
+            U32 checkRead;
+            checkRead = MEM_readLE32(ip);
+            RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
+        }
+        ip += 4;
+        remainingSrcSize -= 4;
+    }
+    ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
+    /* Allow caller to get size read */
+    DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr));
+    *srcPtr = ip;
+    *srcSizePtr = remainingSrcSize;
+    return (size_t)(op-ostart);
+}
+
+static
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
+                                        void* dst, size_t dstCapacity,
+                                  const void* src, size_t srcSize,
+                                  const void* dict, size_t dictSize,
+                                  const ZSTD_DDict* ddict)
+{
+    void* const dststart = dst;
+    int moreThan1Frame = 0;
+
+    DEBUGLOG(5, "ZSTD_decompressMultiFrame");
+    assert(dict==NULL || ddict==NULL);  /* either dict or ddict set, not both */
+
+    if (ddict) {
+        dict = ZSTD_DDict_dictContent(ddict);
+        dictSize = ZSTD_DDict_dictSize(ddict);
+    }
+
+    while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+        if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) {
+            size_t decodedSize;
+            size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
+            if (ZSTD_isError(frameSize)) return frameSize;
+            RETURN_ERROR_IF(dctx->staticSize, memory_allocation,
+                "legacy support is not compatible with static dctx");
+
+            decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
+            if (ZSTD_isError(decodedSize)) return decodedSize;
+
+            {
+                unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize);
+                RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!");
+                if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+                    RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected,
+                        "Frame header size does not match decoded size!");
+                }
+            }
+
+            assert(decodedSize <= dstCapacity);
+            dst = (BYTE*)dst + decodedSize;
+            dstCapacity -= decodedSize;
+
+            src = (const BYTE*)src + frameSize;
+            srcSize -= frameSize;
+
+            continue;
+        }
+#endif
+
+        if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) {
+            U32 const magicNumber = MEM_readLE32(src);
+            DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber);
+            if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+                /* skippable frame detected : skip it */
+                size_t const skippableSize = readSkippableFrameSize(src, srcSize);
+                FORWARD_IF_ERROR(skippableSize, "invalid skippable frame");
+                assert(skippableSize <= srcSize);
+
+                src = (const BYTE *)src + skippableSize;
+                srcSize -= skippableSize;
+                continue; /* check next frame */
+        }   }
+
+        if (ddict) {
+            /* we were called from ZSTD_decompress_usingDDict */
+            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), "");
+        } else {
+            /* this will initialize correctly with no dict if dict == NULL, so
+             * use this in all cases but ddict */
+            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
+        }
+        ZSTD_checkContinuity(dctx, dst, dstCapacity);
+
+        {   const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
+                                                    &src, &srcSize);
+            RETURN_ERROR_IF(
+                (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
+             && (moreThan1Frame==1),
+                srcSize_wrong,
+                "At least one frame successfully completed, "
+                "but following bytes are garbage: "
+                "it's more likely to be a srcSize error, "
+                "specifying more input bytes than size of frame(s). "
+                "Note: one could be unlucky, it might be a corruption error instead, "
+                "happening right at the place where we expect zstd magic bytes. "
+                "But this is _much_ less likely than a srcSize field error.");
+            if (ZSTD_isError(res)) return res;
+            assert(res <= dstCapacity);
+            if (res != 0)
+                dst = (BYTE*)dst + res;
+            dstCapacity -= res;
+        }
+        moreThan1Frame = 1;
+    }  /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
+
+    RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
+
+    return (size_t)((BYTE*)dst - (BYTE*)dststart);
+}
+
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+                                 void* dst, size_t dstCapacity,
+                           const void* src, size_t srcSize,
+                           const void* dict, size_t dictSize)
+{
+    return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
+}
+
+
+static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
+{
+    switch (dctx->dictUses) {
+    default:
+        assert(0 /* Impossible */);
+        ZSTD_FALLTHROUGH;
+    case ZSTD_dont_use:
+        ZSTD_clearDict(dctx);
+        return NULL;
+    case ZSTD_use_indefinitely:
+        return dctx->ddict;
+    case ZSTD_use_once:
+        dctx->dictUses = ZSTD_dont_use;
+        return dctx->ddict;
+    }
+}
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+    return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));
+}
+
+
+size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
+    size_t regenSize;
+    ZSTD_DCtx* const dctx =  ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+    RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
+    regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
+    ZSTD_freeDCtx(dctx);
+    return regenSize;
+#else   /* stack mode */
+    ZSTD_DCtx dctx;
+    ZSTD_initDCtx_internal(&dctx);
+    return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
+#endif
+}
+
+
+/*-**************************************
+*   Advanced Streaming Decompression API
+*   Bufferless and synchronous
+****************************************/
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
+
+/**
+ * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we
+ * allow taking a partial block as the input. Currently only raw uncompressed blocks can
+ * be streamed.
+ *
+ * For blocks that can be streamed, this allows us to reduce the latency until we produce
+ * output, and avoid copying the input.
+ *
+ * @param inputSize - The total amount of input that the caller currently has.
+ */
+static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {
+    if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock))
+        return dctx->expected;
+    if (dctx->bType != bt_raw)
+        return dctx->expected;
+    return BOUNDED(1, inputSize, dctx->expected);
+}
+
+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
+    switch(dctx->stage)
+    {
+    default:   /* should not happen */
+        assert(0);
+        ZSTD_FALLTHROUGH;
+    case ZSTDds_getFrameHeaderSize:
+        ZSTD_FALLTHROUGH;
+    case ZSTDds_decodeFrameHeader:
+        return ZSTDnit_frameHeader;
+    case ZSTDds_decodeBlockHeader:
+        return ZSTDnit_blockHeader;
+    case ZSTDds_decompressBlock:
+        return ZSTDnit_block;
+    case ZSTDds_decompressLastBlock:
+        return ZSTDnit_lastBlock;
+    case ZSTDds_checkChecksum:
+        return ZSTDnit_checksum;
+    case ZSTDds_decodeSkippableHeader:
+        ZSTD_FALLTHROUGH;
+    case ZSTDds_skipFrame:
+        return ZSTDnit_skippableFrame;
+    }
+}
+
+static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
+
+/** ZSTD_decompressContinue() :
+ *  srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
+ *  @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
+ *            or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+    DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
+    /* Sanity check */
+    RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
+    ZSTD_checkContinuity(dctx, dst, dstCapacity);
+
+    dctx->processedCSize += srcSize;
+
+    switch (dctx->stage)
+    {
+    case ZSTDds_getFrameHeaderSize :
+        assert(src != NULL);
+        if (dctx->format == ZSTD_f_zstd1) {  /* allows header */
+            assert(srcSize >= ZSTD_FRAMEIDSIZE);  /* to read skippable magic number */
+            if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {        /* skippable frame */
+                ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
+                dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize;  /* remaining to load to get full skippable frame header */
+                dctx->stage = ZSTDds_decodeSkippableHeader;
+                return 0;
+        }   }
+        dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
+        if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
+        ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
+        dctx->expected = dctx->headerSize - srcSize;
+        dctx->stage = ZSTDds_decodeFrameHeader;
+        return 0;
+
+    case ZSTDds_decodeFrameHeader:
+        assert(src != NULL);
+        ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
+        FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
+        dctx->expected = ZSTD_blockHeaderSize;
+        dctx->stage = ZSTDds_decodeBlockHeader;
+        return 0;
+
+    case ZSTDds_decodeBlockHeader:
+        {   blockProperties_t bp;
+            size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+            if (ZSTD_isError(cBlockSize)) return cBlockSize;
+            RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum");
+            dctx->expected = cBlockSize;
+            dctx->bType = bp.blockType;
+            dctx->rleSize = bp.origSize;
+            if (cBlockSize) {
+                dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
+                return 0;
+            }
+            /* empty block */
+            if (bp.lastBlock) {
+                if (dctx->fParams.checksumFlag) {
+                    dctx->expected = 4;
+                    dctx->stage = ZSTDds_checkChecksum;
+                } else {
+                    dctx->expected = 0; /* end of frame */
+                    dctx->stage = ZSTDds_getFrameHeaderSize;
+                }
+            } else {
+                dctx->expected = ZSTD_blockHeaderSize;  /* jump to next header */
+                dctx->stage = ZSTDds_decodeBlockHeader;
+            }
+            return 0;
+        }
+
+    case ZSTDds_decompressLastBlock:
+    case ZSTDds_decompressBlock:
+        DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock");
+        {   size_t rSize;
+            switch(dctx->bType)
+            {
+            case bt_compressed:
+                DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
+                assert(dctx->isFrameDecompression == 1);
+                rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming);
+                dctx->expected = 0;  /* Streaming not supported */
+                break;
+            case bt_raw :
+                assert(srcSize <= dctx->expected);
+                rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
+                FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed");
+                assert(rSize == srcSize);
+                dctx->expected -= rSize;
+                break;
+            case bt_rle :
+                rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize);
+                dctx->expected = 0;  /* Streaming not supported */
+                break;
+            case bt_reserved :   /* should never happen */
+            default:
+                RETURN_ERROR(corruption_detected, "invalid block type");
+            }
+            FORWARD_IF_ERROR(rSize, "");
+            RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
+            DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
+            dctx->decodedSize += rSize;
+            if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
+            dctx->previousDstEnd = (char*)dst + rSize;
+
+            /* Stay on the same stage until we are finished streaming the block. */
+            if (dctx->expected > 0) {
+                return rSize;
+            }
+
+            if (dctx->stage == ZSTDds_decompressLastBlock) {   /* end of frame */
+                DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
+                RETURN_ERROR_IF(
+                    dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+                 && dctx->decodedSize != dctx->fParams.frameContentSize,
+                    corruption_detected, "");
+                if (dctx->fParams.checksumFlag) {  /* another round for frame checksum */
+                    dctx->expected = 4;
+                    dctx->stage = ZSTDds_checkChecksum;
+                } else {
+                    ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
+                    dctx->expected = 0;   /* ends here */
+                    dctx->stage = ZSTDds_getFrameHeaderSize;
+                }
+            } else {
+                dctx->stage = ZSTDds_decodeBlockHeader;
+                dctx->expected = ZSTD_blockHeaderSize;
+            }
+            return rSize;
+        }
+
+    case ZSTDds_checkChecksum:
+        assert(srcSize == 4);  /* guaranteed by dctx->expected */
+        {
+            if (dctx->validateChecksum) {
+                U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
+                U32 const check32 = MEM_readLE32(src);
+                DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
+                RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
+            }
+            ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
+            dctx->expected = 0;
+            dctx->stage = ZSTDds_getFrameHeaderSize;
+            return 0;
+        }
+
+    case ZSTDds_decodeSkippableHeader:
+        assert(src != NULL);
+        assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
+        assert(dctx->format != ZSTD_f_zstd1_magicless);
+        ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize);   /* complete skippable header */
+        dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE);   /* note : dctx->expected can grow seriously large, beyond local buffer size */
+        dctx->stage = ZSTDds_skipFrame;
+        return 0;
+
+    case ZSTDds_skipFrame:
+        dctx->expected = 0;
+        dctx->stage = ZSTDds_getFrameHeaderSize;
+        return 0;
+
+    default:
+        assert(0);   /* impossible */
+        RETURN_ERROR(GENERIC, "impossible to reach");   /* some compilers require default to do something */
+    }
+}
+
+
+static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+    dctx->dictEnd = dctx->previousDstEnd;
+    dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
+    dctx->prefixStart = dict;
+    dctx->previousDstEnd = (const char*)dict + dictSize;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    dctx->dictContentBeginForFuzzing = dctx->prefixStart;
+    dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
+#endif
+    return 0;
+}
+
+/*! ZSTD_loadDEntropy() :
+ *  dict : must point at beginning of a valid zstd dictionary.
+ * @return : size of entropy tables read */
+size_t
+ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
+                  const void* const dict, size_t const dictSize)
+{
+    const BYTE* dictPtr = (const BYTE*)dict;
+    const BYTE* const dictEnd = dictPtr + dictSize;
+
+    RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small");
+    assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY);   /* dict must be valid */
+    dictPtr += 8;   /* skip header = magic + dictID */
+
+    ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable));
+    ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable));
+    ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE);
+    {   void* const workspace = &entropy->LLTable;   /* use fse tables as temporary workspace; implies fse tables are grouped together */
+        size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
+#ifdef HUF_FORCE_DECOMPRESS_X1
+        /* in minimal huffman, we always use X1 variants */
+        size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
+                                                dictPtr, dictEnd - dictPtr,
+                                                workspace, workspaceSize, /* flags */ 0);
+#else
+        size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
+                                                dictPtr, (size_t)(dictEnd - dictPtr),
+                                                workspace, workspaceSize, /* flags */ 0);
+#endif
+        RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
+        dictPtr += hSize;
+    }
+
+    {   short offcodeNCount[MaxOff+1];
+        unsigned offcodeMaxValue = MaxOff, offcodeLog;
+        size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
+        RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
+        RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
+        RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
+        ZSTD_buildFSETable( entropy->OFTable,
+                            offcodeNCount, offcodeMaxValue,
+                            OF_base, OF_bits,
+                            offcodeLog,
+                            entropy->workspace, sizeof(entropy->workspace),
+                            /* bmi2 */0);
+        dictPtr += offcodeHeaderSize;
+    }
+
+    {   short matchlengthNCount[MaxML+1];
+        unsigned matchlengthMaxValue = MaxML, matchlengthLog;
+        size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
+        RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
+        RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
+        RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
+        ZSTD_buildFSETable( entropy->MLTable,
+                            matchlengthNCount, matchlengthMaxValue,
+                            ML_base, ML_bits,
+                            matchlengthLog,
+                            entropy->workspace, sizeof(entropy->workspace),
+                            /* bmi2 */ 0);
+        dictPtr += matchlengthHeaderSize;
+    }
+
+    {   short litlengthNCount[MaxLL+1];
+        unsigned litlengthMaxValue = MaxLL, litlengthLog;
+        size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
+        RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
+        RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
+        RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
+        ZSTD_buildFSETable( entropy->LLTable,
+                            litlengthNCount, litlengthMaxValue,
+                            LL_base, LL_bits,
+                            litlengthLog,
+                            entropy->workspace, sizeof(entropy->workspace),
+                            /* bmi2 */ 0);
+        dictPtr += litlengthHeaderSize;
+    }
+
+    RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
+    {   int i;
+        size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
+        for (i=0; i<3; i++) {
+            U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
+            RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
+                            dictionary_corrupted, "");
+            entropy->rep[i] = rep;
+    }   }
+
+    return (size_t)(dictPtr - (const BYTE*)dict);
+}
+
+static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+    if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
+    {   U32 const magic = MEM_readLE32(dict);
+        if (magic != ZSTD_MAGIC_DICTIONARY) {
+            return ZSTD_refDictContent(dctx, dict, dictSize);   /* pure content mode */
+    }   }
+    dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
+
+    /* load entropy tables */
+    {   size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
+        RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, "");
+        dict = (const char*)dict + eSize;
+        dictSize -= eSize;
+    }
+    dctx->litEntropy = dctx->fseEntropy = 1;
+
+    /* reference dictionary content */
+    return ZSTD_refDictContent(dctx, dict, dictSize);
+}
+
+size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
+{
+    assert(dctx != NULL);
+#if ZSTD_TRACE
+    dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
+#endif
+    dctx->expected = ZSTD_startingInputLength(dctx->format);  /* dctx->format must be properly set */
+    dctx->stage = ZSTDds_getFrameHeaderSize;
+    dctx->processedCSize = 0;
+    dctx->decodedSize = 0;
+    dctx->previousDstEnd = NULL;
+    dctx->prefixStart = NULL;
+    dctx->virtualStart = NULL;
+    dctx->dictEnd = NULL;
+    dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001);  /* cover both little and big endian */
+    dctx->litEntropy = dctx->fseEntropy = 0;
+    dctx->dictID = 0;
+    dctx->bType = bt_reserved;
+    dctx->isFrameDecompression = 1;
+    ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
+    ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
+    dctx->LLTptr = dctx->entropy.LLTable;
+    dctx->MLTptr = dctx->entropy.MLTable;
+    dctx->OFTptr = dctx->entropy.OFTable;
+    dctx->HUFptr = dctx->entropy.hufTable;
+    return 0;
+}
+
+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+    FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
+    if (dict && dictSize)
+        RETURN_ERROR_IF(
+            ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
+            dictionary_corrupted, "");
+    return 0;
+}
+
+
+/* ======   ZSTD_DDict   ====== */
+
+size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+    DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");
+    assert(dctx != NULL);
+    if (ddict) {
+        const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict);
+        size_t const dictSize = ZSTD_DDict_dictSize(ddict);
+        const void* const dictEnd = dictStart + dictSize;
+        dctx->ddictIsCold = (dctx->dictEnd != dictEnd);
+        DEBUGLOG(4, "DDict is %s",
+                    dctx->ddictIsCold ? "~cold~" : "hot!");
+    }
+    FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
+    if (ddict) {   /* NULL ddict is equivalent to no dictionary */
+        ZSTD_copyDDictParameters(dctx, ddict);
+    }
+    return 0;
+}
+
+/*! ZSTD_getDictID_fromDict() :
+ *  Provides the dictID stored within dictionary.
+ *  if @return == 0, the dictionary is not conformant with Zstandard specification.
+ *  It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
+{
+    if (dictSize < 8) return 0;
+    if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
+    return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
+}
+
+/*! ZSTD_getDictID_fromFrame() :
+ *  Provides the dictID required to decompress frame stored within `src`.
+ *  If @return == 0, the dictID could not be decoded.
+ *  This could for one of the following reasons :
+ *  - The frame does not require a dictionary (most common case).
+ *  - The frame was built with dictID intentionally removed.
+ *    Needed dictionary is a hidden piece of information.
+ *    Note : this use case also happens when using a non-conformant dictionary.
+ *  - `srcSize` is too small, and as a result, frame header could not be decoded.
+ *    Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
+ *  - This is not a Zstandard frame.
+ *  When identifying the exact failure cause, it's possible to use
+ *  ZSTD_getFrameHeader(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
+{
+    ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
+    size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
+    if (ZSTD_isError(hError)) return 0;
+    return zfp.dictID;
+}
+
+
+/*! ZSTD_decompress_usingDDict() :
+*   Decompression using a pre-digested Dictionary
+*   Use dictionary without significant overhead. */
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+                                  void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                            const ZSTD_DDict* ddict)
+{
+    /* pass content and size in case legacy frames are encountered */
+    return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
+                                     NULL, 0,
+                                     ddict);
+}
+
+
+/*=====================================
+*   Streaming decompression
+*====================================*/
+
+ZSTD_DStream* ZSTD_createDStream(void)
+{
+    DEBUGLOG(3, "ZSTD_createDStream");
+    return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+}
+
+ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
+{
+    return ZSTD_initStaticDCtx(workspace, workspaceSize);
+}
+
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
+{
+    return ZSTD_createDCtx_internal(customMem);
+}
+
+size_t ZSTD_freeDStream(ZSTD_DStream* zds)
+{
+    return ZSTD_freeDCtx(zds);
+}
+
+
+/* ***  Initialization  *** */
+
+size_t ZSTD_DStreamInSize(void)  { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
+
+size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
+                                   const void* dict, size_t dictSize,
+                                         ZSTD_dictLoadMethod_e dictLoadMethod,
+                                         ZSTD_dictContentType_e dictContentType)
+{
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+    ZSTD_clearDict(dctx);
+    if (dict && dictSize != 0) {
+        dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
+        RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");
+        dctx->ddict = dctx->ddictLocal;
+        dctx->dictUses = ZSTD_use_indefinitely;
+    }
+    return 0;
+}
+
+size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+    return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
+}
+
+size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+    return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
+}
+
+size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
+{
+    FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");
+    dctx->dictUses = ZSTD_use_once;
+    return 0;
+}
+
+size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
+{
+    return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent);
+}
+
+
+/* ZSTD_initDStream_usingDict() :
+ * return : expected size, aka ZSTD_startingInputLength().
+ * this function cannot fail */
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
+{
+    DEBUGLOG(4, "ZSTD_initDStream_usingDict");
+    FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");
+    FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");
+    return ZSTD_startingInputLength(zds->format);
+}
+
+/* note : this variant can't fail */
+size_t ZSTD_initDStream(ZSTD_DStream* zds)
+{
+    DEBUGLOG(4, "ZSTD_initDStream");
+    FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), "");
+    FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), "");
+    return ZSTD_startingInputLength(zds->format);
+}
+
+/* ZSTD_initDStream_usingDDict() :
+ * ddict will just be referenced, and must outlive decompression session
+ * this function cannot fail */
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
+{
+    DEBUGLOG(4, "ZSTD_initDStream_usingDDict");
+    FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
+    FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
+    return ZSTD_startingInputLength(dctx->format);
+}
+
+/* ZSTD_resetDStream() :
+ * return : expected size, aka ZSTD_startingInputLength().
+ * this function cannot fail */
+size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
+{
+    DEBUGLOG(4, "ZSTD_resetDStream");
+    FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
+    return ZSTD_startingInputLength(dctx->format);
+}
+
+
+size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+    ZSTD_clearDict(dctx);
+    if (ddict) {
+        dctx->ddict = ddict;
+        dctx->dictUses = ZSTD_use_indefinitely;
+        if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
+            if (dctx->ddictSet == NULL) {
+                dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
+                if (!dctx->ddictSet) {
+                    RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
+                }
+            }
+            assert(!dctx->staticSize);  /* Impossible: ddictSet cannot have been allocated if static dctx */
+            FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
+        }
+    }
+    return 0;
+}
+
+/* ZSTD_DCtx_setMaxWindowSize() :
+ * note : no direct equivalence in ZSTD_DCtx_setParameter,
+ * since this version sets windowSize, and the other sets windowLog */
+size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
+{
+    ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
+    size_t const min = (size_t)1 << bounds.lowerBound;
+    size_t const max = (size_t)1 << bounds.upperBound;
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+    RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");
+    RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");
+    dctx->maxWindowSize = maxWindowSize;
+    return 0;
+}
+
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
+{
+    return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format);
+}
+
+ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
+{
+    ZSTD_bounds bounds = { 0, 0, 0 };
+    switch(dParam) {
+        case ZSTD_d_windowLogMax:
+            bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;
+            bounds.upperBound = ZSTD_WINDOWLOG_MAX;
+            return bounds;
+        case ZSTD_d_format:
+            bounds.lowerBound = (int)ZSTD_f_zstd1;
+            bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
+            ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
+            return bounds;
+        case ZSTD_d_stableOutBuffer:
+            bounds.lowerBound = (int)ZSTD_bm_buffered;
+            bounds.upperBound = (int)ZSTD_bm_stable;
+            return bounds;
+        case ZSTD_d_forceIgnoreChecksum:
+            bounds.lowerBound = (int)ZSTD_d_validateChecksum;
+            bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
+            return bounds;
+        case ZSTD_d_refMultipleDDicts:
+            bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
+            bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
+            return bounds;
+        case ZSTD_d_disableHuffmanAssembly:
+            bounds.lowerBound = 0;
+            bounds.upperBound = 1;
+            return bounds;
+        case ZSTD_d_maxBlockSize:
+            bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+            bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+            return bounds;
+
+        default:;
+    }
+    bounds.error = ERROR(parameter_unsupported);
+    return bounds;
+}
+
+/* ZSTD_dParam_withinBounds:
+ * @return 1 if value is within dParam bounds,
+ * 0 otherwise */
+static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
+{
+    ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);
+    if (ZSTD_isError(bounds.error)) return 0;
+    if (value < bounds.lowerBound) return 0;
+    if (value > bounds.upperBound) return 0;
+    return 1;
+}
+
+#define CHECK_DBOUNDS(p,v) {                \
+    RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
+}
+
+size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value)
+{
+    switch (param) {
+        case ZSTD_d_windowLogMax:
+            *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize);
+            return 0;
+        case ZSTD_d_format:
+            *value = (int)dctx->format;
+            return 0;
+        case ZSTD_d_stableOutBuffer:
+            *value = (int)dctx->outBufferMode;
+            return 0;
+        case ZSTD_d_forceIgnoreChecksum:
+            *value = (int)dctx->forceIgnoreChecksum;
+            return 0;
+        case ZSTD_d_refMultipleDDicts:
+            *value = (int)dctx->refMultipleDDicts;
+            return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            *value = (int)dctx->disableHufAsm;
+            return 0;
+        case ZSTD_d_maxBlockSize:
+            *value = dctx->maxBlockSizeParam;
+            return 0;
+        default:;
+    }
+    RETURN_ERROR(parameter_unsupported, "");
+}
+
+size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
+{
+    RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+    switch(dParam) {
+        case ZSTD_d_windowLogMax:
+            if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
+            CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
+            dctx->maxWindowSize = ((size_t)1) << value;
+            return 0;
+        case ZSTD_d_format:
+            CHECK_DBOUNDS(ZSTD_d_format, value);
+            dctx->format = (ZSTD_format_e)value;
+            return 0;
+        case ZSTD_d_stableOutBuffer:
+            CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
+            dctx->outBufferMode = (ZSTD_bufferMode_e)value;
+            return 0;
+        case ZSTD_d_forceIgnoreChecksum:
+            CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
+            dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
+            return 0;
+        case ZSTD_d_refMultipleDDicts:
+            CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
+            if (dctx->staticSize != 0) {
+                RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
+            }
+            dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
+            return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value);
+            dctx->disableHufAsm = value != 0;
+            return 0;
+        case ZSTD_d_maxBlockSize:
+            if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value);
+            dctx->maxBlockSizeParam = value;
+            return 0;
+        default:;
+    }
+    RETURN_ERROR(parameter_unsupported, "");
+}
+
+size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
+{
+    if ( (reset == ZSTD_reset_session_only)
+      || (reset == ZSTD_reset_session_and_parameters) ) {
+        dctx->streamStage = zdss_init;
+        dctx->noForwardProgress = 0;
+        dctx->isFrameDecompression = 1;
+    }
+    if ( (reset == ZSTD_reset_parameters)
+      || (reset == ZSTD_reset_session_and_parameters) ) {
+        RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+        ZSTD_clearDict(dctx);
+        ZSTD_DCtx_resetParameters(dctx);
+    }
+    return 0;
+}
+
+
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
+{
+    return ZSTD_sizeof_DCtx(dctx);
+}
+
+static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax)
+{
+    size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax);
+    /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block
+     * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing
+     * the block at the beginning of the output buffer, and maintain a full window.
+     *
+     * We need another blockSize worth of buffer so that we can store split
+     * literals at the end of the block without overwriting the extDict window.
+     */
+    unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2);
+    unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
+    size_t const minRBSize = (size_t) neededSize;
+    RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
+                    frameParameter_windowTooLarge, "");
+    return minRBSize;
+}
+
+size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
+{
+    return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX);
+}
+
+size_t ZSTD_estimateDStreamSize(size_t windowSize)
+{
+    size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+    size_t const inBuffSize = blockSize;  /* no block can be larger */
+    size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN);
+    return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
+}
+
+size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
+{
+    U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;   /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
+    ZSTD_frameHeader zfh;
+    size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
+    if (ZSTD_isError(err)) return err;
+    RETURN_ERROR_IF(err>0, srcSize_wrong, "");
+    RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
+                    frameParameter_windowTooLarge, "");
+    return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
+}
+
+
+/* *****   Decompression   ***** */
+
+static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
+{
+    return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;
+}
+
+static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
+{
+    if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
+        zds->oversizedDuration++;
+    else
+        zds->oversizedDuration = 0;
+}
+
+static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds)
+{
+    return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;
+}
+
+/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */
+static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)
+{
+    ZSTD_outBuffer const expect = zds->expectedOutBuffer;
+    /* No requirement when ZSTD_obm_stable is not enabled. */
+    if (zds->outBufferMode != ZSTD_bm_stable)
+        return 0;
+    /* Any buffer is allowed in zdss_init, this must be the same for every other call until
+     * the context is reset.
+     */
+    if (zds->streamStage == zdss_init)
+        return 0;
+    /* The buffer must match our expectation exactly. */
+    if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
+        return 0;
+    RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
+}
+
+/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
+ * and updates the stage and the output buffer state. This call is extracted so it can be
+ * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.
+ * NOTE: You must break after calling this function since the streamStage is modified.
+ */
+static size_t ZSTD_decompressContinueStream(
+            ZSTD_DStream* zds, char** op, char* oend,
+            void const* src, size_t srcSize) {
+    int const isSkipFrame = ZSTD_isSkipFrame(zds);
+    if (zds->outBufferMode == ZSTD_bm_buffered) {
+        size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
+        size_t const decodedSize = ZSTD_decompressContinue(zds,
+                zds->outBuff + zds->outStart, dstSize, src, srcSize);
+        FORWARD_IF_ERROR(decodedSize, "");
+        if (!decodedSize && !isSkipFrame) {
+            zds->streamStage = zdss_read;
+        } else {
+            zds->outEnd = zds->outStart + decodedSize;
+            zds->streamStage = zdss_flush;
+        }
+    } else {
+        /* Write directly into the output buffer */
+        size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
+        size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
+        FORWARD_IF_ERROR(decodedSize, "");
+        *op += decodedSize;
+        /* Flushing is not needed. */
+        zds->streamStage = zdss_read;
+        assert(*op <= oend);
+        assert(zds->outBufferMode == ZSTD_bm_stable);
+    }
+    return 0;
+}
+
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+    const char* const src = (const char*)input->src;
+    const char* const istart = input->pos != 0 ? src + input->pos : src;
+    const char* const iend = input->size != 0 ? src + input->size : src;
+    const char* ip = istart;
+    char* const dst = (char*)output->dst;
+    char* const ostart = output->pos != 0 ? dst + output->pos : dst;
+    char* const oend = output->size != 0 ? dst + output->size : dst;
+    char* op = ostart;
+    U32 someMoreWork = 1;
+
+    DEBUGLOG(5, "ZSTD_decompressStream");
+    RETURN_ERROR_IF(
+        input->pos > input->size,
+        srcSize_wrong,
+        "forbidden. in: pos: %u   vs size: %u",
+        (U32)input->pos, (U32)input->size);
+    RETURN_ERROR_IF(
+        output->pos > output->size,
+        dstSize_tooSmall,
+        "forbidden. out: pos: %u   vs size: %u",
+        (U32)output->pos, (U32)output->size);
+    DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
+    FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), "");
+
+    while (someMoreWork) {
+        switch(zds->streamStage)
+        {
+        case zdss_init :
+            DEBUGLOG(5, "stage zdss_init => transparent reset ");
+            zds->streamStage = zdss_loadHeader;
+            zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+            zds->legacyVersion = 0;
+#endif
+            zds->hostageByte = 0;
+            zds->expectedOutBuffer = *output;
+            ZSTD_FALLTHROUGH;
+
+        case zdss_loadHeader :
+            DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+            if (zds->legacyVersion) {
+                RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+                    "legacy support is incompatible with static dctx");
+                {   size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
+                    if (hint==0) zds->streamStage = zdss_init;
+                    return hint;
+            }   }
+#endif
+            {   size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
+                if (zds->refMultipleDDicts && zds->ddictSet) {
+                    ZSTD_DCtx_selectFrameDDict(zds);
+                }
+                if (ZSTD_isError(hSize)) {
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+                    U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
+                    if (legacyVersion) {
+                        ZSTD_DDict const* const ddict = ZSTD_getDDict(zds);
+                        const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL;
+                        size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0;
+                        DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
+                        RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+                            "legacy support is incompatible with static dctx");
+                        FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
+                                    zds->previousLegacyVersion, legacyVersion,
+                                    dict, dictSize), "");
+                        zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
+                        {   size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
+                            if (hint==0) zds->streamStage = zdss_init;   /* or stay in stage zdss_loadHeader */
+                            return hint;
+                    }   }
+#endif
+                    return hSize;   /* error */
+                }
+                if (hSize != 0) {   /* need more input */
+                    size_t const toLoad = hSize - zds->lhSize;   /* if hSize!=0, hSize > zds->lhSize */
+                    size_t const remainingInput = (size_t)(iend-ip);
+                    assert(iend >= ip);
+                    if (toLoad > remainingInput) {   /* not enough input to load full header */
+                        if (remainingInput > 0) {
+                            ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
+                            zds->lhSize += remainingInput;
+                        }
+                        input->pos = input->size;
+                        /* check first few bytes */
+                        FORWARD_IF_ERROR(
+                            ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format),
+                            "First few bytes detected incorrect" );
+                        /* return hint input size */
+                        return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
+                    }
+                    assert(ip != NULL);
+                    ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
+                    break;
+            }   }
+
+            /* check for single-pass mode opportunity */
+            if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+                && zds->fParams.frameType != ZSTD_skippableFrame
+                && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
+                size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format);
+                if (cSize <= (size_t)(iend-istart)) {
+                    /* shortcut : using single-pass mode */
+                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
+                    if (ZSTD_isError(decompressedSize)) return decompressedSize;
+                    DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()");
+                    assert(istart != NULL);
+                    ip = istart + cSize;
+                    op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
+                    zds->expected = 0;
+                    zds->streamStage = zdss_init;
+                    someMoreWork = 0;
+                    break;
+            }   }
+
+            /* Check output buffer is large enough for ZSTD_odm_stable. */
+            if (zds->outBufferMode == ZSTD_bm_stable
+                && zds->fParams.frameType != ZSTD_skippableFrame
+                && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+                && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
+                RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small");
+            }
+
+            /* Consume header (see ZSTDds_decodeFrameHeader) */
+            DEBUGLOG(4, "Consume header");
+            FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
+
+            if (zds->format == ZSTD_f_zstd1
+                && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
+                zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
+                zds->stage = ZSTDds_skipFrame;
+            } else {
+                FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), "");
+                zds->expected = ZSTD_blockHeaderSize;
+                zds->stage = ZSTDds_decodeBlockHeader;
+            }
+
+            /* control buffer memory usage */
+            DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
+                        (U32)(zds->fParams.windowSize >>10),
+                        (U32)(zds->maxWindowSize >> 10) );
+            zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
+            RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
+                            frameParameter_windowTooLarge, "");
+            if (zds->maxBlockSizeParam != 0)
+                zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam);
+
+            /* Adapt buffer sizes to frame header instructions */
+            {   size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
+                size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
+                        ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax)
+                        : 0;
+
+                ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
+
+                {   int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
+                    int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
+
+                    if (tooSmall || tooLarge) {
+                        size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
+                        DEBUGLOG(4, "inBuff  : from %u to %u",
+                                    (U32)zds->inBuffSize, (U32)neededInBuffSize);
+                        DEBUGLOG(4, "outBuff : from %u to %u",
+                                    (U32)zds->outBuffSize, (U32)neededOutBuffSize);
+                        if (zds->staticSize) {  /* static DCtx */
+                            DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
+                            assert(zds->staticSize >= sizeof(ZSTD_DCtx));  /* controlled at init */
+                            RETURN_ERROR_IF(
+                                bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
+                                memory_allocation, "");
+                        } else {
+                            ZSTD_customFree(zds->inBuff, zds->customMem);
+                            zds->inBuffSize = 0;
+                            zds->outBuffSize = 0;
+                            zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem);
+                            RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
+                        }
+                        zds->inBuffSize = neededInBuffSize;
+                        zds->outBuff = zds->inBuff + zds->inBuffSize;
+                        zds->outBuffSize = neededOutBuffSize;
+            }   }   }
+            zds->streamStage = zdss_read;
+            ZSTD_FALLTHROUGH;
+
+        case zdss_read:
+            DEBUGLOG(5, "stage zdss_read");
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip));
+                DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
+                if (neededInSize==0) {  /* end of frame */
+                    zds->streamStage = zdss_init;
+                    someMoreWork = 0;
+                    break;
+                }
+                if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
+                    FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
+                    assert(ip != NULL);
+                    ip += neededInSize;
+                    /* Function modifies the stage so we must break */
+                    break;
+            }   }
+            if (ip==iend) { someMoreWork = 0; break; }   /* no more input */
+            zds->streamStage = zdss_load;
+            ZSTD_FALLTHROUGH;
+
+        case zdss_load:
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
+                size_t const toLoad = neededInSize - zds->inPos;
+                int const isSkipFrame = ZSTD_isSkipFrame(zds);
+                size_t loadedSize;
+                /* At this point we shouldn't be decompressing a block that we can stream. */
+                assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)));
+                if (isSkipFrame) {
+                    loadedSize = MIN(toLoad, (size_t)(iend-ip));
+                } else {
+                    RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
+                                    corruption_detected,
+                                    "should never happen");
+                    loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
+                }
+                if (loadedSize != 0) {
+                    /* ip may be NULL */
+                    ip += loadedSize;
+                    zds->inPos += loadedSize;
+                }
+                if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
+
+                /* decode loaded input */
+                zds->inPos = 0;   /* input is consumed */
+                FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), "");
+                /* Function modifies the stage so we must break */
+                break;
+            }
+        case zdss_flush:
+            {
+                size_t const toFlushSize = zds->outEnd - zds->outStart;
+                size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
+
+                op = op ? op + flushedSize : op;
+
+                zds->outStart += flushedSize;
+                if (flushedSize == toFlushSize) {  /* flush completed */
+                    zds->streamStage = zdss_read;
+                    if ( (zds->outBuffSize < zds->fParams.frameContentSize)
+                        && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
+                        DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
+                                (int)(zds->outBuffSize - zds->outStart),
+                                (U32)zds->fParams.blockSizeMax);
+                        zds->outStart = zds->outEnd = 0;
+                    }
+                    break;
+            }   }
+            /* cannot complete flush */
+            someMoreWork = 0;
+            break;
+
+        default:
+            assert(0);    /* impossible */
+            RETURN_ERROR(GENERIC, "impossible to reach");   /* some compilers require default to do something */
+    }   }
+
+    /* result */
+    input->pos = (size_t)(ip - (const char*)(input->src));
+    output->pos = (size_t)(op - (char*)(output->dst));
+
+    /* Update the expected output buffer for ZSTD_obm_stable. */
+    zds->expectedOutBuffer = *output;
+
+    if ((ip==istart) && (op==ostart)) {  /* no forward progress */
+        zds->noForwardProgress ++;
+        if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
+            RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, "");
+            RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, "");
+            assert(0);
+        }
+    } else {
+        zds->noForwardProgress = 0;
+    }
+    {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
+        if (!nextSrcSizeHint) {   /* frame fully decoded */
+            if (zds->outEnd == zds->outStart) {  /* output fully flushed */
+                if (zds->hostageByte) {
+                    if (input->pos >= input->size) {
+                        /* can't release hostage (not present) */
+                        zds->streamStage = zdss_read;
+                        return 1;
+                    }
+                    input->pos++;  /* release hostage */
+                }   /* zds->hostageByte */
+                return 0;
+            }  /* zds->outEnd == zds->outStart */
+            if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+                input->pos--;   /* note : pos > 0, otherwise, impossible to finish reading last block */
+                zds->hostageByte=1;
+            }
+            return 1;
+        }  /* nextSrcSizeHint==0 */
+        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block);   /* preload header of next block */
+        assert(zds->inPos <= nextSrcSizeHint);
+        nextSrcSizeHint -= zds->inPos;   /* part already loaded*/
+        return nextSrcSizeHint;
+    }
+}
+
+size_t ZSTD_decompressStream_simpleArgs (
+                            ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos)
+{
+    ZSTD_outBuffer output;
+    ZSTD_inBuffer  input;
+    output.dst = dst;
+    output.size = dstCapacity;
+    output.pos = *dstPos;
+    input.src = src;
+    input.size = srcSize;
+    input.pos = *srcPos;
+    {   size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
+        *dstPos = output.pos;
+        *srcPos = input.pos;
+        return cErr;
+    }
+}
+/**** ended inlining decompress/zstd_decompress.c ****/
+/**** start inlining decompress/zstd_decompress_block.c ****/
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* zstd_decompress_block :
+ * this module takes care of decompressing _compressed_ block */
+
+/*-*******************************************************
+*  Dependencies
+*********************************************************/
+/**** skipping file: ../common/zstd_deps.h ****/
+/**** skipping file: ../common/compiler.h ****/
+/**** skipping file: ../common/cpu.h ****/
+/**** skipping file: ../common/mem.h ****/
+#define FSE_STATIC_LINKING_ONLY
+/**** skipping file: ../common/fse.h ****/
+/**** skipping file: ../common/huf.h ****/
+/**** skipping file: ../common/zstd_internal.h ****/
+/**** skipping file: zstd_decompress_internal.h ****/
+/**** skipping file: zstd_ddict.h ****/
+/**** skipping file: zstd_decompress_block.h ****/
+/**** skipping file: ../common/bits.h ****/
+
+/*_*******************************************************
+*  Macros
+**********************************************************/
+
+/* These two optional macros force the use one way or another of the two
+ * ZSTD_decompressSequences implementations. You can't force in both directions
+ * at the same time.
+ */
+#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+    defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!"
+#endif
+
+
+/*_*******************************************************
+*  Memory operations
+**********************************************************/
+static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
+
+
+/*-*************************************************************
+ *   Block decoding
+ ***************************************************************/
+
+static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx)
+{
+    size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX;
+    assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX);
+    return blockSizeMax;
+}
+
+/*! ZSTD_getcBlockSize() :
+ *  Provides the size of compressed block from block header `src` */
+size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
+                          blockProperties_t* bpPtr)
+{
+    RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");
+
+    {   U32 const cBlockHeader = MEM_readLE24(src);
+        U32 const cSize = cBlockHeader >> 3;
+        bpPtr->lastBlock = cBlockHeader & 1;
+        bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
+        bpPtr->origSize = cSize;   /* only useful for RLE */
+        if (bpPtr->blockType == bt_rle) return 1;
+        RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
+        return cSize;
+    }
+}
+
+/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
+static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
+    const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
+{
+    size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
+    assert(litSize <= blockSizeMax);
+    assert(dctx->isFrameDecompression || streaming == not_streaming);
+    assert(expectedWriteSize <= blockSizeMax);
+    if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) {
+        /* If we aren't streaming, we can just put the literals after the output
+         * of the current block. We don't need to worry about overwriting the
+         * extDict of our window, because it doesn't exist.
+         * So if we have space after the end of the block, just put it there.
+         */
+        dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH;
+        dctx->litBufferEnd = dctx->litBuffer + litSize;
+        dctx->litBufferLocation = ZSTD_in_dst;
+    } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) {
+        /* Literals fit entirely within the extra buffer, put them there to avoid
+         * having to split the literals.
+         */
+        dctx->litBuffer = dctx->litExtraBuffer;
+        dctx->litBufferEnd = dctx->litBuffer + litSize;
+        dctx->litBufferLocation = ZSTD_not_in_dst;
+    } else {
+        assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE);
+        /* Literals must be split between the output block and the extra lit
+         * buffer. We fill the extra lit buffer with the tail of the literals,
+         * and put the rest of the literals at the end of the block, with
+         * WILDCOPY_OVERLENGTH of buffer room to allow for overreads.
+         * This MUST not write more than our maxBlockSize beyond dst, because in
+         * streaming mode, that could overwrite part of our extDict window.
+         */
+        if (splitImmediately) {
+            /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
+            dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
+            dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
+        } else {
+            /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */
+            dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
+            dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
+        }
+        dctx->litBufferLocation = ZSTD_split;
+        assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize);
+    }
+}
+
+/*! ZSTD_decodeLiteralsBlock() :
+ * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
+ * in the dstBuffer.  If there is room to do so, it will be stored in full in the excess dst space after where the current
+ * block will be output.  Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
+ * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
+ *
+ * @return : nb of bytes read from src (< srcSize )
+ *  note : symbol not declared but exposed for fullbench */
+static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
+                          const void* src, size_t srcSize,   /* note : srcSize < BLOCKSIZE */
+                          void* dst, size_t dstCapacity, const streaming_operation streaming)
+{
+    DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
+    RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
+
+    {   const BYTE* const istart = (const BYTE*) src;
+        symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
+        size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
+
+        switch(litEncType)
+        {
+        case set_repeat:
+            DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
+            RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, "");
+            ZSTD_FALLTHROUGH;
+
+        case set_compressed:
+            RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3");
+            {   size_t lhSize, litSize, litCSize;
+                U32 singleStream=0;
+                U32 const lhlCode = (istart[0] >> 2) & 3;
+                U32 const lhc = MEM_readLE32(istart);
+                size_t hufSuccess;
+                size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
+                int const flags = 0
+                    | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
+                    | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
+                switch(lhlCode)
+                {
+                case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
+                    /* 2 - 2 - 10 - 10 */
+                    singleStream = !lhlCode;
+                    lhSize = 3;
+                    litSize  = (lhc >> 4) & 0x3FF;
+                    litCSize = (lhc >> 14) & 0x3FF;
+                    break;
+                case 2:
+                    /* 2 - 2 - 14 - 14 */
+                    lhSize = 4;
+                    litSize  = (lhc >> 4) & 0x3FFF;
+                    litCSize = lhc >> 18;
+                    break;
+                case 3:
+                    /* 2 - 2 - 18 - 18 */
+                    lhSize = 5;
+                    litSize  = (lhc >> 4) & 0x3FFFF;
+                    litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
+                    break;
+                }
+                RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+                RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
+                if (!singleStream)
+                    RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
+                        "Not enough literals (%zu) for the 4-streams mode (min %u)",
+                        litSize, MIN_LITERALS_FOR_4_STREAMS);
+                RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
+                RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
+                ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
+
+                /* prefetch huffman table if cold */
+                if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
+                    PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
+                }
+
+                if (litEncType==set_repeat) {
+                    if (singleStream) {
+                        hufSuccess = HUF_decompress1X_usingDTable(
+                            dctx->litBuffer, litSize, istart+lhSize, litCSize,
+                            dctx->HUFptr, flags);
+                    } else {
+                        assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
+                        hufSuccess = HUF_decompress4X_usingDTable(
+                            dctx->litBuffer, litSize, istart+lhSize, litCSize,
+                            dctx->HUFptr, flags);
+                    }
+                } else {
+                    if (singleStream) {
+#if defined(HUF_FORCE_DECOMPRESS_X2)
+                        hufSuccess = HUF_decompress1X_DCtx_wksp(
+                            dctx->entropy.hufTable, dctx->litBuffer, litSize,
+                            istart+lhSize, litCSize, dctx->workspace,
+                            sizeof(dctx->workspace), flags);
+#else
+                        hufSuccess = HUF_decompress1X1_DCtx_wksp(
+                            dctx->entropy.hufTable, dctx->litBuffer, litSize,
+                            istart+lhSize, litCSize, dctx->workspace,
+                            sizeof(dctx->workspace), flags);
+#endif
+                    } else {
+                        hufSuccess = HUF_decompress4X_hufOnly_wksp(
+                            dctx->entropy.hufTable, dctx->litBuffer, litSize,
+                            istart+lhSize, litCSize, dctx->workspace,
+                            sizeof(dctx->workspace), flags);
+                    }
+                }
+                if (dctx->litBufferLocation == ZSTD_split)
+                {
+                    assert(litSize > ZSTD_LITBUFFEREXTRASIZE);
+                    ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
+                    ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
+                    dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
+                    dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
+                    assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax);
+                }
+
+                RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
+
+                dctx->litPtr = dctx->litBuffer;
+                dctx->litSize = litSize;
+                dctx->litEntropy = 1;
+                if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
+                return litCSize + lhSize;
+            }
+
+        case set_basic:
+            {   size_t litSize, lhSize;
+                U32 const lhlCode = ((istart[0]) >> 2) & 3;
+                size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
+                switch(lhlCode)
+                {
+                case 0: case 2: default:   /* note : default is impossible, since lhlCode into [0..3] */
+                    lhSize = 1;
+                    litSize = istart[0] >> 3;
+                    break;
+                case 1:
+                    lhSize = 2;
+                    litSize = MEM_readLE16(istart) >> 4;
+                    break;
+                case 3:
+                    lhSize = 3;
+                    RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3");
+                    litSize = MEM_readLE24(istart) >> 4;
+                    break;
+                }
+
+                RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+                RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
+                RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
+                ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
+                if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) {  /* risk reading beyond src buffer with wildcopy */
+                    RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
+                    if (dctx->litBufferLocation == ZSTD_split)
+                    {
+                        ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
+                        ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
+                    }
+                    else
+                    {
+                        ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
+                    }
+                    dctx->litPtr = dctx->litBuffer;
+                    dctx->litSize = litSize;
+                    return lhSize+litSize;
+                }
+                /* direct reference into compressed stream */
+                dctx->litPtr = istart+lhSize;
+                dctx->litSize = litSize;
+                dctx->litBufferEnd = dctx->litPtr + litSize;
+                dctx->litBufferLocation = ZSTD_not_in_dst;
+                return lhSize+litSize;
+            }
+
+        case set_rle:
+            {   U32 const lhlCode = ((istart[0]) >> 2) & 3;
+                size_t litSize, lhSize;
+                size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
+                switch(lhlCode)
+                {
+                case 0: case 2: default:   /* note : default is impossible, since lhlCode into [0..3] */
+                    lhSize = 1;
+                    litSize = istart[0] >> 3;
+                    break;
+                case 1:
+                    lhSize = 2;
+                    RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3");
+                    litSize = MEM_readLE16(istart) >> 4;
+                    break;
+                case 3:
+                    lhSize = 3;
+                    RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4");
+                    litSize = MEM_readLE24(istart) >> 4;
+                    break;
+                }
+                RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+                RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
+                RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
+                ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
+                if (dctx->litBufferLocation == ZSTD_split)
+                {
+                    ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
+                    ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
+                }
+                else
+                {
+                    ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
+                }
+                dctx->litPtr = dctx->litBuffer;
+                dctx->litSize = litSize;
+                return lhSize+1;
+            }
+        default:
+            RETURN_ERROR(corruption_detected, "impossible");
+        }
+    }
+}
+
+/* Hidden declaration for fullbench */
+size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
+                          const void* src, size_t srcSize,
+                          void* dst, size_t dstCapacity);
+size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
+                          const void* src, size_t srcSize,
+                          void* dst, size_t dstCapacity)
+{
+    dctx->isFrameDecompression = 0;
+    return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming);
+}
+
+/* Default FSE distribution tables.
+ * These are pre-calculated FSE decoding tables using default distributions as defined in specification :
+ * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
+ * They were generated programmatically with following method :
+ * - start from default distributions, present in /lib/common/zstd_internal.h
+ * - generate tables normally, using ZSTD_buildFSETable()
+ * - printout the content of tables
+ * - prettify output, report below, test with fuzzer to ensure it's correct */
+
+/* Default FSE distribution table for Literal Lengths */
+static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
+     {  1,  1,  1, LL_DEFAULTNORMLOG},  /* header : fastMode, tableLog */
+     /* nextState, nbAddBits, nbBits, baseVal */
+     {  0,  0,  4,    0},  { 16,  0,  4,    0},
+     { 32,  0,  5,    1},  {  0,  0,  5,    3},
+     {  0,  0,  5,    4},  {  0,  0,  5,    6},
+     {  0,  0,  5,    7},  {  0,  0,  5,    9},
+     {  0,  0,  5,   10},  {  0,  0,  5,   12},
+     {  0,  0,  6,   14},  {  0,  1,  5,   16},
+     {  0,  1,  5,   20},  {  0,  1,  5,   22},
+     {  0,  2,  5,   28},  {  0,  3,  5,   32},
+     {  0,  4,  5,   48},  { 32,  6,  5,   64},
+     {  0,  7,  5,  128},  {  0,  8,  6,  256},
+     {  0, 10,  6, 1024},  {  0, 12,  6, 4096},
+     { 32,  0,  4,    0},  {  0,  0,  4,    1},
+     {  0,  0,  5,    2},  { 32,  0,  5,    4},
+     {  0,  0,  5,    5},  { 32,  0,  5,    7},
+     {  0,  0,  5,    8},  { 32,  0,  5,   10},
+     {  0,  0,  5,   11},  {  0,  0,  6,   13},
+     { 32,  1,  5,   16},  {  0,  1,  5,   18},
+     { 32,  1,  5,   22},  {  0,  2,  5,   24},
+     { 32,  3,  5,   32},  {  0,  3,  5,   40},
+     {  0,  6,  4,   64},  { 16,  6,  4,   64},
+     { 32,  7,  5,  128},  {  0,  9,  6,  512},
+     {  0, 11,  6, 2048},  { 48,  0,  4,    0},
+     { 16,  0,  4,    1},  { 32,  0,  5,    2},
+     { 32,  0,  5,    3},  { 32,  0,  5,    5},
+     { 32,  0,  5,    6},  { 32,  0,  5,    8},
+     { 32,  0,  5,    9},  { 32,  0,  5,   11},
+     { 32,  0,  5,   12},  {  0,  0,  6,   15},
+     { 32,  1,  5,   18},  { 32,  1,  5,   20},
+     { 32,  2,  5,   24},  { 32,  2,  5,   28},
+     { 32,  3,  5,   40},  { 32,  4,  5,   48},
+     {  0, 16,  6,65536},  {  0, 15,  6,32768},
+     {  0, 14,  6,16384},  {  0, 13,  6, 8192},
+};   /* LL_defaultDTable */
+
+/* Default FSE distribution table for Offset Codes */
+static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
+    {  1,  1,  1, OF_DEFAULTNORMLOG},  /* header : fastMode, tableLog */
+    /* nextState, nbAddBits, nbBits, baseVal */
+    {  0,  0,  5,    0},     {  0,  6,  4,   61},
+    {  0,  9,  5,  509},     {  0, 15,  5,32765},
+    {  0, 21,  5,2097149},   {  0,  3,  5,    5},
+    {  0,  7,  4,  125},     {  0, 12,  5, 4093},
+    {  0, 18,  5,262141},    {  0, 23,  5,8388605},
+    {  0,  5,  5,   29},     {  0,  8,  4,  253},
+    {  0, 14,  5,16381},     {  0, 20,  5,1048573},
+    {  0,  2,  5,    1},     { 16,  7,  4,  125},
+    {  0, 11,  5, 2045},     {  0, 17,  5,131069},
+    {  0, 22,  5,4194301},   {  0,  4,  5,   13},
+    { 16,  8,  4,  253},     {  0, 13,  5, 8189},
+    {  0, 19,  5,524285},    {  0,  1,  5,    1},
+    { 16,  6,  4,   61},     {  0, 10,  5, 1021},
+    {  0, 16,  5,65533},     {  0, 28,  5,268435453},
+    {  0, 27,  5,134217725}, {  0, 26,  5,67108861},
+    {  0, 25,  5,33554429},  {  0, 24,  5,16777213},
+};   /* OF_defaultDTable */
+
+
+/* Default FSE distribution table for Match Lengths */
+static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
+    {  1,  1,  1, ML_DEFAULTNORMLOG},  /* header : fastMode, tableLog */
+    /* nextState, nbAddBits, nbBits, baseVal */
+    {  0,  0,  6,    3},  {  0,  0,  4,    4},
+    { 32,  0,  5,    5},  {  0,  0,  5,    6},
+    {  0,  0,  5,    8},  {  0,  0,  5,    9},
+    {  0,  0,  5,   11},  {  0,  0,  6,   13},
+    {  0,  0,  6,   16},  {  0,  0,  6,   19},
+    {  0,  0,  6,   22},  {  0,  0,  6,   25},
+    {  0,  0,  6,   28},  {  0,  0,  6,   31},
+    {  0,  0,  6,   34},  {  0,  1,  6,   37},
+    {  0,  1,  6,   41},  {  0,  2,  6,   47},
+    {  0,  3,  6,   59},  {  0,  4,  6,   83},
+    {  0,  7,  6,  131},  {  0,  9,  6,  515},
+    { 16,  0,  4,    4},  {  0,  0,  4,    5},
+    { 32,  0,  5,    6},  {  0,  0,  5,    7},
+    { 32,  0,  5,    9},  {  0,  0,  5,   10},
+    {  0,  0,  6,   12},  {  0,  0,  6,   15},
+    {  0,  0,  6,   18},  {  0,  0,  6,   21},
+    {  0,  0,  6,   24},  {  0,  0,  6,   27},
+    {  0,  0,  6,   30},  {  0,  0,  6,   33},
+    {  0,  1,  6,   35},  {  0,  1,  6,   39},
+    {  0,  2,  6,   43},  {  0,  3,  6,   51},
+    {  0,  4,  6,   67},  {  0,  5,  6,   99},
+    {  0,  8,  6,  259},  { 32,  0,  4,    4},
+    { 48,  0,  4,    4},  { 16,  0,  4,    5},
+    { 32,  0,  5,    7},  { 32,  0,  5,    8},
+    { 32,  0,  5,   10},  { 32,  0,  5,   11},
+    {  0,  0,  6,   14},  {  0,  0,  6,   17},
+    {  0,  0,  6,   20},  {  0,  0,  6,   23},
+    {  0,  0,  6,   26},  {  0,  0,  6,   29},
+    {  0,  0,  6,   32},  {  0, 16,  6,65539},
+    {  0, 15,  6,32771},  {  0, 14,  6,16387},
+    {  0, 13,  6, 8195},  {  0, 12,  6, 4099},
+    {  0, 11,  6, 2051},  {  0, 10,  6, 1027},
+};   /* ML_defaultDTable */
+
+
+static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits)
+{
+    void* ptr = dt;
+    ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;
+    ZSTD_seqSymbol* const cell = dt + 1;
+
+    DTableH->tableLog = 0;
+    DTableH->fastMode = 0;
+
+    cell->nbBits = 0;
+    cell->nextState = 0;
+    assert(nbAddBits < 255);
+    cell->nbAdditionalBits = nbAddBits;
+    cell->baseValue = baseValue;
+}
+
+
+/* ZSTD_buildFSETable() :
+ * generate FSE decoding table for one symbol (ll, ml or off)
+ * cannot fail if input is valid =>
+ * all inputs are presumed validated at this stage */
+FORCE_INLINE_TEMPLATE
+void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
+            const short* normalizedCounter, unsigned maxSymbolValue,
+            const U32* baseValue, const U8* nbAdditionalBits,
+            unsigned tableLog, void* wksp, size_t wkspSize)
+{
+    ZSTD_seqSymbol* const tableDecode = dt+1;
+    U32 const maxSV1 = maxSymbolValue + 1;
+    U32 const tableSize = 1 << tableLog;
+
+    U16* symbolNext = (U16*)wksp;
+    BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1);
+    U32 highThreshold = tableSize - 1;
+
+
+    /* Sanity Checks */
+    assert(maxSymbolValue <= MaxSeq);
+    assert(tableLog <= MaxFSELog);
+    assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE);
+    (void)wkspSize;
+    /* Init, lay down lowprob symbols */
+    {   ZSTD_seqSymbol_header DTableH;
+        DTableH.tableLog = tableLog;
+        DTableH.fastMode = 1;
+        {   S16 const largeLimit= (S16)(1 << (tableLog-1));
+            U32 s;
+            for (s=0; s<maxSV1; s++) {
+                if (normalizedCounter[s]==-1) {
+                    tableDecode[highThreshold--].baseValue = s;
+                    symbolNext[s] = 1;
+                } else {
+                    if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
+                    assert(normalizedCounter[s]>=0);
+                    symbolNext[s] = (U16)normalizedCounter[s];
+        }   }   }
+        ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
+    }
+
+    /* Spread symbols */
+    assert(tableSize <= 512);
+    /* Specialized symbol spreading for the case when there are
+     * no low probability (-1 count) symbols. When compressing
+     * small blocks we avoid low probability symbols to hit this
+     * case, since header decoding speed matters more.
+     */
+    if (highThreshold == tableSize - 1) {
+        size_t const tableMask = tableSize-1;
+        size_t const step = FSE_TABLESTEP(tableSize);
+        /* First lay down the symbols in order.
+         * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
+         * misses since small blocks generally have small table logs, so nearly
+         * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
+         * our buffer to handle the over-write.
+         */
+        {
+            U64 const add = 0x0101010101010101ull;
+            size_t pos = 0;
+            U64 sv = 0;
+            U32 s;
+            for (s=0; s<maxSV1; ++s, sv += add) {
+                int i;
+                int const n = normalizedCounter[s];
+                MEM_write64(spread + pos, sv);
+                for (i = 8; i < n; i += 8) {
+                    MEM_write64(spread + pos + i, sv);
+                }
+                assert(n>=0);
+                pos += (size_t)n;
+            }
+        }
+        /* Now we spread those positions across the table.
+         * The benefit of doing it in two stages is that we avoid the
+         * variable size inner loop, which caused lots of branch misses.
+         * Now we can run through all the positions without any branch misses.
+         * We unroll the loop twice, since that is what empirically worked best.
+         */
+        {
+            size_t position = 0;
+            size_t s;
+            size_t const unroll = 2;
+            assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+            for (s = 0; s < (size_t)tableSize; s += unroll) {
+                size_t u;
+                for (u = 0; u < unroll; ++u) {
+                    size_t const uPosition = (position + (u * step)) & tableMask;
+                    tableDecode[uPosition].baseValue = spread[s + u];
+                }
+                position = (position + (unroll * step)) & tableMask;
+            }
+            assert(position == 0);
+        }
+    } else {
+        U32 const tableMask = tableSize-1;
+        U32 const step = FSE_TABLESTEP(tableSize);
+        U32 s, position = 0;
+        for (s=0; s<maxSV1; s++) {
+            int i;
+            int const n = normalizedCounter[s];
+            for (i=0; i<n; i++) {
+                tableDecode[position].baseValue = s;
+                position = (position + step) & tableMask;
+                while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask;   /* lowprob area */
+        }   }
+        assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
+    }
+
+    /* Build Decoding table */
+    {
+        U32 u;
+        for (u=0; u<tableSize; u++) {
+            U32 const symbol = tableDecode[u].baseValue;
+            U32 const nextState = symbolNext[symbol]++;
+            tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
+            tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
+            assert(nbAdditionalBits[symbol] < 255);
+            tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
+            tableDecode[u].baseValue = baseValue[symbol];
+        }
+    }
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt,
+            const short* normalizedCounter, unsigned maxSymbolValue,
+            const U32* baseValue, const U8* nbAdditionalBits,
+            unsigned tableLog, void* wksp, size_t wkspSize)
+{
+    ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
+            baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt,
+            const short* normalizedCounter, unsigned maxSymbolValue,
+            const U32* baseValue, const U8* nbAdditionalBits,
+            unsigned tableLog, void* wksp, size_t wkspSize)
+{
+    ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
+            baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+#endif
+
+void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
+            const short* normalizedCounter, unsigned maxSymbolValue,
+            const U32* baseValue, const U8* nbAdditionalBits,
+            unsigned tableLog, void* wksp, size_t wkspSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+    if (bmi2) {
+        ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue,
+                baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+        return;
+    }
+#endif
+    (void)bmi2;
+    ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue,
+            baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+
+
+/*! ZSTD_buildSeqTable() :
+ * @return : nb bytes read from src,
+ *           or an error code if it fails */
+static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
+                                 symbolEncodingType_e type, unsigned max, U32 maxLog,
+                                 const void* src, size_t srcSize,
+                                 const U32* baseValue, const U8* nbAdditionalBits,
+                                 const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
+                                 int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize,
+                                 int bmi2)
+{
+    switch(type)
+    {
+    case set_rle :
+        RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");
+        RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, "");
+        {   U32 const symbol = *(const BYTE*)src;
+            U32 const baseline = baseValue[symbol];
+            U8 const nbBits = nbAdditionalBits[symbol];
+            ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);
+        }
+        *DTablePtr = DTableSpace;
+        return 1;
+    case set_basic :
+        *DTablePtr = defaultTable;
+        return 0;
+    case set_repeat:
+        RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");
+        /* prefetch FSE table if used */
+        if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
+            const void* const pStart = *DTablePtr;
+            size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
+            PREFETCH_AREA(pStart, pSize);
+        }
+        return 0;
+    case set_compressed :
+        {   unsigned tableLog;
+            S16 norm[MaxSeq+1];
+            size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
+            RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");
+            RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");
+            ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2);
+            *DTablePtr = DTableSpace;
+            return headerSize;
+        }
+    default :
+        assert(0);
+        RETURN_ERROR(GENERIC, "impossible");
+    }
+}
+
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+                             const void* src, size_t srcSize)
+{
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* ip = istart;
+    int nbSeq;
+    DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
+
+    /* check */
+    RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, "");
+
+    /* SeqHead */
+    nbSeq = *ip++;
+    if (nbSeq > 0x7F) {
+        if (nbSeq == 0xFF) {
+            RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, "");
+            nbSeq = MEM_readLE16(ip) + LONGNBSEQ;
+            ip+=2;
+        } else {
+            RETURN_ERROR_IF(ip >= iend, srcSize_wrong, "");
+            nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+        }
+    }
+    *nbSeqPtr = nbSeq;
+
+    if (nbSeq == 0) {
+        /* No sequence : section ends immediately */
+        RETURN_ERROR_IF(ip != iend, corruption_detected,
+            "extraneous data present in the Sequences section");
+        return (size_t)(ip - istart);
+    }
+
+    /* FSE table descriptors */
+    RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
+    RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */
+    {   symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
+        symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
+        symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
+        ip++;
+
+        /* Build DTables */
+        {   size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
+                                                      LLtype, MaxLL, LLFSELog,
+                                                      ip, iend-ip,
+                                                      LL_base, LL_bits,
+                                                      LL_defaultDTable, dctx->fseEntropy,
+                                                      dctx->ddictIsCold, nbSeq,
+                                                      dctx->workspace, sizeof(dctx->workspace),
+                                                      ZSTD_DCtx_get_bmi2(dctx));
+            RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+            ip += llhSize;
+        }
+
+        {   size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
+                                                      OFtype, MaxOff, OffFSELog,
+                                                      ip, iend-ip,
+                                                      OF_base, OF_bits,
+                                                      OF_defaultDTable, dctx->fseEntropy,
+                                                      dctx->ddictIsCold, nbSeq,
+                                                      dctx->workspace, sizeof(dctx->workspace),
+                                                      ZSTD_DCtx_get_bmi2(dctx));
+            RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+            ip += ofhSize;
+        }
+
+        {   size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
+                                                      MLtype, MaxML, MLFSELog,
+                                                      ip, iend-ip,
+                                                      ML_base, ML_bits,
+                                                      ML_defaultDTable, dctx->fseEntropy,
+                                                      dctx->ddictIsCold, nbSeq,
+                                                      dctx->workspace, sizeof(dctx->workspace),
+                                                      ZSTD_DCtx_get_bmi2(dctx));
+            RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+            ip += mlhSize;
+        }
+    }
+
+    return ip-istart;
+}
+
+
+typedef struct {
+    size_t litLength;
+    size_t matchLength;
+    size_t offset;
+} seq_t;
+
+typedef struct {
+    size_t state;
+    const ZSTD_seqSymbol* table;
+} ZSTD_fseState;
+
+typedef struct {
+    BIT_DStream_t DStream;
+    ZSTD_fseState stateLL;
+    ZSTD_fseState stateOffb;
+    ZSTD_fseState stateML;
+    size_t prevOffset[ZSTD_REP_NUM];
+} seqState_t;
+
+/*! ZSTD_overlapCopy8() :
+ *  Copies 8 bytes from ip to op and updates op and ip where ip <= op.
+ *  If the offset is < 8 then the offset is spread to at least 8 bytes.
+ *
+ *  Precondition: *ip <= *op
+ *  Postcondition: *op - *op >= 8
+ */
+HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
+    assert(*ip <= *op);
+    if (offset < 8) {
+        /* close range match, overlap */
+        static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
+        static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* subtracted */
+        int const sub2 = dec64table[offset];
+        (*op)[0] = (*ip)[0];
+        (*op)[1] = (*ip)[1];
+        (*op)[2] = (*ip)[2];
+        (*op)[3] = (*ip)[3];
+        *ip += dec32table[offset];
+        ZSTD_copy4(*op+4, *ip);
+        *ip -= sub2;
+    } else {
+        ZSTD_copy8(*op, *ip);
+    }
+    *ip += 8;
+    *op += 8;
+    assert(*op - *ip >= 8);
+}
+
+/*! ZSTD_safecopy() :
+ *  Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
+ *  and write up to 16 bytes past oend_w (op >= oend_w is allowed).
+ *  This function is only called in the uncommon case where the sequence is near the end of the block. It
+ *  should be fast for a single long sequence, but can be slow for several short sequences.
+ *
+ *  @param ovtype controls the overlap detection
+ *         - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
+ *         - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
+ *           The src buffer must be before the dst buffer.
+ */
+static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
+    ptrdiff_t const diff = op - ip;
+    BYTE* const oend = op + length;
+
+    assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
+           (ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
+
+    if (length < 8) {
+        /* Handle short lengths. */
+        while (op < oend) *op++ = *ip++;
+        return;
+    }
+    if (ovtype == ZSTD_overlap_src_before_dst) {
+        /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
+        assert(length >= 8);
+        ZSTD_overlapCopy8(&op, &ip, diff);
+        length -= 8;
+        assert(op - ip >= 8);
+        assert(op <= oend);
+    }
+
+    if (oend <= oend_w) {
+        /* No risk of overwrite. */
+        ZSTD_wildcopy(op, ip, length, ovtype);
+        return;
+    }
+    if (op <= oend_w) {
+        /* Wildcopy until we get close to the end. */
+        assert(oend > oend_w);
+        ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
+        ip += oend_w - op;
+        op += oend_w - op;
+    }
+    /* Handle the leftovers. */
+    while (op < oend) *op++ = *ip++;
+}
+
+/* ZSTD_safecopyDstBeforeSrc():
+ * This version allows overlap with dst before src, or handles the non-overlap case with dst after src
+ * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
+static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) {
+    ptrdiff_t const diff = op - ip;
+    BYTE* const oend = op + length;
+
+    if (length < 8 || diff > -8) {
+        /* Handle short lengths, close overlaps, and dst not before src. */
+        while (op < oend) *op++ = *ip++;
+        return;
+    }
+
+    if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
+        ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
+        ip += oend - WILDCOPY_OVERLENGTH - op;
+        op += oend - WILDCOPY_OVERLENGTH - op;
+    }
+
+    /* Handle the leftovers. */
+    while (op < oend) *op++ = *ip++;
+}
+
+/* ZSTD_execSequenceEnd():
+ * This version handles cases that are near the end of the output buffer. It requires
+ * more careful checks to make sure there is no overflow. By separating out these hard
+ * and unlikely cases, we can speed up the common cases.
+ *
+ * NOTE: This function needs to be fast for a single long sequence, but doesn't need
+ * to be optimized for many small sequences, since those fall into ZSTD_execSequence().
+ */
+FORCE_NOINLINE
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+size_t ZSTD_execSequenceEnd(BYTE* op,
+    BYTE* const oend, seq_t sequence,
+    const BYTE** litPtr, const BYTE* const litLimit,
+    const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+    BYTE* const oLitEnd = op + sequence.litLength;
+    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+    const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+    const BYTE* match = oLitEnd - sequence.offset;
+    BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
+
+    /* bounds checks : careful of address space overflow in 32-bit mode */
+    RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
+    RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
+    assert(op < op + sequenceLength);
+    assert(oLitEnd < op + sequenceLength);
+
+    /* copy literals */
+    ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
+    op = oLitEnd;
+    *litPtr = iLitEnd;
+
+    /* copy Match */
+    if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+        /* offset beyond prefix */
+        RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
+        match = dictEnd - (prefixStart - match);
+        if (match + sequence.matchLength <= dictEnd) {
+            ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+            return sequenceLength;
+        }
+        /* span extDict & currentPrefixSegment */
+        {   size_t const length1 = dictEnd - match;
+        ZSTD_memmove(oLitEnd, match, length1);
+        op = oLitEnd + length1;
+        sequence.matchLength -= length1;
+        match = prefixStart;
+        }
+    }
+    ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
+    return sequenceLength;
+}
+
+/* ZSTD_execSequenceEndSplitLitBuffer():
+ * This version is intended to be used during instances where the litBuffer is still split.  It is kept separate to avoid performance impact for the good case.
+ */
+FORCE_NOINLINE
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
+    BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
+    const BYTE** litPtr, const BYTE* const litLimit,
+    const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+    BYTE* const oLitEnd = op + sequence.litLength;
+    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+    const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+    const BYTE* match = oLitEnd - sequence.offset;
+
+
+    /* bounds checks : careful of address space overflow in 32-bit mode */
+    RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
+    RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
+    assert(op < op + sequenceLength);
+    assert(oLitEnd < op + sequenceLength);
+
+    /* copy literals */
+    RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
+    ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
+    op = oLitEnd;
+    *litPtr = iLitEnd;
+
+    /* copy Match */
+    if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+        /* offset beyond prefix */
+        RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
+        match = dictEnd - (prefixStart - match);
+        if (match + sequence.matchLength <= dictEnd) {
+            ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+            return sequenceLength;
+        }
+        /* span extDict & currentPrefixSegment */
+        {   size_t const length1 = dictEnd - match;
+        ZSTD_memmove(oLitEnd, match, length1);
+        op = oLitEnd + length1;
+        sequence.matchLength -= length1;
+        match = prefixStart;
+        }
+    }
+    ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
+    return sequenceLength;
+}
+
+HINT_INLINE
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+size_t ZSTD_execSequence(BYTE* op,
+    BYTE* const oend, seq_t sequence,
+    const BYTE** litPtr, const BYTE* const litLimit,
+    const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+    BYTE* const oLitEnd = op + sequence.litLength;
+    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+    BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
+    BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;   /* risk : address space underflow on oend=NULL */
+    const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+    const BYTE* match = oLitEnd - sequence.offset;
+
+    assert(op != NULL /* Precondition */);
+    assert(oend_w < oend /* No underflow */);
+
+#if defined(__aarch64__)
+    /* prefetch sequence starting from match that will be used for copy later */
+    PREFETCH_L1(match);
+#endif
+    /* Handle edge cases in a slow path:
+     *   - Read beyond end of literals
+     *   - Match end is within WILDCOPY_OVERLIMIT of oend
+     *   - 32-bit mode and the match length overflows
+     */
+    if (UNLIKELY(
+        iLitEnd > litLimit ||
+        oMatchEnd > oend_w ||
+        (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
+        return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
+
+    /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
+    assert(op <= oLitEnd /* No overflow */);
+    assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
+    assert(oMatchEnd <= oend /* No underflow */);
+    assert(iLitEnd <= litLimit /* Literal length is in bounds */);
+    assert(oLitEnd <= oend_w /* Can wildcopy literals */);
+    assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
+
+    /* Copy Literals:
+     * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
+     * We likely don't need the full 32-byte wildcopy.
+     */
+    assert(WILDCOPY_OVERLENGTH >= 16);
+    ZSTD_copy16(op, (*litPtr));
+    if (UNLIKELY(sequence.litLength > 16)) {
+        ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
+    }
+    op = oLitEnd;
+    *litPtr = iLitEnd;   /* update for next sequence */
+
+    /* Copy Match */
+    if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+        /* offset beyond prefix -> go into extDict */
+        RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
+        match = dictEnd + (match - prefixStart);
+        if (match + sequence.matchLength <= dictEnd) {
+            ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+            return sequenceLength;
+        }
+        /* span extDict & currentPrefixSegment */
+        {   size_t const length1 = dictEnd - match;
+        ZSTD_memmove(oLitEnd, match, length1);
+        op = oLitEnd + length1;
+        sequence.matchLength -= length1;
+        match = prefixStart;
+        }
+    }
+    /* Match within prefix of 1 or more bytes */
+    assert(op <= oMatchEnd);
+    assert(oMatchEnd <= oend_w);
+    assert(match >= prefixStart);
+    assert(sequence.matchLength >= 1);
+
+    /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
+     * without overlap checking.
+     */
+    if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
+        /* We bet on a full wildcopy for matches, since we expect matches to be
+         * longer than literals (in general). In silesia, ~10% of matches are longer
+         * than 16 bytes.
+         */
+        ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
+        return sequenceLength;
+    }
+    assert(sequence.offset < WILDCOPY_VECLEN);
+
+    /* Copy 8 bytes and spread the offset to be >= 8. */
+    ZSTD_overlapCopy8(&op, &match, sequence.offset);
+
+    /* If the match length is > 8 bytes, then continue with the wildcopy. */
+    if (sequence.matchLength > 8) {
+        assert(op < oMatchEnd);
+        ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
+    }
+    return sequenceLength;
+}
+
+HINT_INLINE
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
+    BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
+    const BYTE** litPtr, const BYTE* const litLimit,
+    const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+    BYTE* const oLitEnd = op + sequence.litLength;
+    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+    BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
+    const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+    const BYTE* match = oLitEnd - sequence.offset;
+
+    assert(op != NULL /* Precondition */);
+    assert(oend_w < oend /* No underflow */);
+    /* Handle edge cases in a slow path:
+     *   - Read beyond end of literals
+     *   - Match end is within WILDCOPY_OVERLIMIT of oend
+     *   - 32-bit mode and the match length overflows
+     */
+    if (UNLIKELY(
+            iLitEnd > litLimit ||
+            oMatchEnd > oend_w ||
+            (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
+        return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
+
+    /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
+    assert(op <= oLitEnd /* No overflow */);
+    assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
+    assert(oMatchEnd <= oend /* No underflow */);
+    assert(iLitEnd <= litLimit /* Literal length is in bounds */);
+    assert(oLitEnd <= oend_w /* Can wildcopy literals */);
+    assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
+
+    /* Copy Literals:
+     * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
+     * We likely don't need the full 32-byte wildcopy.
+     */
+    assert(WILDCOPY_OVERLENGTH >= 16);
+    ZSTD_copy16(op, (*litPtr));
+    if (UNLIKELY(sequence.litLength > 16)) {
+        ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
+    }
+    op = oLitEnd;
+    *litPtr = iLitEnd;   /* update for next sequence */
+
+    /* Copy Match */
+    if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+        /* offset beyond prefix -> go into extDict */
+        RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
+        match = dictEnd + (match - prefixStart);
+        if (match + sequence.matchLength <= dictEnd) {
+            ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+            return sequenceLength;
+        }
+        /* span extDict & currentPrefixSegment */
+        {   size_t const length1 = dictEnd - match;
+            ZSTD_memmove(oLitEnd, match, length1);
+            op = oLitEnd + length1;
+            sequence.matchLength -= length1;
+            match = prefixStart;
+    }   }
+    /* Match within prefix of 1 or more bytes */
+    assert(op <= oMatchEnd);
+    assert(oMatchEnd <= oend_w);
+    assert(match >= prefixStart);
+    assert(sequence.matchLength >= 1);
+
+    /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
+     * without overlap checking.
+     */
+    if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
+        /* We bet on a full wildcopy for matches, since we expect matches to be
+         * longer than literals (in general). In silesia, ~10% of matches are longer
+         * than 16 bytes.
+         */
+        ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
+        return sequenceLength;
+    }
+    assert(sequence.offset < WILDCOPY_VECLEN);
+
+    /* Copy 8 bytes and spread the offset to be >= 8. */
+    ZSTD_overlapCopy8(&op, &match, sequence.offset);
+
+    /* If the match length is > 8 bytes, then continue with the wildcopy. */
+    if (sequence.matchLength > 8) {
+        assert(op < oMatchEnd);
+        ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
+    }
+    return sequenceLength;
+}
+
+
+static void
+ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
+{
+    const void* ptr = dt;
+    const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;
+    DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
+    DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",
+                (U32)DStatePtr->state, DTableH->tableLog);
+    BIT_reloadDStream(bitD);
+    DStatePtr->table = dt + 1;
+}
+
+FORCE_INLINE_TEMPLATE void
+ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
+{
+    size_t const lowBits = BIT_readBits(bitD, nbBits);
+    DStatePtr->state = nextState + lowBits;
+}
+
+/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
+ * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
+ * bits before reloading. This value is the maximum number of bytes we read
+ * after reloading when we are decoding long offsets.
+ */
+#define LONG_OFFSETS_MAX_EXTRA_BITS_32                       \
+    (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32       \
+        ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32  \
+        : 0)
+
+typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;
+
+/**
+ * ZSTD_decodeSequence():
+ * @p longOffsets : tells the decoder to reload more bit while decoding large offsets
+ *                  only used in 32-bit mode
+ * @return : Sequence (litL + matchL + offset)
+ */
+FORCE_INLINE_TEMPLATE seq_t
+ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq)
+{
+    seq_t seq;
+    /*
+     * ZSTD_seqSymbol is a 64 bits wide structure.
+     * It can be loaded in one operation
+     * and its fields extracted by simply shifting or bit-extracting on aarch64.
+     * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh
+     * operations that cause performance drop. This can be avoided by using this
+     * ZSTD_memcpy hack.
+     */
+#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__))
+    ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS;
+    ZSTD_seqSymbol* const llDInfo = &llDInfoS;
+    ZSTD_seqSymbol* const mlDInfo = &mlDInfoS;
+    ZSTD_seqSymbol* const ofDInfo = &ofDInfoS;
+    ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol));
+    ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol));
+    ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol));
+#else
+    const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
+    const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
+    const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
+#endif
+    seq.matchLength = mlDInfo->baseValue;
+    seq.litLength = llDInfo->baseValue;
+    {   U32 const ofBase = ofDInfo->baseValue;
+        BYTE const llBits = llDInfo->nbAdditionalBits;
+        BYTE const mlBits = mlDInfo->nbAdditionalBits;
+        BYTE const ofBits = ofDInfo->nbAdditionalBits;
+        BYTE const totalBits = llBits+mlBits+ofBits;
+
+        U16 const llNext = llDInfo->nextState;
+        U16 const mlNext = mlDInfo->nextState;
+        U16 const ofNext = ofDInfo->nextState;
+        U32 const llnbBits = llDInfo->nbBits;
+        U32 const mlnbBits = mlDInfo->nbBits;
+        U32 const ofnbBits = ofDInfo->nbBits;
+
+        assert(llBits <= MaxLLBits);
+        assert(mlBits <= MaxMLBits);
+        assert(ofBits <= MaxOff);
+        /*
+         * As gcc has better branch and block analyzers, sometimes it is only
+         * valuable to mark likeliness for clang, it gives around 3-4% of
+         * performance.
+         */
+
+        /* sequence */
+        {   size_t offset;
+            if (ofBits > 1) {
+                ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
+                ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
+                if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
+                    /* Always read extra bits, this keeps the logic simple,
+                     * avoids branches, and avoids accidentally reading 0 bits.
+                     */
+                    U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
+                    offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
+                    BIT_reloadDStream(&seqState->DStream);
+                    offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+                } else {
+                    offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
+                    if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+                }
+                seqState->prevOffset[2] = seqState->prevOffset[1];
+                seqState->prevOffset[1] = seqState->prevOffset[0];
+                seqState->prevOffset[0] = offset;
+            } else {
+                U32 const ll0 = (llDInfo->baseValue == 0);
+                if (LIKELY((ofBits == 0))) {
+                    offset = seqState->prevOffset[ll0];
+                    seqState->prevOffset[1] = seqState->prevOffset[!ll0];
+                    seqState->prevOffset[0] = offset;
+                } else {
+                    offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
+                    {   size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+                        temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */
+                        if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
+                        seqState->prevOffset[1] = seqState->prevOffset[0];
+                        seqState->prevOffset[0] = offset = temp;
+            }   }   }
+            seq.offset = offset;
+        }
+
+        if (mlBits > 0)
+            seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
+
+        if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
+            BIT_reloadDStream(&seqState->DStream);
+        if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
+            BIT_reloadDStream(&seqState->DStream);
+        /* Ensure there are enough bits to read the rest of data in 64-bit mode. */
+        ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
+
+        if (llBits > 0)
+            seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
+
+        if (MEM_32bits())
+            BIT_reloadDStream(&seqState->DStream);
+
+        DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
+                    (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
+
+        if (!isLastSeq) {
+            /* don't update FSE state for last Sequence */
+            ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits);    /* <=  9 bits */
+            ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits);    /* <=  9 bits */
+            if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);    /* <= 18 bits */
+            ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits);  /* <=  8 bits */
+            BIT_reloadDStream(&seqState->DStream);
+        }
+    }
+
+    return seq;
+}
+
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+#if DEBUGLEVEL >= 1
+static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd)
+{
+    size_t const windowSize = dctx->fParams.windowSize;
+    /* No dictionary used. */
+    if (dctx->dictContentEndForFuzzing == NULL) return 0;
+    /* Dictionary is our prefix. */
+    if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
+    /* Dictionary is not our ext-dict. */
+    if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
+    /* Dictionary is not within our window size. */
+    if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
+    /* Dictionary is active. */
+    return 1;
+}
+#endif
+
+static void ZSTD_assertValidSequence(
+        ZSTD_DCtx const* dctx,
+        BYTE const* op, BYTE const* oend,
+        seq_t const seq,
+        BYTE const* prefixStart, BYTE const* virtualStart)
+{
+#if DEBUGLEVEL >= 1
+    if (dctx->isFrameDecompression) {
+        size_t const windowSize = dctx->fParams.windowSize;
+        size_t const sequenceSize = seq.litLength + seq.matchLength;
+        BYTE const* const oLitEnd = op + seq.litLength;
+        DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",
+                (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
+        assert(op <= oend);
+        assert((size_t)(oend - op) >= sequenceSize);
+        assert(sequenceSize <= ZSTD_blockSizeMax(dctx));
+        if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {
+            size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing);
+            /* Offset must be within the dictionary. */
+            assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
+            assert(seq.offset <= windowSize + dictSize);
+        } else {
+            /* Offset must be within our window. */
+            assert(seq.offset <= windowSize);
+        }
+    }
+#else
+    (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;
+#endif
+}
+#endif
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+
+
+FORCE_INLINE_TEMPLATE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
+                               void* dst, size_t maxDstSize,
+                         const void* seqStart, size_t seqSize, int nbSeq,
+                         const ZSTD_longOffset_e isLongOffset)
+{
+    const BYTE* ip = (const BYTE*)seqStart;
+    const BYTE* const iend = ip + seqSize;
+    BYTE* const ostart = (BYTE*)dst;
+    BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
+    BYTE* op = ostart;
+    const BYTE* litPtr = dctx->litPtr;
+    const BYTE* litBufferEnd = dctx->litBufferEnd;
+    const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
+    const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
+    const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+    DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq);
+
+    /* Literals are split between internal buffer & output buffer */
+    if (nbSeq) {
+        seqState_t seqState;
+        dctx->fseEntropy = 1;
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+        RETURN_ERROR_IF(
+            ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+            corruption_detected, "");
+        ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+        ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+        ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+        assert(dst != NULL);
+
+        ZSTD_STATIC_ASSERT(
+                BIT_DStream_unfinished < BIT_DStream_completed &&
+                BIT_DStream_endOfBuffer < BIT_DStream_completed &&
+                BIT_DStream_completed < BIT_DStream_overflow);
+
+        /* decompress without overrunning litPtr begins */
+        {   seq_t sequence = {0,0,0};  /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */
+            /* Align the decompression loop to 32 + 16 bytes.
+                *
+                * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
+                * speed swings based on the alignment of the decompression loop. This
+                * performance swing is caused by parts of the decompression loop falling
+                * out of the DSB. The entire decompression loop should fit in the DSB,
+                * when it can't we get much worse performance. You can measure if you've
+                * hit the good case or the bad case with this perf command for some
+                * compressed file test.zst:
+                *
+                *   perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
+                *             -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
+                *
+                * If you see most cycles served out of the MITE you've hit the bad case.
+                * If you see most cycles served out of the DSB you've hit the good case.
+                * If it is pretty even then you may be in an okay case.
+                *
+                * This issue has been reproduced on the following CPUs:
+                *   - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
+                *               Use Instruments->Counters to get DSB/MITE cycles.
+                *               I never got performance swings, but I was able to
+                *               go from the good case of mostly DSB to half of the
+                *               cycles served from MITE.
+                *   - Coffeelake: Intel i9-9900k
+                *   - Coffeelake: Intel i7-9700k
+                *
+                * I haven't been able to reproduce the instability or DSB misses on any
+                * of the following CPUS:
+                *   - Haswell
+                *   - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
+                *   - Skylake
+                *
+                * Alignment is done for each of the three major decompression loops:
+                *   - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
+                *   - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
+                *   - ZSTD_decompressSequences_body
+                * Alignment choices are made to minimize large swings on bad cases and influence on performance
+                * from changes external to this code, rather than to overoptimize on the current commit.
+                *
+                * If you are seeing performance stability this script can help test.
+                * It tests on 4 commits in zstd where I saw performance change.
+                *
+                *   https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
+                */
+#if defined(__GNUC__) && defined(__x86_64__)
+            __asm__(".p2align 6");
+#  if __GNUC__ >= 7
+	    /* good for gcc-7, gcc-9, and gcc-11 */
+            __asm__("nop");
+            __asm__(".p2align 5");
+            __asm__("nop");
+            __asm__(".p2align 4");
+#    if __GNUC__ == 8 || __GNUC__ == 10
+	    /* good for gcc-8 and gcc-10 */
+            __asm__("nop");
+            __asm__(".p2align 3");
+#    endif
+#  endif
+#endif
+
+            /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
+            for ( ; nbSeq; nbSeq--) {
+                sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
+                if (litPtr + sequence.litLength > dctx->litBufferEnd) break;
+                {   size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                    assert(!ZSTD_isError(oneSeqSize));
+                    ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+                    if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+                        return oneSeqSize;
+                    DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+                    op += oneSeqSize;
+            }   }
+            DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)");
+
+            /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
+            if (nbSeq > 0) {
+                const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+                DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength);
+                if (leftoverLit) {
+                    RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+                    ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+                    sequence.litLength -= leftoverLit;
+                    op += leftoverLit;
+                }
+                litPtr = dctx->litExtraBuffer;
+                litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+                dctx->litBufferLocation = ZSTD_not_in_dst;
+                {   size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                    assert(!ZSTD_isError(oneSeqSize));
+                    ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+                    if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+                        return oneSeqSize;
+                    DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+                    op += oneSeqSize;
+                }
+                nbSeq--;
+            }
+        }
+
+        if (nbSeq > 0) {
+            /* there is remaining lit from extra buffer */
+
+#if defined(__GNUC__) && defined(__x86_64__)
+            __asm__(".p2align 6");
+            __asm__("nop");
+#  if __GNUC__ != 7
+            /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
+            __asm__(".p2align 4");
+            __asm__("nop");
+            __asm__(".p2align 3");
+#  elif __GNUC__ >= 11
+            __asm__(".p2align 3");
+#  else
+            __asm__(".p2align 5");
+            __asm__("nop");
+            __asm__(".p2align 3");
+#  endif
+#endif
+
+            for ( ; nbSeq ; nbSeq--) {
+                seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
+                size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                assert(!ZSTD_isError(oneSeqSize));
+                ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+                if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+                    return oneSeqSize;
+                DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+                op += oneSeqSize;
+            }
+        }
+
+        /* check if reached exact end */
+        DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
+        RETURN_ERROR_IF(nbSeq, corruption_detected, "");
+        DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed);
+        RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
+        /* save reps for next block */
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+    }
+
+    /* last literal segment */
+    if (dctx->litBufferLocation == ZSTD_split) {
+        /* split hasn't been reached yet, first get dst then copy litExtraBuffer */
+        size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
+        DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize);
+        RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
+        if (op != NULL) {
+            ZSTD_memmove(op, litPtr, lastLLSize);
+            op += lastLLSize;
+        }
+        litPtr = dctx->litExtraBuffer;
+        litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+        dctx->litBufferLocation = ZSTD_not_in_dst;
+    }
+    /* copy last literals from internal buffer */
+    {   size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
+        DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize);
+        RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+        if (op != NULL) {
+            ZSTD_memcpy(op, litPtr, lastLLSize);
+            op += lastLLSize;
+    }   }
+
+    DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
+    return (size_t)(op - ostart);
+}
+
+FORCE_INLINE_TEMPLATE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
+    void* dst, size_t maxDstSize,
+    const void* seqStart, size_t seqSize, int nbSeq,
+    const ZSTD_longOffset_e isLongOffset)
+{
+    const BYTE* ip = (const BYTE*)seqStart;
+    const BYTE* const iend = ip + seqSize;
+    BYTE* const ostart = (BYTE*)dst;
+    BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer;
+    BYTE* op = ostart;
+    const BYTE* litPtr = dctx->litPtr;
+    const BYTE* const litEnd = litPtr + dctx->litSize;
+    const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
+    const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
+    const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
+    DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
+
+    /* Regen sequences */
+    if (nbSeq) {
+        seqState_t seqState;
+        dctx->fseEntropy = 1;
+        { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+        RETURN_ERROR_IF(
+            ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
+            corruption_detected, "");
+        ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+        ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+        ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+        assert(dst != NULL);
+
+#if defined(__GNUC__) && defined(__x86_64__)
+            __asm__(".p2align 6");
+            __asm__("nop");
+#  if __GNUC__ >= 7
+            __asm__(".p2align 5");
+            __asm__("nop");
+            __asm__(".p2align 3");
+#  else
+            __asm__(".p2align 4");
+            __asm__("nop");
+            __asm__(".p2align 3");
+#  endif
+#endif
+
+        for ( ; nbSeq ; nbSeq--) {
+            seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
+            size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+            assert(!ZSTD_isError(oneSeqSize));
+            ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+            if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+                return oneSeqSize;
+            DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+            op += oneSeqSize;
+        }
+
+        /* check if reached exact end */
+        assert(nbSeq == 0);
+        RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
+        /* save reps for next block */
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+    }
+
+    /* last literal segment */
+    {   size_t const lastLLSize = (size_t)(litEnd - litPtr);
+        DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize);
+        RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+        if (op != NULL) {
+            ZSTD_memcpy(op, litPtr, lastLLSize);
+            op += lastLLSize;
+    }   }
+
+    DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
+    return (size_t)(op - ostart);
+}
+
+static size_t
+ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
+                                 void* dst, size_t maxDstSize,
+                           const void* seqStart, size_t seqSize, int nbSeq,
+                           const ZSTD_longOffset_e isLongOffset)
+{
+    return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+
+static size_t
+ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
+                                               void* dst, size_t maxDstSize,
+                                         const void* seqStart, size_t seqSize, int nbSeq,
+                                         const ZSTD_longOffset_e isLongOffset)
+{
+    return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+
+FORCE_INLINE_TEMPLATE
+
+size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
+                   const BYTE* const prefixStart, const BYTE* const dictEnd)
+{
+    prefetchPos += sequence.litLength;
+    {   const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
+        /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
+         * No consequence though : memory address is only used for prefetching, not for dereferencing */
+        const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset);
+        PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE);   /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
+    }
+    return prefetchPos + sequence.matchLength;
+}
+
+/* This decoding function employs prefetching
+ * to reduce latency impact of cache misses.
+ * It's generally employed when block contains a significant portion of long-distance matches
+ * or when coupled with a "cold" dictionary */
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_decompressSequencesLong_body(
+                               ZSTD_DCtx* dctx,
+                               void* dst, size_t maxDstSize,
+                         const void* seqStart, size_t seqSize, int nbSeq,
+                         const ZSTD_longOffset_e isLongOffset)
+{
+    const BYTE* ip = (const BYTE*)seqStart;
+    const BYTE* const iend = ip + seqSize;
+    BYTE* const ostart = (BYTE*)dst;
+    BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
+    BYTE* op = ostart;
+    const BYTE* litPtr = dctx->litPtr;
+    const BYTE* litBufferEnd = dctx->litBufferEnd;
+    const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
+    const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
+    const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+
+    /* Regen sequences */
+    if (nbSeq) {
+#define STORED_SEQS 8
+#define STORED_SEQS_MASK (STORED_SEQS-1)
+#define ADVANCED_SEQS STORED_SEQS
+        seq_t sequences[STORED_SEQS];
+        int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
+        seqState_t seqState;
+        int seqNb;
+        size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
+
+        dctx->fseEntropy = 1;
+        { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+        assert(dst != NULL);
+        assert(iend >= ip);
+        RETURN_ERROR_IF(
+            ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+            corruption_detected, "");
+        ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+        ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+        ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+
+        /* prepare in advance */
+        for (seqNb=0; seqNb<seqAdvance; seqNb++) {
+            seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1);
+            prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+            sequences[seqNb] = sequence;
+        }
+
+        /* decompress without stomping litBuffer */
+        for (; seqNb < nbSeq; seqNb++) {
+            seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1);
+
+            if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) {
+                /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
+                const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+                if (leftoverLit)
+                {
+                    RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+                    ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+                    sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
+                    op += leftoverLit;
+                }
+                litPtr = dctx->litExtraBuffer;
+                litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+                dctx->litBufferLocation = ZSTD_not_in_dst;
+                {   size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                    assert(!ZSTD_isError(oneSeqSize));
+                    ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+                    if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+
+                    prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+                    sequences[seqNb & STORED_SEQS_MASK] = sequence;
+                    op += oneSeqSize;
+            }   }
+            else
+            {
+                /* lit buffer is either wholly contained in first or second split, or not split at all*/
+                size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
+                    ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
+                    ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                assert(!ZSTD_isError(oneSeqSize));
+                ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+                if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+
+                prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+                sequences[seqNb & STORED_SEQS_MASK] = sequence;
+                op += oneSeqSize;
+            }
+        }
+        RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
+
+        /* finish queue */
+        seqNb -= seqAdvance;
+        for ( ; seqNb<nbSeq ; seqNb++) {
+            seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
+            if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) {
+                const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+                if (leftoverLit) {
+                    RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+                    ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+                    sequence->litLength -= leftoverLit;
+                    op += leftoverLit;
+                }
+                litPtr = dctx->litExtraBuffer;
+                litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+                dctx->litBufferLocation = ZSTD_not_in_dst;
+                {   size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                    assert(!ZSTD_isError(oneSeqSize));
+                    ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+                    if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+                    op += oneSeqSize;
+                }
+            }
+            else
+            {
+                size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
+                    ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
+                    ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+                assert(!ZSTD_isError(oneSeqSize));
+                ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+                if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+                op += oneSeqSize;
+            }
+        }
+
+        /* save reps for next block */
+        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+    }
+
+    /* last literal segment */
+    if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */
+        size_t const lastLLSize = litBufferEnd - litPtr;
+        RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
+        if (op != NULL) {
+            ZSTD_memmove(op, litPtr, lastLLSize);
+            op += lastLLSize;
+        }
+        litPtr = dctx->litExtraBuffer;
+        litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+    }
+    {   size_t const lastLLSize = litBufferEnd - litPtr;
+        RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+        if (op != NULL) {
+            ZSTD_memmove(op, litPtr, lastLLSize);
+            op += lastLLSize;
+        }
+    }
+
+    return (size_t)(op - ostart);
+}
+
+static size_t
+ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx,
+                                 void* dst, size_t maxDstSize,
+                           const void* seqStart, size_t seqSize, int nbSeq,
+                           const ZSTD_longOffset_e isLongOffset)
+{
+    return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+
+
+#if DYNAMIC_BMI2
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+static BMI2_TARGET_ATTRIBUTE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
+                                 void* dst, size_t maxDstSize,
+                           const void* seqStart, size_t seqSize, int nbSeq,
+                           const ZSTD_longOffset_e isLongOffset)
+{
+    return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+static BMI2_TARGET_ATTRIBUTE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
+                                 void* dst, size_t maxDstSize,
+                           const void* seqStart, size_t seqSize, int nbSeq,
+                           const ZSTD_longOffset_e isLongOffset)
+{
+    return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+static BMI2_TARGET_ATTRIBUTE size_t
+ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
+                                 void* dst, size_t maxDstSize,
+                           const void* seqStart, size_t seqSize, int nbSeq,
+                           const ZSTD_longOffset_e isLongOffset)
+{
+    return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+#endif /* DYNAMIC_BMI2 */
+
+typedef size_t (*ZSTD_decompressSequences_t)(
+                            ZSTD_DCtx* dctx,
+                            void* dst, size_t maxDstSize,
+                            const void* seqStart, size_t seqSize, int nbSeq,
+                            const ZSTD_longOffset_e isLongOffset);
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+static size_t
+ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
+                   const void* seqStart, size_t seqSize, int nbSeq,
+                   const ZSTD_longOffset_e isLongOffset)
+{
+    DEBUGLOG(5, "ZSTD_decompressSequences");
+#if DYNAMIC_BMI2
+    if (ZSTD_DCtx_get_bmi2(dctx)) {
+        return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+    }
+#endif
+    return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+static size_t
+ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
+                                 const void* seqStart, size_t seqSize, int nbSeq,
+                                 const ZSTD_longOffset_e isLongOffset)
+{
+    DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
+#if DYNAMIC_BMI2
+    if (ZSTD_DCtx_get_bmi2(dctx)) {
+        return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+    }
+#endif
+    return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+/* ZSTD_decompressSequencesLong() :
+ * decompression function triggered when a minimum share of offsets is considered "long",
+ * aka out of cache.
+ * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
+ * This function will try to mitigate main memory latency through the use of prefetching */
+static size_t
+ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
+                             void* dst, size_t maxDstSize,
+                             const void* seqStart, size_t seqSize, int nbSeq,
+                             const ZSTD_longOffset_e isLongOffset)
+{
+    DEBUGLOG(5, "ZSTD_decompressSequencesLong");
+#if DYNAMIC_BMI2
+    if (ZSTD_DCtx_get_bmi2(dctx)) {
+        return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+    }
+#endif
+  return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+
+/**
+ * @returns The total size of the history referenceable by zstd, including
+ * both the prefix and the extDict. At @p op any offset larger than this
+ * is invalid.
+ */
+static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
+{
+    return (size_t)(op - virtualStart);
+}
+
+typedef struct {
+    unsigned longOffsetShare;
+    unsigned maxNbAdditionalBits;
+} ZSTD_OffsetInfo;
+
+/* ZSTD_getOffsetInfo() :
+ * condition : offTable must be valid
+ * @return : "share" of long offsets (arbitrarily defined as > (1<<23))
+ *           compared to maximum possible of (1<<OffFSELog),
+ *           as well as the maximum number additional bits required.
+ */
+static ZSTD_OffsetInfo
+ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq)
+{
+    ZSTD_OffsetInfo info = {0, 0};
+    /* If nbSeq == 0, then the offTable is uninitialized, but we have
+     * no sequences, so both values should be 0.
+     */
+    if (nbSeq != 0) {
+        const void* ptr = offTable;
+        U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
+        const ZSTD_seqSymbol* table = offTable + 1;
+        U32 const max = 1 << tableLog;
+        U32 u;
+        DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
+
+        assert(max <= (1 << OffFSELog));  /* max not too large */
+        for (u=0; u<max; u++) {
+            info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
+            if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
+        }
+
+        assert(tableLog <= OffFSELog);
+        info.longOffsetShare <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
+    }
+
+    return info;
+}
+
+/**
+ * @returns The maximum offset we can decode in one read of our bitstream, without
+ * reloading more bits in the middle of the offset bits read. Any offsets larger
+ * than this must use the long offset decoder.
+ */
+static size_t ZSTD_maxShortOffset(void)
+{
+    if (MEM_64bits()) {
+        /* We can decode any offset without reloading bits.
+         * This might change if the max window size grows.
+         */
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+        return (size_t)-1;
+    } else {
+        /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
+         * This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
+         * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
+         */
+        size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
+        size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
+        assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
+        return maxOffset;
+    }
+}
+
+size_t
+ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
+                              void* dst, size_t dstCapacity,
+                        const void* src, size_t srcSize, const streaming_operation streaming)
+{   /* blockType == blockCompressed */
+    const BYTE* ip = (const BYTE*)src;
+    DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize);
+
+    /* Note : the wording of the specification
+     * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx).
+     * This generally does not happen, as it makes little sense,
+     * since an uncompressed block would feature same size and have no decompression cost.
+     * Also, note that decoder from reference libzstd before < v1.5.4
+     * would consider this edge case as an error.
+     * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx)
+     * for broader compatibility with the deployed ecosystem of zstd decoders */
+    RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, "");
+
+    /* Decode literals section */
+    {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
+        DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
+        if (ZSTD_isError(litCSize)) return litCSize;
+        ip += litCSize;
+        srcSize -= litCSize;
+    }
+
+    /* Build Decoding Tables */
+    {
+        /* Compute the maximum block size, which must also work when !frame and fParams are unset.
+         * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
+         */
+        size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx));
+        size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart);
+        /* isLongOffset must be true if there are long offsets.
+         * Offsets are long if they are larger than ZSTD_maxShortOffset().
+         * We don't expect that to be the case in 64-bit mode.
+         *
+         * We check here to see if our history is large enough to allow long offsets.
+         * If it isn't, then we can't possible have (valid) long offsets. If the offset
+         * is invalid, then it is okay to read it incorrectly.
+         *
+         * If isLongOffsets is true, then we will later check our decoding table to see
+         * if it is even possible to generate long offsets.
+         */
+        ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
+        /* These macros control at build-time which decompressor implementation
+         * we use. If neither is defined, we do some inspection and dispatch at
+         * runtime.
+         */
+#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+        int usePrefetchDecoder = dctx->ddictIsCold;
+#else
+        /* Set to 1 to avoid computing offset info if we don't need to.
+         * Otherwise this value is ignored.
+         */
+        int usePrefetchDecoder = 1;
+#endif
+        int nbSeq;
+        size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
+        if (ZSTD_isError(seqHSize)) return seqHSize;
+        ip += seqHSize;
+        srcSize -= seqHSize;
+
+        RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
+        RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall,
+                "invalid dst");
+
+        /* If we could potentially have long offsets, or we might want to use the prefetch decoder,
+         * compute information about the share of long offsets, and the maximum nbAdditionalBits.
+         * NOTE: could probably use a larger nbSeq limit
+         */
+        if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
+            ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
+            if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
+                /* If isLongOffset, but the maximum number of additional bits that we see in our table is small
+                 * enough, then we know it is impossible to have too long an offset in this block, so we can
+                 * use the regular offset decoder.
+                 */
+                isLongOffset = ZSTD_lo_isRegularOffset;
+            }
+            if (!usePrefetchDecoder) {
+                U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
+                usePrefetchDecoder = (info.longOffsetShare >= minShare);
+            }
+        }
+
+        dctx->ddictIsCold = 0;
+
+#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+        if (usePrefetchDecoder) {
+#else
+        (void)usePrefetchDecoder;
+        {
+#endif
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+            return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
+#endif
+        }
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+        /* else */
+        if (dctx->litBufferLocation == ZSTD_split)
+            return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
+        else
+            return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
+#endif
+    }
+}
+
+
+ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
+{
+    if (dst != dctx->previousDstEnd && dstSize > 0) {   /* not contiguous */
+        dctx->dictEnd = dctx->previousDstEnd;
+        dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
+        dctx->prefixStart = dst;
+        dctx->previousDstEnd = dst;
+    }
+}
+
+
+size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
+                                       void* dst, size_t dstCapacity,
+                                 const void* src, size_t srcSize)
+{
+    size_t dSize;
+    dctx->isFrameDecompression = 0;
+    ZSTD_checkContinuity(dctx, dst, dstCapacity);
+    dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming);
+    FORWARD_IF_ERROR(dSize, "");
+    dctx->previousDstEnd = (char*)dst + dSize;
+    return dSize;
+}
+
+
+/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity,
+                      const void* src, size_t srcSize)
+{
+    return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize);
+}
+/**** ended inlining decompress/zstd_decompress_block.c ****/
diff --git a/tools/gpu-aot/d3d-hook-map.md b/tools/gpu-aot/d3d-hook-map.md
new file mode 100644
index 0000000..fc18665
--- /dev/null
+++ b/tools/gpu-aot/d3d-hook-map.md
@@ -0,0 +1,66 @@
+# LEGO Dimensions native D3D hook map
+
+Target: TU23 `Default.xex`, mapped image `xexdump/dump/default.bin` at
+`0x82000000`. The embedded XDK shader compiler identifies the SDK family as
+`xdksep13`; its shader compiler reports `2.0.21256.0`.
+
+Do not infer a hook from function size alone. Several LEGO functions happen to
+have the exact size of a named re:Blue XDK function while implementing unrelated
+work.
+
+## Confirmed by instruction semantics
+
+| Guest address | Meaning | Evidence |
+|---|---|---|
+| `0x83FB7DA0` | `D3DDevice_SetRenderState_CullMode` | Replaces bits 0..2 of the device raster state and marks the raster group dirty. |
+| `0x83FB7DD0` | `D3DDevice_SetRenderState_FillMode` | Replaces the adjacent fill-mode bits and marks the same group dirty. |
+| `0x83FBA068` | `D3DDevice_SetScissorRect` | Reads a four-dword LTRB rectangle, clamps it to the current surface, and stores the scissor shadow. |
+| `0x83FBA9F8` | `D3DDevice_SetViewport` | Loads X/Y/W/H/MinZ/MaxZ from the six-field viewport and tail-calls the viewport state writer. |
+| `0x83FBA160` | texture state writer | Indexes the device texture slots and consumes the resource fetch address at offset `+0x1C`. The public entrypoint still needs naming. |
+| `0x83FBAAA8` | render-target state writer | Indexes four surface slots, stores the bound surface and its fetch address, and marks the corresponding target dirty. The public entrypoint still needs naming. |
+
+## High-confidence control-path anchors
+
+| Guest address | Meaning / role | Evidence |
+|---|---|---|
+| `0x83FAF9A0` | `Direct3D_CreateDevice` | Six-argument ABI matches `(adapter, deviceType, focusWindow, behaviorFlags, presentParams, outDevice)`: clears `*outDevice`, allocates `24704` (`0x6080`) bytes aligned to 128, initializes the allocation, then stores it through the sixth argument. |
+| `0x83FC9450` | reset/default-state initializer | Walks the render- and sampler-state descriptor tables, installs getter/setter dispatch pointers in the device, applies defaults, and clears all 26 fetch slots. Calling it from the native path is unsafe because applying the defaults also emits PM4. |
+| `0x83FAF180` | ring-buffer blocker poll | Checks the device read pointer and timeout state; called from the blocking wait loop. |
+| `0x83FBF7D0` | ring-buffer space wait | Compares requested and available ring space and loops through `0x83FAF180`. |
+| `0x83FB33E0` | surface copy/resolve path | Consumes a 52-byte surface descriptor, rewrites Xenos formats and emits copy/resolve packets. It is not safe to name this `Swap` from position or size alone. |
+| `0x83FB0DA8` | scanout-buffer programming helper | Takes `(device, buffer descriptor, flags)`, walks descriptor fields and emits display/ring packets. A caller at `0x83FC4994` passes one of the device's rotating 16-byte buffer records. This is **not** `Direct3D_CreateDevice`. |
+
+## TU23 device layout anchors
+
+The Sep'13 device is `0x6080` bytes, not the `0x5000`-byte device used by the
+older re:Blue target. The reset loop at `0x83FC9450` proves these offsets:
+
+| Device offset | Field |
+|---|---|
+| `0x040` | 101 render-state setter entrypoints |
+| `0x1D4` | 20 sampler-state setter entrypoints |
+| `0x224` | 101 render-state getter entrypoints |
+| `0x3B8` | 20 sampler-state getter entrypoints |
+| `0x480` | 26 six-dword fetch constants |
+| `0x32E0` | cached viewport (`X/Y/W/H/MinZ/MaxZ`) |
+| `0x32FC` | cached scissor rectangle (`left/top/right/bottom`) |
+
+The render-state descriptor triples begin at `0x847F9B18`; the sampler-state
+descriptor triples begin at `0x847F9FD8`. Each triple is
+`{ getter, setter, default }`. The native create hook copies only the dispatch
+pointers and seeds host-visible shadows directly, avoiding the original
+initializer's ring traffic.
+
+## Function-boundary source
+
+The range beginning at image address `0x8260E400` is `.pdata`: pairs of a
+function entrypoint and unwind metadata. It is useful for exact boundaries, but
+it is not a D3D dispatch table and therefore does not provide names.
+
+## Fast path to a complete map
+
+The preferred input is the user's own `xdksep13` build 21256 static graphics
+library (normally `d3d9.lib` plus its companion graphics libraries). Normalize
+relocations in each PowerPC COFF member, then match instruction windows against
+the decrypted image. Ambiguous matches must be resolved by call graph and
+argument semantics before a hook is enabled.
diff --git a/tools/gpu-aot/prelink_shader_cache.cpp b/tools/gpu-aot/prelink_shader_cache.cpp
new file mode 100644
index 0000000..fc8b4b7
--- /dev/null
+++ b/tools/gpu-aot/prelink_shader_cache.cpp
@@ -0,0 +1,198 @@
+/**
+ * @file    tools/prelink_shader_cache.cpp
+ * @brief   Build-time DXC pre-linker: emits linked_shader_cache.cpp with a
+ *          DXIL blob for every (shader, specConstantsMask subset) pair.
+ *
+ *          Standalone console tool: fprintf/stderr diagnostics by design
+ *          (the engine's BD_* logging is not linked here).
+ *
+ * @copyright Copyright (c) 2026 Tom Clay <tomc@tctechstuff.com>
+ *            All rights reserved.
+ * @license   BSD 3-Clause License
+ *            See LICENSE file in the project root for full license text.
+ */
+#include <algorithm>
+#include <atomic>
+#include <cstdint>
+#include <cstdio>
+#include <map>
+#include <thread>
+#include <vector>
+#define MINIZ_HEADER_FILE_ONLY
+#include <miniz.h>
+#include <zstd.h>
+
+#include "gpu_native/dxc_link.h"
+#include "gpu/shaders/shader_cache.h"
+
+namespace {
+
+void EmitBytes(FILE* f, const uint8_t* data, size_t size) {
+  for (size_t i = 0; i < size; ++i) {
+    std::fprintf(f, "%u,", data[i]);
+    if ((i & 31) == 31) std::fputc('\n', f);
+  }
+  std::fputc('\n', f);
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  if (argc != 2) {
+    std::fprintf(stderr, "usage: lego_gpu_prelink <output.cpp>\n");
+    return 1;
+  }
+
+  std::vector<uint8_t> dxil(g_dxilCacheDecompressedSize);
+  const size_t n =
+      ZSTD_decompress(dxil.data(), dxil.size(), g_compressedDxilCache,
+                      g_dxilCacheCompressedSize);
+  if (ZSTD_isError(n) || n != dxil.size()) {
+    std::fprintf(stderr, "prelink: DXIL cache decompression failed\n");
+    return 1;
+  }
+
+  struct Job {
+    const ShaderCacheEntry* entry;
+    uint32_t masked;
+  };
+  std::vector<Job> jobs;
+  size_t spec_shader_count = 0;
+  for (size_t i = 0; i < g_shaderCacheEntryCount; ++i) {
+    const auto& e = g_shaderCacheEntries[i];
+    if (!e.specConstantsMask) continue;
+    ++spec_shader_count;
+    // Every subset of the mask, including 0 (a spec-constant library still
+    // needs g_SpecConstants() resolved for the value-0 variant).
+    uint32_t s = 0;
+    do {
+      jobs.push_back({&e, s});
+      s = (s - e.specConstantsMask) & e.specConstantsMask;
+    } while (s);
+  }
+
+  std::map<uint32_t, std::vector<uint8_t>> specLibs;
+  for (const auto& j : jobs) specLibs.try_emplace(j.masked);
+  for (auto& [value, blob] : specLibs) {
+    blob = legodimensions::gpu_native::CompileSpecConstantLib(value);
+    if (blob.empty()) {
+      std::fprintf(stderr, "prelink: spec lib compile failed (value=%u)\n",
+                   value);
+      return 1;
+    }
+  }
+
+  struct Result {
+    uint64_t hash;
+    uint32_t masked;
+    std::vector<uint8_t> dxil;
+  };
+  std::vector<Result> results(jobs.size());
+  std::atomic<size_t> next{0};
+  std::atomic<bool> failed{false};
+
+  auto worker = [&] {
+    for (;;) {
+      const size_t i = next.fetch_add(1);
+      if (i >= jobs.size() || failed.load()) return;
+      const Job& j = jobs[i];
+      const uint8_t* lib = dxil.data() + j.entry->dxilOffset;
+      const auto& spec = specLibs.find(j.masked)->second;
+      // The cache stores no shader type, and a library's main only validates
+      // against its own stage, so the failing profile identifies it.
+      auto vs = legodimensions::gpu_native::LinkSpecConstantLib(
+          lib, j.entry->dxilSize, spec.data(), spec.size(), L"vs_6_0");
+      auto ps = legodimensions::gpu_native::LinkSpecConstantLib(
+          lib, j.entry->dxilSize, spec.data(), spec.size(), L"ps_6_0");
+      if (vs.empty() == ps.empty()) {
+        std::fprintf(stderr,
+                     "prelink: %s link hash=%016llX mask=0x%X (vs=%zu ps=%zu)\n",
+                     vs.empty() ? "failed" : "ambiguous",
+                     static_cast<unsigned long long>(j.entry->hash), j.masked,
+                     vs.size(), ps.size());
+        failed.store(true);
+        return;
+      }
+      results[i] = {j.entry->hash, j.masked,
+                    vs.empty() ? std::move(ps) : std::move(vs)};
+    }
+  };
+  {
+    unsigned hw = std::thread::hardware_concurrency();
+    if (!hw) hw = 4;
+    std::vector<std::thread> pool;
+    for (unsigned t = 0; t < hw; ++t) pool.emplace_back(worker);
+    for (auto& t : pool) t.join();
+  }
+  if (failed.load()) return 1;
+
+  std::sort(results.begin(), results.end(), [](const Result& a,
+                                               const Result& b) {
+    if (a.hash != b.hash) return a.hash < b.hash;
+    return a.masked < b.masked;
+  });
+
+  struct OutEntry {
+    uint64_t hash;
+    uint32_t masked;
+    uint32_t offset;
+    uint32_t size;
+  };
+  std::vector<OutEntry> entries;
+  std::vector<uint8_t> blob;
+  for (const auto& r : results) {
+    entries.push_back({r.hash, r.masked, static_cast<uint32_t>(blob.size()),
+                       static_cast<uint32_t>(r.dxil.size())});
+    blob.insert(blob.end(), r.dxil.begin(), r.dxil.end());
+  }
+  if (blob.empty()) blob.push_back(0);  // keep the C arrays non-empty
+
+  mz_ulong comp_len = mz_compressBound(static_cast<mz_ulong>(blob.size()));
+  std::vector<uint8_t> comp(comp_len);
+  if (mz_compress2(comp.data(), &comp_len, blob.data(),
+                   static_cast<mz_ulong>(blob.size()),
+                   MZ_UBER_COMPRESSION) != MZ_OK) {
+    std::fprintf(stderr, "prelink: deflate failed\n");
+    return 1;
+  }
+  // Exercise the exact runtime decompressor before emitting the translation
+  // unit. A corrupt linked archive must fail the build, not the first draw.
+  std::vector<uint8_t> round_trip(blob.size());
+  mz_ulong round_trip_len = static_cast<mz_ulong>(round_trip.size());
+  if (mz_uncompress(round_trip.data(), &round_trip_len, comp.data(), comp_len) != MZ_OK ||
+      round_trip_len != blob.size() || round_trip != blob) {
+    std::fprintf(stderr, "prelink: deflate round-trip validation failed\n");
+    return 1;
+  }
+
+  FILE* f = std::fopen(argv[1], "wb");
+  if (!f) {
+    std::fprintf(stderr, "prelink: cannot open %s\n", argv[1]);
+    return 1;
+  }
+  std::fprintf(f, "#include \"gpu/shaders/linked_shader_cache.h\"\n");
+  std::fprintf(f, "LinkedShaderCacheEntry g_linkedShaderCacheEntries[] = {\n");
+  for (const auto& e : entries) {
+    std::fprintf(f, "\t{ 0x%llX, 0x%X, %u, %u },\n",
+                 static_cast<unsigned long long>(e.hash), e.masked, e.offset,
+                 e.size);
+  }
+  if (entries.empty()) std::fprintf(f, "\t{ 0, 0, 0, 0 },\n");
+  std::fprintf(f, "};\n");
+  std::fprintf(f, "const size_t g_linkedShaderCacheEntryCount = %zu;\n",
+               entries.size());
+  std::fprintf(f, "const uint8_t g_compressedLinkedDxilCache[] = {\n");
+  EmitBytes(f, comp.data(), comp_len);
+  std::fprintf(f, "};\n");
+  std::fprintf(f, "const size_t g_linkedDxilCacheCompressedSize = %zu;\n",
+               static_cast<size_t>(comp_len));
+  std::fprintf(f, "const size_t g_linkedDxilCacheDecompressedSize = %zu;\n",
+               blob.size());
+  std::fclose(f);
+
+  std::printf("prelink: %zu variant(s) from %zu spec-constant shader(s), "
+              "%zu -> %zu bytes\n",
+              entries.size(), spec_shader_count, blob.size(),
+              static_cast<size_t>(comp_len));
+  return 0;
+}

From abc6a3547233d5180f7e0f46047aa8cefdb21de0 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 14:17:28 +0300
Subject: [PATCH 05/44] gpu: map Sep13 D3D functions

---
 tools/gpu-aot/d3d-hook-map.md       |  22 +-
 tools/gpu-aot/d3d-sep13-matches.tsv | 420 ++++++++++++++++++++++++++++
 tools/gpu-aot/match_map_symbols.py  | 119 ++++++++
 3 files changed, 560 insertions(+), 1 deletion(-)
 create mode 100644 tools/gpu-aot/d3d-sep13-matches.tsv
 create mode 100644 tools/gpu-aot/match_map_symbols.py

diff --git a/tools/gpu-aot/d3d-hook-map.md b/tools/gpu-aot/d3d-hook-map.md
index fc18665..8af0fb9 100644
--- a/tools/gpu-aot/d3d-hook-map.md
+++ b/tools/gpu-aot/d3d-hook-map.md
@@ -27,7 +27,7 @@ work.
 | `0x83FC9450` | reset/default-state initializer | Walks the render- and sampler-state descriptor tables, installs getter/setter dispatch pointers in the device, applies defaults, and clears all 26 fetch slots. Calling it from the native path is unsafe because applying the defaults also emits PM4. |
 | `0x83FAF180` | ring-buffer blocker poll | Checks the device read pointer and timeout state; called from the blocking wait loop. |
 | `0x83FBF7D0` | ring-buffer space wait | Compares requested and available ring space and loops through `0x83FAF180`. |
-| `0x83FB33E0` | surface copy/resolve path | Consumes a 52-byte surface descriptor, rewrites Xenos formats and emits copy/resolve packets. It is not safe to name this `Swap` from position or size alone. |
+| `0x83FB33E0` | `D3DDevice_Swap` | Exact normalized instruction match against the Sep'13 `d3d9:swap.obj` symbol at `0x82467B10`. Its surface descriptor processing is the scanout/present path, not a standalone resolve as initially suspected. |
 | `0x83FB0DA8` | scanout-buffer programming helper | Takes `(device, buffer descriptor, flags)`, walks descriptor fields and emits display/ring packets. A caller at `0x83FC4994` passes one of the device's rotating 16-byte buffer records. This is **not** `Direct3D_CreateDevice`. |
 
 ## TU23 device layout anchors
@@ -64,3 +64,23 @@ library (normally `d3d9.lib` plus its companion graphics libraries). Normalize
 relocations in each PowerPC COFF member, then match instruction windows against
 the decrypted image. Ambiguous matches must be resolved by call graph and
 argument semantics before a hook is enabled.
+
+An independently published 2013-09-13 Xbox 360 MAP-labelled image now provides
+a second route. `match_map_symbols.py` normalizes link-dependent PPC immediates
+and branches and matches the statically linked `d3d9:*` functions against TU23.
+The known `Direct3D_CreateDevice` match is the control. High-confidence results:
+
+| Guest address | MAP symbol |
+|---|---|
+| `0x83FB33E0` | `D3DDevice_Swap` |
+| `0x83FB5570` | `D3DDevice_CreateTexture` |
+| `0x83FB58A8` | `D3DDevice_SetTexture` |
+| `0x83FB6828` | `D3DDevice_SetPixelShader` |
+| `0x83FB6A30` | `D3DDevice_SetVertexShader` |
+| `0x83FB7528` | `D3DDevice_CreatePixelShader` |
+| `0x83FB7750` | `D3DDevice_CreateVertexShader` |
+| `0x83FBC9D8` | `D3DDevice_Clear` |
+| `0x83FBDF80` | `D3DDevice_ResolveEx` |
+| `0x83FBF0D8` | `D3DDevice_Resolve` |
+| `0x83FC6640` | `D3DDevice_DrawVertices` |
+| `0x83FC6A58` | `D3DDevice_DrawIndexedVertices` |
diff --git a/tools/gpu-aot/d3d-sep13-matches.tsv b/tools/gpu-aot/d3d-sep13-matches.tsv
new file mode 100644
index 0000000..da6b6df
--- /dev/null
+++ b/tools/gpu-aot/d3d-sep13-matches.tsv
@@ -0,0 +1,420 @@
+?PixCaptureVideoFrame@D3D@@YAXPAVCDevice@1@KKKW4_D3DFORMAT@@KKPBU_D3DPRIVATE_SCALER_PARAMETERS@@PBU_D3DGAMMARAMP@@@Z	0x823CBA28	0x83FC4338	d3d9:pix.obj
+?QueueShaderWriteback@D3D@@YAXPAVCDevice@1@PAX@Z	0x823CC1E0	0x83FC45E0	d3d9:pix.obj
+# ambiguous ?SetupFormat@D3D@@YAXPAVCDevice@1@W4_D3DFORMAT@@@Z	0x823CCF10	0x83FAF5D8,0x83FB27F0	d3d9:swap.obj
+?SimpleSwap@D3D@@YAXPAVCDevice@1@KW4_D3DFORMAT@@@Z	0x823CCFB8	0x83FB3100	d3d9:swap.obj
+D3DDevice_CreateSurface	0x823D1150	0x83FB5690	d3d9:texture.obj
+?P_QPC@D3D@@YAXPAUD3DDevice@@PAUD3DPerfCounters@@K@Z	0x823D42A8	0x83FB0DA8	d3d9:query.obj
+?DestroyResource@D3D@@YAXPAUD3DResource@@@Z	0x823D8918	0x83FC5730	d3d9:resource.obj
+?PixSyncronizeToPresentationInterval@D3D@@YAXPAVCDevice@1@H@Z	0x823DC628	0x83FC4670	d3d9:pix.obj
+D3DDevice_BlockOnFence	0x823E00A8	0x83FBF8A0	d3d9:ring.obj
+D3DVolumeTexture_LockBox	0x823E0110	0x83FA5F88	d3d9:texture.obj
+?IssueSampleCountQuery@D3D@@YAXPAVCDevice@1@PAUCQuery@1@K@Z	0x823E0248	0x83FCBF00	d3d9:xps.obj
+D3DQuery_Issue	0x823E0410	0x83FB0738	d3d9:query.obj
+?PixSwap@D3D@@YAXPAVCDevice@1@@Z	0x823E06C0	0x83FC4748	d3d9:pix.obj
+?PIXMetaRecordTilingInfo@D3D@@YAXPAVCDevice@1@KPBU_D3DRECT@@K@Z	0x823E5070	0x83FC2C40	d3d9:pix.obj
+D3DDevice_SetShaderGPRAllocation	0x823E9430	0x83FB6DD0	d3d9:shader.obj
+?BuildGprAllocationSetup@D3D@@YAKPAVCDevice@1@PAK1@Z	0x823E9518	0x83FB6120	d3d9:shader.obj
+D3DDevice_SetScreenExtentQueryMode	0x823F0C38	0x83FAFC78	d3d9:query.obj
+?PatchVertexShaderToMatchVertexDeclaration@D3D@@YAXPAVCVertexShader@1@PAKPBVCVertexDeclaration@1@PBEK@Z	0x823F0CA8	0x83FD1470	d3d9:lazy.obj
+D3DDevice_IsFencePending	0x823F83C8	0x83FC0D50	d3d9:ring.obj
+?SetPending_ClipPlanes@D3D@@YAXPAVCDevice@1@_K@Z	0x824001D8	0x83FD1048	d3d9:lazy.obj
+?ResetAllState@D3D@@YAXPAVCDevice@1@H@Z	0x82403C18	0x83FBA618	d3d9:state.obj
+D3DDevice_BeginTiling	0x824041B0	0x83FBCD28	d3d9:tile.obj
+?StartWorkerQueue@D3D@@YAXPAVCDevice@1@@Z	0x82404520	0x83FCAB10	d3d9:xps.obj
+?ClearRect@D3D@@YAXPAVCDevice@1@JJJJPBU__vector4@@MTGPU_SURFACEINFO@@TGPU_COLORINFO@@KKKK@Z	0x82408C90	0x83FBBA18	d3d9:clear.obj
+?ClearRects@D3D@@YAXPAVCDevice@1@KPBU_D3DRECT@@PBU__vector4@@MTGPU_SURFACEINFO@@TGPU_COLORINFO@@KK@Z	0x82408EF8	0x83FBB760	d3d9:clear.obj
+# ambiguous D3DDevice_SetPixelShaderConstantB	0x824152C8	0x83FB6630,0x83FB6690	d3d9:shader.obj
+# ambiguous D3DDevice_SetPixelShaderConstantI	0x82415328	0x83FB66F0,0x83FB6748	d3d9:shader.obj
+D3DDevice_SetRenderState_BlendOpAlpha	0x82415380	0x83FB8078	d3d9:state.obj
+D3DDevice_SetRenderState_DestBlendAlpha	0x82415498	0x83FB8158	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_HighPrecisionBlendEnable1	0x824154F8	0x83FB9020,0x83FB90B8,0x83FB9150,0x83FB91E8	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_HighPrecisionBlendEnable3	0x82415588	0x83FB9020,0x83FB90B8,0x83FB9150,0x83FB91E8	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_PointSizeMax	0x82415658	0x83FB8BD0,0x83FB8C20	d3d9:state.obj
+D3DDevice_SetRenderState_SeparateAlphaBlendEnable	0x824156A0	0x83FB81C8	d3d9:state.obj
+D3DDevice_SetRenderState_SrcBlendAlpha	0x82415728	0x83FB80E8	d3d9:state.obj
+# ambiguous D3DDevice_SetVertexShaderConstantB	0x82415920	0x83FB6630,0x83FB6690	d3d9:shader.obj
+?SetFence@CDevice@D3D@@QAAPAKPAK@Z	0x824169B0	0x83FC0180	d3d9:ring.obj
+D3DDevice_SetRenderState_BlendFactor	0x82418610	0x83FB82F0	d3d9:state.obj
+D3DDevice_SetRenderState_BlendOp	0x824186A0	0x83FB7EC8	d3d9:state.obj
+?ClearSurface@D3D@@YAXPAVCDevice@1@KKJJJJPBU__vector4@@MK@Z	0x8241A928	0x83FBBC80	d3d9:clear.obj
+D3DDevice_SetPixelShader	0x8241DED0	0x83FB6828	d3d9:shader.obj
+D3DDevice_SetVertexShader	0x8241E090	0x83FB6A30	d3d9:shader.obj
+D3DDevice_SetIndices	0x8241E710	0x83FBA308	d3d9:state.obj
+?BlockOnPrimaryRange@CDevice@D3D@@QAAKKK@Z	0x8241F220	0x83FBF260	d3d9:ring.obj
+?AddCallsToPrimaryBuffer@CDevice@D3D@@QAAXPBUSegmentCall@2@K@Z	0x8241F2D0	0x83FBFCA0	d3d9:ring.obj
+?KickOffSegment@CDevice@D3D@@QAAXXZ	0x8241F5A0	0x83FC08D0	d3d9:ring.obj
+D3DDevice_Resolve	0x82420310	0x83FBF0D8	d3d9:resolve.obj
+D3DDevice_AcquireThreadOwnership	0x82424310	0x83FAF588	d3d9:device.obj
+?SetSurfaceInfo@D3D@@YAXPAVCDevice@1@PAUD3DSurface@@@Z	0x82428840	0x83FBAA18	d3d9:state.obj
+D3DDevice_SetViewport	0x824288D0	0x83FBA978	d3d9:state.obj
+?SetViewport@D3D@@YAXPAUD3DDevice@@MMMMMMK@Z	0x82428950	0x83FBA710	d3d9:state.obj
+D3DDevice_SetScissorRect	0x82428BA8	0x83FBA068	d3d9:state.obj
+?SetSurfaceClip@D3D@@YAXPAVCDevice@1@HHHH@Z	0x82428CA0	0x83FB7AB8	d3d9:state.obj
+?KickOff@CDevice@D3D@@QAAPAKXZ	0x82429230	0x83FC0C10	d3d9:ring.obj
+?FindTextureSize@D3D@@YAKKKKKKKHHKHKPAK00@Z	0x82429E20	0x83FB3ED0	d3d9:texture.obj
+?AlignTextureDimensions@D3D@@YAKPAK00KKKH@Z	0x8242A1A0	0x83FB3DB0	d3d9:texture.obj
+D3DDevice_SetPredication	0x8242A778	0x83FBCBB8	d3d9:tile.obj
+D3DDevice_InsertCallback	0x8242A8E8	0x83FC0F38	d3d9:ring.obj
+?InsertCallback@D3D@@YAPAKPAVCDevice@1@PAKKP6AXK@ZK@Z	0x8242A940	0x83FC02C0	d3d9:ring.obj
+?SetPending_HiZEnable@D3D@@YA_KPAVCDevice@1@_K@Z	0x8242ABC0	0x83FD0A00	d3d9:lazy.obj
+?SetPending_FetchConstants@D3D@@YAXPAVCDevice@1@_K@Z	0x8242ADB8	0x83FD11C8	d3d9:lazy.obj
+?SetLiteralShaderConstants@D3D@@YAXPAVCDevice@1@PAU_UCODE_HEADER@@PAX@Z	0x8242AF10	0x83FD1EC8	d3d9:lazy.obj
+?SetPending_Shaders@D3D@@YA_KPAVCDevice@1@_K@Z	0x8242AFC0	0x83FD20B8	d3d9:lazy.obj
+D3DDevice_ResolveEx	0x8242B940	0x83FBDF80	d3d9:resolve.obj
+D3DDevice_DrawIndexedVertices	0x8242CA98	0x83FC6A58	d3d9:draw.obj
+?SetPending_RenderStates@D3D@@YAXPAVCDevice@1@_KKPAX@Z	0x8242CF10	0x83FD0F88	d3d9:lazy.obj
+D3DDevice_BeginVertices	0x8242CFD0	0x83FC6180	d3d9:draw.obj
+?SetPending_AluConstants@D3D@@YAXPAVCDevice@1@_KKPAX@Z	0x8242D490	0x83FD1320	d3d9:lazy.obj
+?BeginRingAlloc@CDevice@D3D@@QAAPAKKK@Z	0x8242D5E0	0x83FBFA48	d3d9:ring.obj
+?RingBufferDeviceAllocate@D3D@@YAPAXPAVCDevice@1@KPAKK@Z	0x8242D6C8	0x83FBF5A0	d3d9:ring.obj
+?BlockOnSecondaryPosition@CDevice@D3D@@QAAXPAKK@Z	0x8242D8F8	0x83FBF1C8	d3d9:ring.obj
+D3DDevice_SetRenderTarget	0x8242D9E0	0x83FBAAA8	d3d9:state.obj
+?GrowableAllocate@CCommandBuffer@D3D@@QAAPAKKPAKK@Z	0x8242E458	0x83FC71A8	d3d9:commands.obj
+D3DDevice_SetStreamSource	0x82443090	0x83FBA160	d3d9:state.obj
+D3DDevice_UnsetAll	0x824431B0	0x83FAF670	d3d9:device.obj
+# ambiguous D3DDevice_SetSamplerState_AnisotropyBias	0x824432C0	0x83FB9B00,0x83FB9BA8	d3d9:state.obj
+D3DDevice_SetSamplerState_MagFilterZ	0x82443320	0x83FB98E8	d3d9:state.obj
+D3DDevice_SetSamplerState_MaxMipLevel	0x824433B0	0x83FB9C48	d3d9:state.obj
+D3DDevice_SetSamplerState_MinFilterZ	0x82443420	0x83FB9740	d3d9:state.obj
+# ambiguous D3DDevice_SetSamplerState_MipMapLodBias	0x824434B8	0x83FB9B00,0x83FB9BA8	d3d9:state.obj
+D3DDevice_SetTexture	0x82443518	0x83FB58A8	d3d9:texture.obj
+# ambiguous D3DDevice_SetVertexShaderConstantFN	0x82444500	0x83FB6480,0x83FB6558	d3d9:shader.obj
+D3DDevice_SetSamplerState_SeparateZFilterEnable	0x824445D8	0x83FB99E0	d3d9:state.obj
+D3DDevice_SetSamplerState_MinMipLevel	0x82444670	0x83FB9CC8	d3d9:state.obj
+# ambiguous D3DDevice_SetSamplerState_MinFilter	0x824446E0	0x83FB9640,0x83FB97E8	d3d9:state.obj
+D3DDevice_SetSamplerState_MaxAnisotropy	0x824447A8	0x83FB9A88	d3d9:state.obj
+# ambiguous D3DDevice_SetSamplerState_MagFilter	0x82444810	0x83FB9640,0x83FB97E8	d3d9:state.obj
+D3DDevice_SetSamplerState_MipFilter	0x82445868	0x83FB9988	d3d9:state.obj
+D3DDevice_SetRenderState_AlphaBlendEnable	0x82445E50	0x83FB7E38	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_HighPrecisionBlendEnable	0x82445F68	0x83FB9020,0x83FB90B8,0x83FB9150,0x83FB91E8	d3d9:state.obj
+D3DDevice_SetRenderState_SrcBlend	0x82445FF8	0x83FB7F58	d3d9:state.obj
+D3DDevice_SetRenderState_DestBlend	0x82446078	0x83FB7FE8	d3d9:state.obj
+?DirectShaderPatch@D3D@@YAHPAVCDevice@1@PAVCVertexShader@1@PAVCVertexDeclaration@1@K@Z	0x82449930	0x83FD1F78	d3d9:lazy.obj
+D3DDevice_SetDepthStencilSurface	0x8244A190	0x83FBAE38	d3d9:state.obj
+?ClearF@D3D@@YAXPAVCDevice@1@KPBU_D3DRECT@@PBU__vector4@@MK@Z	0x8244A5B0	0x83FBC220	d3d9:clear.obj
+?CreateInvalidateBuffer@CDevice@D3D@@QAAXPAK0@Z	0x8244B818	0x83FBFF70	d3d9:ring.obj
+?GrowableFree@CCommandBuffer@D3D@@QAAXXZ	0x8244BCB0	0x83FC7228	d3d9:commands.obj
+?SetPending_Predicated@D3D@@YA_KPAVCDevice@1@_KK@Z	0x8244C0C8	0x83FD0BF8	d3d9:lazy.obj
+?BeginRingBig@CDevice@D3D@@QAAPAKK@Z	0x8244C660	0x83FC0E48	d3d9:ring.obj
+?SetTextureHeader@D3D@@YAXW4_D3DRESOURCETYPE@@IIIIKW4_D3DFORMAT@@W4MIPPACKINGTYPE@1@KHIPAUD3DBaseTexture@@PAK4@Z	0x82450930	0x83FB4FC8	d3d9:texture.obj
+D3DResource_GetType	0x82450E28	0x83FC49E8	d3d9:resource.obj
+?StartNewSegment@CDevice@D3D@@QAAPAKK@Z	0x82451210	0x83FBFB30	d3d9:ring.obj
+D3DResource_Release	0x824599A8	0x83FC59D8	d3d9:resource.obj
+# ambiguous ?MakeSegmentCallSpace@D3D@@YAPAUSegmentCall@1@PAVCDevice@1@@Z	0x8245A4E8	0x83FC7048,0x83FC70F8	d3d9:commands.obj
+# ambiguous D3DDevice_SetVertexShaderConstantI	0x8245B390	0x83FB66F0,0x83FB6748	d3d9:shader.obj
+D3DDevice_SetRenderState_ViewportEnable	0x8245B518	0x83FB8FD0	d3d9:state.obj
+D3DDevice_SetRenderState_SlopeScaleDepthBias	0x8245B5D8	0x83FB88A8	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_PointSizeMin	0x8245B6A0	0x83FB8BD0,0x83FB8C20	d3d9:state.obj
+D3DDevice_SetRenderState_PointSize	0x8245B6E0	0x83FB8B78	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_HighPrecisionBlendEnable2	0x8245B770	0x83FB9020,0x83FB90B8,0x83FB9150,0x83FB91E8	d3d9:state.obj
+D3DDevice_SetRenderState_DepthBias	0x8245B828	0x83FB8970	d3d9:state.obj
+D3DDevice_SetRenderState_ClipPlaneEnable	0x8245B8F8	0x83FB8848	d3d9:state.obj
+# ambiguous D3DDevice_SetPixelShaderConstantFN	0x8245B9A8	0x83FB6480,0x83FB6558	d3d9:shader.obj
+D3DDevice_ClearF	0x8245EA50	0x83FBC8A0	d3d9:clear.obj
+D3DDevice_Clear	0x8245EB88	0x83FBC9D8	d3d9:clear.obj
+?MakeSpace@CRingAllocList@D3D@@AAAPAKXZ	0x82461F28	0x83FC0540	d3d9:ring.obj
+?FlushCachedMemory@D3D@@YAXPAX0H@Z	0x82462038	0x83FCA518	d3d9:memory.obj
+?IncrementalShaderPatchAndLoad@D3D@@YAXPAVCDevice@1@HPAVCVertexShader@1@PATGPU_PROGRAMCONTROL@@PAVCVertexDeclaration@1@PAU_UCODE_R500VSUCODE@@PAU_UCODE_R500PSUCODE@@K@Z	0x82462100	0x83FD1D70	d3d9:lazy.obj
+?StopWorkerQueue@D3D@@YAHPAVCDevice@1@KK@Z	0x82465D80	0x83FCBA68	d3d9:xps.obj
+?Finalize@CRingAllocList@D3D@@QAAXXZ	0x82465F68	0x83FBF960	d3d9:ring.obj
+# ambiguous ?MakeBufferResourceSpace@D3D@@YAPAVBufferResource@1@PAVCDevice@1@@Z	0x824664D8	0x83FC7048,0x83FC70F8	d3d9:commands.obj
+D3DDevice_Swap	0x82467B10	0x83FB33E0	d3d9:swap.obj
+?BlockOnFence@CDevice@D3D@@QAAXKW4_D3DBLOCKTYPE@@PAUD3DResource@@_N@Z	0x82468278	0x83FBF7D0	d3d9:ring.obj
+D3DDevice_EndTiling	0x82468348	0x83FBD098	d3d9:tile.obj
+D3DDevice_GetRenderTarget	0x82469888	0x83FBA398	d3d9:state.obj
+D3DResource_AddRef	0x824698D0	0x83FC4CE0	d3d9:resource.obj
+D3DDevice_GetDepthStencilSurface	0x82469948	0x83FBA3E0	d3d9:state.obj
+?LockResource@D3D@@YAPAXPAUD3DResource@@_KW4_D3DBLOCKTYPE@@KKKKKKK@Z	0x82473648	0x83FC5D68	d3d9:resource.obj
+D3DQuery_GetData	0x82479068	0x83FB09E8	d3d9:query.obj
+?SetTileAndDepthClear@D3D@@YAXPAVCDevice@1@PBU_D3DRESOLVE_PARAMETERSEX@@MK@Z	0x8248DE50	0x83FBD6B0	d3d9:resolve.obj
+?TrainEDRAM@D3D@@YAHPAVCDevice@1@KH@Z	0x82497138	0x83FCE5A8	d3d9:hsio.obj
+?SynchronizeToPresentationInterval@D3D@@YAXPAVCDevice@1@K@Z	0x82497298	0x83FB2F20	d3d9:swap.obj
+?AddChunkNode@CRingAllocList@D3D@@QAAXPAURingAllocChunk@2@PAK@Z	0x8249ACC8	0x83FBF9A8	d3d9:ring.obj
+D3DVertexBuffer_Lock	0x824A0CF8	0x83FC60D8	d3d9:resource.obj
+?UnlockResource@D3D@@YAXPAUD3DResource@@KK@Z	0x824A38B0	0x83FC4A78	d3d9:resource.obj
+?SetColorClear@D3D@@YAXPAVCDevice@1@PBU_D3DRESOLVE_PARAMETERSEX@@KU__vector4@@@Z	0x824A3D78	0x83FBDC48	d3d9:resolve.obj
+?LockSurface@D3D@@YAXPAUD3DBaseTexture@@_KIIKPAK222@Z	0x824A4E68	0x83FB4E60	d3d9:texture.obj
+?FindSurfaceWithinTexture@D3D@@YAXPAUD3DBaseTexture@@IIPAK1111111@Z	0x824A4F40	0x83FB4978	d3d9:texture.obj
+?ComputeClearColor@D3D@@YAKKKU__vector4@@PAK1@Z	0x824A6E58	0x83FBD7D0	d3d9:resolve.obj
+D3DPerfCounters_GetValues	0x824ACCF8	0x83FB22F0	d3d9:query.obj
+?GetGPUPerfCounters@D3D@@YAHPAVCDevice@1@@Z	0x824ACD68	0x83FC1DC0	d3d9:pix.obj
+?CalculateBandwidth@D3D@@YAM_K0@Z	0x824ACE78	0x83FC1640	d3d9:pix.obj
+?HandleCaptureState@D3D@@YAXPAVCDevice@1@@Z	0x824ACEE8	0x83FC4468	d3d9:pix.obj
+D3DDevice_SetRenderState_HiZWriteEnable	0x824AD0A8	0x83FB94F8	d3d9:state.obj
+?XBMProcessSwap@D3D@@YAXPAVCDevice@1@PAUD3DBaseTexture@@@Z	0x824AE1E0	0x83FD0480	d3d9:movie.obj
+?SetSurfaceHeader@D3D@@YAXIIW4_D3DFORMAT@@W4_D3DMULTISAMPLE_TYPE@@PBU_D3DSURFACE_PARAMETERS@@PAUD3DSurface@@PAK4@Z	0x824B09D8	0x83FB52C0	d3d9:texture.obj
+??0CBlocker@D3D@@QAA@PAVCDevice@1@W4_D3DBLOCKTYPE@@@Z	0x83578948	0x83FAF088	d3d9:device.obj
+??1CBlocker@D3D@@QAA@XZ	0x83578978	0x83FAF0B8	d3d9:device.obj
+?Check@CBlocker@D3D@@QAAHXZ	0x83578A40	0x83FAF180	d3d9:device.obj
+?UninitializeAsyncResources@D3D@@YAXPAVCDevice@1@@Z	0x83578B20	0x83FAF260	d3d9:device.obj
+?InitializeAsyncCommandBuffers@D3D@@YAHPAVCDevice@1@@Z	0x83578B70	0x83FAF2B0	d3d9:device.obj
+?UninitializeAsyncCommandBuffers@D3D@@YAXPAVCDevice@1@@Z	0x83578C70	0x83FAF3B0	d3d9:device.obj
+?WaitUntilIdleOrFlushCaches@D3D@@YAXPAVCDevice@1@@Z	0x83578CE8	0x83FAF428	d3d9:device.obj
+D3DDevice_AddRef	0x83578D98	0x83FAF4D8	d3d9:device.obj
+D3DDevice_Release	0x83578DC0	0x83FAF500	d3d9:device.obj
+# ambiguous D3DDevice_FlushHiZStencil	0x83578F68	0x83FAF5D8,0x83FB27F0	d3d9:device.obj
+?InitializeAsyncResources@D3D@@YAHPAVCDevice@1@@Z	0x835790A8	0x83FAF828	d3d9:device.obj
+Direct3D_CreateDevice	0x83579220	0x83FAF9A0	d3d9:device.obj
+D3DDevice_GetRenderState_BlendFactor	0x835793E0	0x83FB8380	d3d9:state.obj
+D3DDevice_SetRenderState_LastPixel	0x83579450	0x83FB83F0	d3d9:state.obj
+D3DDevice_GetRenderState_LastPixel	0x83579478	0x83FB8418	d3d9:state.obj
+D3DDevice_SetRenderState_LineWidth	0x83579488	0x83FB8428	d3d9:state.obj
+D3DDevice_GetRenderState_LineWidth	0x835794C8	0x83FB8468	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_StencilWriteMask	0x835795F8	0x83FB87A0,0x83FB87C0,0x83FB87E0,0x83FB8800	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_CCWStencilRef	0x83579600	0x83FB8788,0x83FB87A8,0x83FB87C8,0x83FB87E8,0x83FB8808	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_CCWStencilRef	0x83579618	0x83FB87A0,0x83FB87C0,0x83FB87E0,0x83FB8800	d3d9:state.obj
+# ambiguous D3DDevice_SetRenderState_CCWStencilMask	0x83579620	0x83FB8788,0x83FB87A8,0x83FB87C8,0x83FB87E8,0x83FB8808	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap0	0x83579718	0x83FB8ED0,0x83FB8F50	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap1	0x83579728	0x83FB8EE0,0x83FB8F60	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap2	0x83579738	0x83FB8EF0,0x83FB8F70	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap3	0x83579748	0x83FB8F00,0x83FB8F80	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap4	0x83579758	0x83FB8F10,0x83FB8F90	d3d9:state.obj
+D3DDevice_GetRenderState_Wrap5	0x83579768	0x83FB8F20	d3d9:state.obj
+D3DDevice_GetRenderState_Wrap6	0x83579778	0x83FB8F30	d3d9:state.obj
+D3DDevice_GetRenderState_Wrap7	0x83579788	0x83FB8F40	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap8	0x83579798	0x83FB8ED0,0x83FB8F50	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap9	0x835797A8	0x83FB8EE0,0x83FB8F60	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap10	0x835797B8	0x83FB8EF0,0x83FB8F70	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap11	0x835797C8	0x83FB8F00,0x83FB8F80	d3d9:state.obj
+# ambiguous D3DDevice_GetRenderState_Wrap12	0x835797D8	0x83FB8F10,0x83FB8F90	d3d9:state.obj
+D3DDevice_GetRenderState_AlphaToMaskOffsets	0x835798B0	0x83FB9408	d3d9:state.obj
+D3DDevice_SetRenderState_GuardBand_X	0x835798B8	0x83FB9410	d3d9:state.obj
+D3DDevice_GetRenderState_GuardBand_X	0x835798E0	0x83FB9438	d3d9:state.obj
+D3DDevice_SetRenderState_GuardBand_Y	0x835798E8	0x83FB9440	d3d9:state.obj
+D3DDevice_GetRenderState_GuardBand_Y	0x83579910	0x83FB9468	d3d9:state.obj
+D3DDevice_SetRenderState_DiscardBand_X	0x83579918	0x83FB9470	d3d9:state.obj
+D3DDevice_GetRenderState_DiscardBand_X	0x83579938	0x83FB9490	d3d9:state.obj
+D3DDevice_SetRenderState_DiscardBand_Y	0x83579940	0x83FB9498	d3d9:state.obj
+D3DDevice_GetRenderState_HiZWriteEnable	0x835799A0	0x83FB9550	d3d9:state.obj
+D3DDevice_SetRenderState_HiStencilEnable	0x835799B0	0x83FB9560	d3d9:state.obj
+D3DDevice_GetRenderState_HiStencilEnable	0x835799D0	0x83FB9580	d3d9:state.obj
+D3DDevice_SetRenderState_HiStencilWriteEnable	0x835799E0	0x83FB9590	d3d9:state.obj
+D3DDevice_GetRenderState_HiStencilWriteEnable	0x83579A00	0x83FB95B0	d3d9:state.obj
+D3DDevice_SetRenderState_HiStencilFunc	0x83579A10	0x83FB95C0	d3d9:state.obj
+D3DDevice_GetRenderState_HiStencilFunc	0x83579A30	0x83FB95E0	d3d9:state.obj
+D3DDevice_SetRenderState_HiStencilRef	0x83579A40	0x83FB95F0	d3d9:state.obj
+D3DDevice_GetSamplerState_AnisotropyBias	0x83579B58	0x83FB9B60	d3d9:state.obj
+D3DDevice_GetSamplerState_MipMapLodBias	0x83579BA0	0x83FB9C08	d3d9:state.obj
+D3DDevice_GetSamplerState_MinMipLevel	0x83579BF0	0x83FB9D38	d3d9:state.obj
+D3DDevice_SetSamplerState_BorderColor	0x83579C00	0x83FB9D48	d3d9:state.obj
+D3DDevice_GetSamplerState_AddressW	0x83579CA0	0x83FB9E90	d3d9:state.obj
+D3DDevice_SetSamplerState_TrilinearThreshold	0x83579CB8	0x83FB9EA8	d3d9:state.obj
+D3DDevice_GetSamplerState_TrilinearThreshold	0x83579CF8	0x83FB9EE8	d3d9:state.obj
+D3DDevice_SetSamplerState_HGradientExpBias	0x83579D10	0x83FB9F00	d3d9:state.obj
+D3DDevice_GetSamplerState_HGradientExpBias	0x83579D50	0x83FB9F40	d3d9:state.obj
+D3DDevice_SetSamplerState_VGradientExpBias	0x83579D68	0x83FB9F58	d3d9:state.obj
+D3DDevice_GetSamplerState_VGradientExpBias	0x83579DA8	0x83FB9F98	d3d9:state.obj
+D3DDevice_SetSamplerState_WhiteBorderColorW	0x83579DC0	0x83FB9FB0	d3d9:state.obj
+D3DDevice_GetSamplerState_WhiteBorderColorW	0x83579E00	0x83FB9FF0	d3d9:state.obj
+D3DDevice_SetSamplerState_PointBorderEnable	0x83579E18	0x83FBA008	d3d9:state.obj
+# ambiguous D3DDevice_SetGammaRamp	0x83579E78	0x83FBA420,0x83FBA4F0	d3d9:state.obj
+# ambiguous D3DDevice_SetPWLGamma	0x83579F48	0x83FBA420,0x83FBA4F0	d3d9:state.obj
+?GetTextureDimensions@D3D@@YAXPBUD3DBaseTexture@@PAK11@Z	0x8357A018	0x83FB3C60	d3d9:texture.obj
+?GetGpuTextureDimensions@D3D@@YAXPBTGPUTEXTURE_FETCH_CONSTANT@@PAK11@Z	0x8357A0B0	0x83FB3CF8	d3d9:texture.obj
+?GetMipTailLevelOffsetCoords@D3D@@YAKKKKKKKKPAI00@Z	0x8357A1F0	0x83FB4250	d3d9:texture.obj
+?GetLevelSize@D3D@@YAXPAUD3DBaseTexture@@KPAK1111@Z	0x8357A348	0x83FB43A8	d3d9:texture.obj
+# ambiguous ?Get2DSurfaceDesc@D3D@@YAXPAUD3DBaseTexture@@IPAU_D3DSURFACE_DESC@@@Z	0x8357A718	0x83FB4778,0x83FB4878	d3d9:texture.obj
+# ambiguous ?Get3DSurfaceDesc@D3D@@YAXPAUD3DBaseTexture@@IPAU_D3DVOLUME_DESC@@@Z	0x8357A818	0x83FB4778,0x83FB4878	d3d9:texture.obj
+?Lock2DSurface@D3D@@YAXPAUD3DBaseTexture@@_KIIPAU_D3DLOCKED_RECT@@PBUtagRECT@@K@Z	0x8357A918	0x83FB4F38	d3d9:texture.obj
+D3DVolumeTexture_GetLevelDesc	0x8357AAA8	0x83FB5568	d3d9:texture.obj
+D3DDevice_CreateTexture	0x8357AAB0	0x83FB5570	d3d9:texture.obj
+D3DSurface_GetDesc	0x8357ABD0	0x83FB57C8	d3d9:texture.obj
+D3DVolume_GetDesc	0x8357AC90	0x83FB5860	d3d9:texture.obj
+?_ComputeDummyValue@@YAKK@Z	0x8357ACD8	0x83FB2448	d3d9:swap.obj
+?IsStereo3DValidationEnabled@@YAHXZ	0x8357ACF0	0x83FB2460	d3d9:swap.obj
+?SetupGamma@D3D@@YAXPAVCDevice@1@W4_D3DFORMAT@@PBU_D3DGAMMARAMP@@H@Z	0x8357AD28	0x83FB2498	d3d9:swap.obj
+?VerticalBlankInterrupt@D3D@@YAXPAVCDevice@1@@Z	0x8357ADA0	0x83FB2510	d3d9:swap.obj
+?SwapCallback@D3D@@YAXK@Z	0x8357AED0	0x83FB2640	d3d9:swap.obj
+?ValidateStereo3DBlankRegion@D3D@@YAXPAVCDevice@1@PAUD3DBaseTexture@@@Z	0x8357B080	0x83FB2898	d3d9:swap.obj
+?ComplexSwap@D3D@@YAXPAVCDevice@1@KKW4_D3DFORMAT@@KKPBU_D3DPRIVATE_SCALER_PARAMETERS@@PBU_D3DGAMMARAMP@@@Z	0x8357B788	0x83FB31B8	d3d9:swap.obj
+?InitVertexDeclarationFromFVF@D3D@@YAXKPAVCVertexDeclaration@1@@Z	0x8357B9A8	0x83FB5A38	d3d9:shader.obj
+?BuildGprAllocationSet@D3D@@YAKPAVCDevice@1@KPAK1@Z	0x8357BFF8	0x83FB6410	d3d9:shader.obj
+XGGetMicrocodeShaderParts	0x8357C068	0x83FB67A0	d3d9:shader.obj
+# ambiguous D3DDevice_GetPixelShader	0x8357C0F0	0x83FB69E8,0x83FB6C00	d3d9:shader.obj
+# ambiguous D3DDevice_GetVertexShader	0x8357C138	0x83FB69E8,0x83FB6C00	d3d9:shader.obj
+XGSetVertexDeclaration	0x8357C180	0x83FB6C60	d3d9:shader.obj
+D3DDevice_CreateVertexDeclaration	0x8357C270	0x83FB6D50	d3d9:shader.obj
+D3DDevice_GetShaderGPRAllocation	0x8357C2F0	0x83FB6EB8	d3d9:shader.obj
+?InitializeGprAllocation@D3D@@YAHPAVCDevice@1@@Z	0x8357C330	0x83FB7310	d3d9:shader.obj
+XGSetPixelShaderHeader	0x8357C460	0x83FB74F0	d3d9:shader.obj
+D3DDevice_CreatePixelShader	0x8357C498	0x83FB7528	d3d9:shader.obj
+XGSetVertexShaderHeader	0x8357C588	0x83FB7618	d3d9:shader.obj
+XGRegisterVertexShader	0x8357C5C8	0x83FB7658	d3d9:shader.obj
+D3DDevice_CreateVertexShader	0x8357C6C0	0x83FB7750	d3d9:shader.obj
+D3DDevice_SetFVF	0x8357C7B0	0x83FB7840	d3d9:shader.obj
+D3DDevice_DrawVertices	0x8357C8A0	0x83FC6640	d3d9:draw.obj
+?SetPendingState_NoInline@D3D@@YAXPAVCDevice@1@W4LazyPrim@1@@Z	0x8357CD60	0x83FBDD20	d3d9:resolve.obj
+?GetExportIdentifier@D3D@@YAKPAUD3DResource@@@Z	0x8357CFC0	0x83FC4BC8	d3d9:resource.obj
+?InvalidateCaches@D3D@@YAXPAVCDevice@1@KK@Z	0x8357D018	0x83FC4C20	d3d9:resource.obj
+D3DDevice_CreateVertexBuffer	0x8357D140	0x83FC4D58	d3d9:resource.obj
+D3DDevice_CreateIndexBuffer	0x8357D2B8	0x83FC4E30	d3d9:resource.obj
+D3DDevice_BeginExport	0x8357D948	0x83FC53D0	d3d9:resource.obj
+D3DDevice_EndExport	0x8357DA40	0x83FC54C8	d3d9:resource.obj
+D3DDevice_InvalidateGpuCache	0x8357DCA8	0x83FC5A58	d3d9:resource.obj
+D3DIndexBuffer_Lock	0x8357DE18	0x83FC6128	d3d9:resource.obj
+?SetPending_Split@D3D@@YAPAKPAVCDevice@1@PIAKK1KK@Z	0x8357E090	0x83FD0948	d3d9:lazy.obj
+?ConfigureForOnePassZPass@D3D@@YAXPAVCDevice@1@HKKK@Z	0x8357E5F8	0x83FBCA98	d3d9:tile.obj
+?RestoreRingBufferState@CDevice@D3D@@QAAXXZ	0x8357E718	0x83FBF188	d3d9:ring.obj
+?AddCommandsToPrimaryBuffer@CDevice@D3D@@QAAXPBKK@Z	0x8357E758	0x83FBF310	d3d9:ring.obj
+?SetPrimaryBufferIndex@D3D@@YAXPAUD3DDevice@@K@Z	0x8357E870	0x83FBF428	d3d9:ring.obj
+?AddAsynchronousInvalidateRange@D3D@@YAXPA_KKK@Z	0x8357E890	0x83FBF448	d3d9:ring.obj
+?GetAsynchronousInvalidateRange@D3D@@YAXPA_KPAK1@Z	0x8357E8F8	0x83FBF4B0	d3d9:ring.obj
+?InterruptCallback@D3D@@YAXKPAVCDevice@1@@Z	0x8357E950	0x83FBF508	d3d9:ring.obj
+D3DResource_BlockUntilNotBusy	0x8357EA30	0x83FBF900	d3d9:ring.obj
+?AppendArrayData@CRingAllocList@D3D@@QAAXPBXKK@Z	0x8357EA90	0x83FC0650	d3d9:ring.obj
+?QueueIndirectBuffer@CDevice@D3D@@QAAPAKPAKKKKPAVCWorkerThreadCommandList@2@H@Z	0x8357EB10	0x83FC06D0	d3d9:ring.obj
+?QueueCommands@CDevice@D3D@@QAAHPBXKKHH@Z	0x8357ECF0	0x83FC0A58	d3d9:ring.obj
+?InvalidateConstants@D3D@@YAXPAUD3DDevice@@@Z	0x8357EEF0	0x83FC0DE0	d3d9:ring.obj
+D3DDevice_BlockUntilIdle	0x8357EF58	0x83FC0EC0	d3d9:ring.obj
+D3DDevice_SetRingBufferParameters	0x8357EFD0	0x83FC0F90	d3d9:ring.obj
+D3DDevice_GetNumPasses	0x8357F418	0x83FAFB08	d3d9:query.obj
+?GetOpenQuerySlot@@YAJPAVCDevice@D3D@@PAPAUCQueryBlock@2@PAPAK@Z	0x8357F460	0x83FAFB50	d3d9:query.obj
+?P_EPC@D3D@@YAXPAUD3DDevice@@H@Z	0x8357F5D0	0x83FAFCE8	d3d9:query.obj
+?P_CPC@D3D@@YAPAUCPerfCounters@1@PAUD3DDevice@@IPAU_D3DPERFCOUNTER_VALUES@@PAU21@@Z	0x8357F6B8	0x83FAFDD0	d3d9:query.obj
+?P_SPCE@D3D@@YAXPAUD3DDevice@@PBU_D3DPERFCOUNTER_EVENTS@@K@Z	0x8357F758	0x83FAFE70	d3d9:query.obj
+# ambiguous ?Destroy@CPerfCounterBatch@D3D@@QAAXXZ	0x8357FD60	0x83E70240,0x83FB0490,0x83FB04D8	d3d9:query.obj
+# ambiguous ?Destroy@COcclusionQueryBatch@D3D@@QAAXXZ	0x8357FDA8	0x83E70240,0x83FB0490,0x83FB04D8	d3d9:query.obj
+?FreeEmptyQueryBlock@@YAXPAVCDevice@D3D@@PAUCQueryBlock@2@@Z	0x8357FDF0	0x83FB0520	d3d9:query.obj
+D3DQuery_Release	0x8357FE80	0x83FB05B0	d3d9:query.obj
+D3DDevice_CreateQueryTiled	0x83580008	0x83FB2360	d3d9:query.obj
+?Destroy@CAsyncCall@D3D@@QAAXXZ	0x835803F8	0x83FC6ED0	d3d9:commands.obj
+?CommandBufferDeviceAllocate@D3D@@YAPAXPAVCDevice@1@KPAKK@Z	0x83580460	0x83FC6F38	d3d9:commands.obj
+?FlushCachedAllocation@CCommandBuffer@D3D@@QAAXXZ	0x83580510	0x83FC6FE8	d3d9:commands.obj
+?Destroy@CCommandBuffer@D3D@@QAAXXZ	0x83580570	0x83FC72A0	d3d9:commands.obj
+?ReadMMRegUlong@D3D@@YAKK@Z	0x83581130	0x83FC8358	d3d9:debug.obj
+?WriteMMRegUlong@D3D@@YAXKK@Z	0x83581140	0x83FC8368	d3d9:debug.obj
+# ambiguous ?Out@Hang@D3D@@QAAXPBDZZ	0x83581158	0x82AB10C0,0x83FC8380	d3d9:debug.obj
+?HangStateDump@D3D@@YAXPAVHang@1@@Z	0x835811E0	0x83FC8408	d3d9:debug.obj
+?HangTextureDump@D3D@@YAXP6AKK@ZP6AXKK@ZKKKPATGPUTEXTURE_FETCH_CONSTANT@@@Z	0x83581448	0x83FC8670	d3d9:debug.obj
+?HangCommandBufferDump@D3D@@YAXPAVHang@1@KH@Z	0x83581508	0x83FC8730	d3d9:debug.obj
+?HangRecognizer@D3D@@YAXPAUD3DDevice@@P6AXPBD@ZP6AKK@ZP6AXKK@Z@Z	0x835816E8	0x83FC8910	d3d9:debug.obj
+?DebugGpuDeadlock@D3D@@YAHPAVCDevice@1@@Z	0x83581FA8	0x83FC91D0	d3d9:debug.obj
+?SetAllSignals@D3D@@YAXPAVCDevice@1@@Z	0x83582068	0x83FC9290	d3d9:initialize.obj
+?Set3DVideoFormat@D3D@@YAKK@Z	0x83582140	0x83FC9368	d3d9:initialize.obj
+?DriverCleanup@D3D@@YAXPAX@Z	0x835821C8	0x83FC93F0	d3d9:initialize.obj
+?D3DHangAvoidanceNotification@D3D@@YAXPAU_EX_TITLE_TERMINATE_REGISTRATION@@@Z	0x835821E8	0x83FC9410	d3d9:initialize.obj
+?InitializeApiState@D3D@@YAXPAVCDevice@1@@Z	0x83582228	0x83FC9450	d3d9:initialize.obj
+?UninitializePresentationParameters@D3D@@YAXPAVCDevice@1@@Z	0x83582490	0x83FC96B8	d3d9:initialize.obj
+?DeleteQueryBlock@D3D@@YAXPAUCQueryBlock@1@@Z	0x83582540	0x83FC9768	d3d9:initialize.obj
+?CreateStereo3DXamBuffer@D3D@@YAHPAVCDevice@1@@Z	0x83582598	0x83FC97C0	d3d9:initialize.obj
+?DestroyStereo3DXamBuffer@D3D@@YAXPAVCDevice@1@@Z	0x83582610	0x83FC9838	d3d9:initialize.obj
+?VideoModeInitialize@D3D@@YAXPAVCDevice@1@PBU_D3DPRESENT_PARAMETERS_@@H@Z	0x83582678	0x83FC98A0	d3d9:initialize.obj
+?D3DTerminateNotification@D3D@@YAXPAU_EX_TITLE_TERMINATE_REGISTRATION@@@Z	0x835827F8	0x83FC9A20	d3d9:initialize.obj
+?DriverInitialize@D3D@@YAHPAVCDevice@1@@Z	0x83582848	0x83FC9A70	d3d9:initialize.obj
+?InitializeCommandBufferDevice@D3D@@YAHPAVCDevice@1@@Z	0x83582950	0x83FC9B78	d3d9:initialize.obj
+?FreePerfCounterMemory@D3D@@YAXPAUD3DDevice@@@Z	0x835829A0	0x83FC9BC8	d3d9:initialize.obj
+?UninitializeDevice@D3D@@YAXPAVCDevice@1@@Z	0x83582A20	0x83FC9C48	d3d9:initialize.obj
+?InitializePresentationParameters@D3D@@YAHPAVCDevice@1@PBU_D3DPRESENT_PARAMETERS_@@@Z	0x83582BF8	0x83FC9E20	d3d9:initialize.obj
+?InitializeHardwareDevice@D3D@@YAHPAVCDevice@1@PBU_D3DPRESENT_PARAMETERS_@@@Z	0x83582E78	0x83FCA0A0	d3d9:initialize.obj
+?AdjustPWLGamma@D3D@@YAXPAU_D3DPWLGAMMAENTRY@@PBU2@PAG@Z	0x835830F8	0x83FCC0C8	d3d9:gamma.obj
+?Gamma_sRGB@D3D@@YAMMH@Z	0x83583148	0x83FCC118	d3d9:gamma.obj
+?Gamma_TV@D3D@@YAMMH@Z	0x835831F8	0x83FCC1C8	d3d9:gamma.obj
+?GammaRampCorrected@D3D@@YAXPAU_D3DGAMMARAMP@@PBU2@H@Z	0x83583430	0x83FCC400	d3d9:gamma.obj
+?GammaPWLCorrected@D3D@@YAXPAU_D3DPWLGAMMA@@PBU2@H@Z	0x835834C8	0x83FCC498	d3d9:gamma.obj
+?SetRawGammaRamp@D3D@@YAXPAVCDevice@1@PBU_D3DGAMMARAMP@@@Z	0x83583550	0x83FCC520	d3d9:gamma.obj
+?SetRawPWLGamma@D3D@@YAXPAVCDevice@1@PBU_D3DPWLGAMMA@@@Z	0x83583640	0x83FCC610	d3d9:gamma.obj
+?ResetGamma@D3D@@YAXPAU_D3DGAMMARAMP@@H@Z	0x83583770	0x83FCC740	d3d9:gamma.obj
+?ResetGamma@D3D@@YAXPAU_D3DPWLGAMMA@@H@Z	0x835837B8	0x83FCC788	d3d9:gamma.obj
+?FindFreeChunk@D3D@@YAKPAUEdram@1@PAK@Z	0x83583820	0x83FD0570	d3d9:allocate.obj
+?MarkBits@D3D@@YAXPAUEdram@1@KKH@Z	0x835838F8	0x83FD0648	d3d9:allocate.obj
+?FreeEdramMemory@D3D@@YAXKK@Z	0x83583A38	0x83FD0788	d3d9:allocate.obj
+?AllocateEdramMemory@D3D@@YAHKPAK@Z	0x83583A88	0x83FD07D8	d3d9:allocate.obj
+?ConvertPWLGammaToGammaRamp@D3D@@YAXPAU_D3DGAMMARAMP@@PBU_D3DPWLGAMMA@@@Z	0x83583BF8	0x83FC1358	d3d9:pix.obj
+?SetAllBlackGamma@D3D@@YAXPAVCDevice@1@@Z	0x83583C68	0x83FC13C8	d3d9:pix.obj
+?PixEnableVideoPlayback@D3D@@YAKPAX@Z	0x83583CC0	0x83FC1420	d3d9:pix.obj
+?PixDisableVideoPlayback@D3D@@YAXPAX@Z	0x83583D48	0x83FC14A8	d3d9:pix.obj
+?PixEstimateVideoCost@D3D@@YAXPAVCDevice@1@PAUPIX_ESTIMATE_VIDEO_COST@@@Z	0x83583DB0	0x83FC1510	d3d9:pix.obj
+?CommandHandler@D3D@@YAJPBDPADKPAU_DM_CMDCONT@@@Z	0x83583F38	0x83FC1718	d3d9:pix.obj
+PIXCaptureGpuFrame	0x83583FD8	0x83FC17B8	d3d9:pix.obj
+?PixTimeSinceVBlank@D3D@@YAKPAX@Z	0x835843A0	0x83FC1B80	d3d9:pix.obj
+?CodecSetState@D3D@@YAXPAVCDevice@1@@Z	0x835843E8	0x83FC1BC8	d3d9:pix.obj
+?FrameDone@D3D@@YAXK@Z	0x83584540	0x83FC1D20	d3d9:pix.obj
+?GetCounter@D3D@@YAMPAVCDevice@1@W4_D3DCOUNTER@@@Z	0x83584A30	0x83FC2320	d3d9:pix.obj
+?BlockUntilIdleCallback@D3D@@YAXPAX@Z	0x83584F10	0x83FC2800	d3d9:pix.obj
+?CounterHandler@D3D@@YAXKK@Z	0x83584F18	0x83FC2808	d3d9:pix.obj
+?InitXBDMInterface@D3D@@YAXPAVCDevice@1@@Z	0x835850F8	0x83FC29E8	d3d9:pix.obj
+?UninitXBDMInterface@D3D@@YAXPAVCDevice@1@@Z	0x835851F8	0x83FC2AE8	d3d9:pix.obj
+?PixInitializePerfMemory@D3D@@YA_NPAVCDevice@1@@Z	0x835852B0	0x83FC2BA0	d3d9:pix.obj
+?CodecShowFrame@D3D@@YAHPAVCDevice@1@KPAPAUPIX_CHUNK@@KKK@Z	0x83585350	0x83FC2C80	d3d9:pix.obj
+?CodecVideoEncode@D3D@@YAXPAVCDevice@1@KPAPAUPIX_CHUNK@@KKPAUD3DTexture@@QBM33K@Z	0x83585B08	0x83FC3438	d3d9:pix.obj
+?PixDecodeVideoFrame@D3D@@YAKPAXKPAPAUPIX_CHUNK@@KKPAUPIX_DECODE_VIDEO_DATA@@K@Z	0x83585FA0	0x83FC38D0	d3d9:pix.obj
+?BuildVideoRecord@D3D@@YAKPAVCDevice@1@KW4_D3DFORMAT@@KKPBU_D3DPRIVATE_SCALER_PARAMETERS@@PBU_D3DGAMMARAMP@@@Z	0x835862F0	0x83FC3C20	d3d9:pix.obj
+?InitializeDefaultScalerParameters@D3D@@YAXPAVCDevice@1@KKPBU_D3DPRIVATE_SCALER_PARAMETERS@@PAU3@H@Z	0x83586A08	0x83FCC7F0	d3d9:scaler.obj
+?ConvertScalerParametersInternal@D3D@@YAXPAVCDevice@1@PBU_D3DVIDEO_SCALER_PARAMETERS@@PAU_D3DPRIVATE_SCALER_PARAMETERS@@@Z	0x83586B30	0x83FCC918	d3d9:scaler.obj
+?SendScalerNotification@D3D@@YAXPAVCDevice@1@KKPBU_D3DPRIVATE_SCALER_PARAMETERS@@@Z	0x83586BA8	0x83FCC990	d3d9:scaler.obj
+?coef_f2c@D3D@@YAXPAMPAIHH@Z	0x83586F78	0x83FCCD60	d3d9:scaler.obj
+?Bessi0@D3D@@YAMM@Z	0x835873D0	0x83FCD1B8	d3d9:scaler.obj
+?KaiserWindow@D3D@@YAMMMM@Z	0x83587578	0x83FCD360	d3d9:scaler.obj
+?sinc@D3D@@YAMM@Z	0x83587638	0x83FCD420	d3d9:scaler.obj
+# ambiguous ?mitchell@D3D@@YAMM@Z	0x83587690	0x83FCD478,0x83FCD580	d3d9:scaler.obj
+# ambiguous ?catmull@D3D@@YAMM@Z	0x83587798	0x83FCD478,0x83FCD580	d3d9:scaler.obj
+?GenerateFilterCoefficients@D3D@@YAXW4_D3DFILTER_TYPE@@KKHMMPAM@Z	0x835878A0	0x83FCD688	d3d9:scaler.obj
+?BuildFilter@D3D@@YAXPAU_FILTER@@HW4_D3DFILTER_TYPE@@PBU_D3DFILTER_PARAMETERS@@MIH@Z	0x83587CD8	0x83FCDAC0	d3d9:scaler.obj
+?InternalScalerSetup@D3D@@YAHPAVCDevice@1@KKKKKAAW4_D3DFILTER_TYPE@@PBU_D3DFILTER_PARAMETERS@@12PAU_HARDWARE_SCALER_PARAMETERS@@@Z	0x83587FB8	0x83FCDDA0	d3d9:scaler.obj
+?SetupScaler@D3D@@YAXPAVCDevice@1@KKPBU_D3DPRIVATE_SCALER_PARAMETERS@@H@Z	0x83588548	0x83FCE330	d3d9:scaler.obj
+D3DDevice_SetScalerParametersInternal	0x835886F8	0x83FCE4E0	d3d9:scaler.obj
+?LowPriSetGprAllocation@D3D@@YAXPAVCXpsGlobal@1@PAVCDevice@1@K@Z	0x835887A8	0x83FC72F8	d3d9:lowpri.obj
+?FinishExecutingLowPriSubmission@D3D@@YAXPAVCXpsGlobal@1@PAVCDevice@1@K@Z	0x83588888	0x83FC73D8	d3d9:lowpri.obj
+?LowPriPrbWritePerfCounter@D3D@@YAXPAVCDevice@1@PCKK@Z	0x83588900	0x83FC7450	d3d9:lowpri.obj
+?D3DDevice_InsertYieldPoint@@YAJPAUD3DDevice@@KK@Z	0x83588A30	0x83FC7580	d3d9:lowpri.obj
+D3DDevice_ResetLowPriorityQueues	0x83588B80	0x83FC76D0	d3d9:lowpri.obj
+?LowPriReportGpuDoneWithYield@D3D@@YAXPAVCDevice@1@@Z	0x83588D08	0x83FC7858	d3d9:lowpri.obj
+?LowPriDoneInterruptCallback@D3D@@YAXK@Z	0x83588E00	0x83FC7950	d3d9:lowpri.obj
+?ProcessLowPriBlockOnFence@D3D@@YA_NPAVCDevice@1@K@Z	0x83588E50	0x83FC79A0	d3d9:lowpri.obj
+?LowPriPrbWriteTimestamp@D3D@@YAXPAVCDevice@1@PCK@Z	0x83588EC0	0x83FC7A10	d3d9:lowpri.obj
+?ExecuteLowPriCommandBuffer@D3D@@YAXPAVCXpsGlobal@1@PAVCDevice@1@KPAULowPrioritySubmission@@@Z	0x83588F28	0x83FC7A78	d3d9:lowpri.obj
+?ExecutePendingSave@D3D@@YAXPAVCXpsGlobal@1@PAVCDevice@1@K@Z	0x83589018	0x83FC7B68	d3d9:lowpri.obj
+?ExecuteLowPriSubmission@D3D@@YAXPAVCXpsGlobal@1@PAVCDevice@1@KPAK@Z	0x835890B8	0x83FC7C08	d3d9:lowpri.obj
+?HandleRequiredLowPriWork@D3D@@YAXPAVCXpsGlobal@1@PAVCDevice@1@PAUQueueRecord_YieldPointParams@1@@Z	0x83589370	0x83FC7EC0	d3d9:lowpri.obj
+?HandleLowPriWork@D3D@@YAPAKPAVCXpsGlobal@1@PAUQueueRecord_YieldPointParams@1@@Z	0x83589478	0x83FC7FC8	d3d9:lowpri.obj
+?XBMHandleConnect@D3D@@YAHPBD@Z	0x83589808	0x83FCE690	d3d9:movie.obj
+?XBMHandleDisconnect@D3D@@YAHPBD@Z	0x83589820	0x83FCE6A8	d3d9:movie.obj
+?XBMHandleVersionCheck@D3D@@YAHPBD@Z	0x83589838	0x83FCE6C0	d3d9:movie.obj
+?XBMHandleLimitCaptureSize@D3D@@YAHPBD@Z	0x83589870	0x83FCE6F8	d3d9:movie.obj
+?XBMHandleBeginCaptureFileCreation@D3D@@YAHPBD@Z	0x835898C8	0x83FCE750	d3d9:movie.obj
+?XBMHandleBeginCapture@D3D@@YAHPBD@Z	0x83589910	0x83FCE798	d3d9:movie.obj
+?XBMHandleEndCapture@D3D@@YAHPBD@Z	0x83589928	0x83FCE7B0	d3d9:movie.obj
+?XBMInitialize@D3D@@YAHPAVCDevice@1@@Z	0x83589940	0x83FCE7C8	d3d9:movie.obj
+?XBMProcessCommand@D3D@@YAJPBDPADKPAU_DM_CMDCONT@@@Z	0x83589968	0x83FCE7F0	d3d9:movie.obj
+?XBMAllocateFrameMemory@D3D@@YAXIPAXPAPAXPAW4MEMORYREGION@1@@Z	0x83589C20	0x83FCEAA8	d3d9:movie.obj
+?XBMDestroyContext@D3D@@YAXPAVCDevice@1@@Z	0x83589CA0	0x83FCEB28	d3d9:movie.obj
+?XBMCreateContext@D3D@@YAJPAVCDevice@1@PAUD3DBaseTexture@@PBD@Z	0x83589DA8	0x83FCEC30	d3d9:movie.obj
+?XBMEndCapture@D3D@@YAXPAVCDevice@1@@Z	0x8358A578	0x83FCF400	d3d9:movie.obj
+?XBMUninitialize@D3D@@YAXPAVCDevice@1@@Z	0x8358A998	0x83FCF820	d3d9:movie.obj
+?XBMBeginCapture@D3D@@YAHPAVCDevice@1@PAUD3DBaseTexture@@PBD@Z	0x8358A9C8	0x83FCF850	d3d9:movie.obj
+?XBMCaptureFrame@D3D@@YAXPAVCDevice@1@PAUD3DBaseTexture@@@Z	0x8358AD30	0x83FCFBB8	d3d9:movie.obj
+?LockL2@D3D@@YAXPAVCXpsGlobal@1@K@Z	0x8358B5F8	0x83FCA5C0	d3d9:xps.obj
+?CalculatePredicationSelect@D3D@@YAKPAVCXpsGlobal@1@@Z	0x8358B6B0	0x83FCA678	d3d9:xps.obj
+?UpdatePredicationSelect@D3D@@YAXPAVCXpsGlobal@1@@Z	0x8358B718	0x83FCA6E0	d3d9:xps.obj
+?WaitForXpsIdle@D3D@@YAXPAVCXpsGlobal@1@@Z	0x8358B7A0	0x83FCA768	d3d9:xps.obj
+?AutoPatch@D3D@@YAXPAUWorkerState@1@PAVCPatchListChunk@1@H@Z	0x8358B840	0x83FCA808	d3d9:xps.obj
+?DispatchCompletion@D3D@@YAPAKPAVCXpsGlobal@1@PAUD3DXpsThread@@PCK@Z	0x8358B948	0x83FCA910	d3d9:xps.obj
+?KickOffWorkerThreadDpc@D3D@@YAXPAU_KDPC@@PAX11@Z	0x8358BA80	0x83FCAA48	d3d9:xps.obj
+?WorkerDpc@D3D@@YAXK@Z	0x8358BAB8	0x83FCAA80	d3d9:xps.obj
+?UninitializeXps@D3D@@YAXPAVCDevice@1@@Z	0x8358BB48	0x83FCAC80	d3d9:xps.obj
+?XpsConfigureForWork@D3D@@YAXPAVCDevice@1@PAVCXpsGlobal@1@K@Z	0x8358BBE8	0x83FCAD20	d3d9:xps.obj
+?XpsVisibilityPatch@D3D@@YAXPBVCXpsVisibilityPatchListChunk@1@@Z	0x8358BD90	0x83FCAEC8	d3d9:xps.obj
+?XpsQueryPatch@D3D@@YAXPAUWorkerState@1@PAVCXpsQueryPatchListChunk@1@@Z	0x8358BE70	0x83FCAFA8	d3d9:xps.obj
+?XpsAutoPatch@D3D@@YAXPAUWorkerState@1@PAVCXpsPatchListChunk@1@@Z	0x8358BFB0	0x83FCB0E8	d3d9:xps.obj
+?TilingPatch@D3D@@YAXPAUWorkerState@1@PAVCPatchListChunk@1@PAVCCommandBufferCallListChunk@1@@Z	0x8358C0A8	0x83FCB1E0	d3d9:xps.obj
+?DispatchCommands@D3D@@YAPAKPAKPAVCXpsGlobal@1@@Z	0x8358C160	0x83FCB298	d3d9:xps.obj
+?DispatchWorker@D3D@@YAXPAUD3DXpsThread@@@Z	0x8358C4D0	0x83FCB608	d3d9:xps.obj
+?WorkerThread@D3D@@YAKPAX@Z	0x8358C790	0x83FCB8C8	d3d9:xps.obj
+?KickOffOnWorkerQueue@D3D@@YAXPAVCDevice@1@KKPAVCWorkerThreadCommandList@1@W4Queue_Commands@1@@Z	0x8358C930	0x83FCBC50	d3d9:xps.obj
+?InitializeXps@D3D@@YAHPAVCDevice@1@@Z	0x8358C9E8	0x83FCBD08	d3d9:xps.obj
+?XenonGpu_Initialize@D3D@@YAXPAVCDevice@1@@Z	0x8358CBE0	0x83FCA320	d3d9:xenongpu.obj
+?PixNullHardwareInterruptCallback@D3D@@YAXPAU_KDPC@@PAX11@Z	0x8358CDD8	0x83FD2A38	d3d9:capture.obj
+?CleanupGprAllocationSetup@D3D@@YAXPAVCDevice@1@@Z	0x8357BEE0	0x83FB62F8	d3d9:shader.obj
+?DoBeginExportNanVertex@@YAXPAUD3DDevice@@@Z	0x8357D468	0x83FC4EF0	d3d9:resource.obj
+?DoEndExport512Pixels@@YAXPAUD3DDevice@@@Z	0x8357D5C0	0x83FC5048	d3d9:resource.obj
+?VertexShaderStubOutput@D3D@@YAXPBU_UCODE_R500VSUCODE@@PATGPUSHADER_INSTRUCTION@@PBT_UCODE_VSOUTPUTDECLARATION@@@Z	0x8357E148	0x83FD1A98	d3d9:lazy.obj
+?VertexShaderRedirectOutput@D3D@@YAXPBU_UCODE_R500VSUCODE@@PATGPUSHADER_INSTRUCTION@@PBT_UCODE_VSOUTPUTDECLARATION@@PBT_UCODE_PSINPUTDECLARATION@@@Z	0x8357E218	0x83FD1B68	d3d9:lazy.obj
+?PatchVertexShaderToMatchPixelShader@D3D@@YAXPAUPIXPatchVertexShaderToMatchPixelShaderData@@PBU_UCODE_R500VSUCODE@@PATGPUSHADER_INSTRUCTION@@PATGPU_PROGRAMCONTROL@@PAU_UCODE_R500PSUCODE@@@Z	0x8357E2D0	0x83FD1C20	d3d9:lazy.obj
+?GetCurrentGammaTable@D3D@@YAPAGH@Z	0x835832A8	0x83FCC278	d3d9:gamma.obj
+?ShortPercentClamp@D3D@@YAK_J0@Z	0x83583ED0	0x83FC16B0	d3d9:pix.obj
+?SubmitGeneralCounters@D3D@@YAXPAVCDevice@1@@Z	0x835845E0	0x83FC1ED0	d3d9:pix.obj
+# ambiguous ??__Eg_RGBScaledToYUV601@D3D@@YAXXZ	0x8394BB88	0x840EBF88,0x840EC020,0x840EC0B8,0x840EC150	d3d9:movie.obj
+# ambiguous ??__Eg_RGBScaledToYUV709@D3D@@YAXXZ	0x8394BC20	0x840EBF88,0x840EC020,0x840EC0B8,0x840EC150	d3d9:movie.obj
+# ambiguous ??__Eg_RGBOneNinthScaledToYUV601@D3D@@YAXXZ	0x8394BCB8	0x840EBF88,0x840EC020,0x840EC0B8,0x840EC150	d3d9:movie.obj
+# ambiguous ??__Eg_RGBOneNinthScaledToYUV709@D3D@@YAXXZ	0x8394BD50	0x840EBF88,0x840EC020,0x840EC0B8,0x840EC150	d3d9:movie.obj
+# matched=370 ambiguous=49 missing=211
diff --git a/tools/gpu-aot/match_map_symbols.py b/tools/gpu-aot/match_map_symbols.py
new file mode 100644
index 0000000..e7b3d73
--- /dev/null
+++ b/tools/gpu-aot/match_map_symbols.py
@@ -0,0 +1,119 @@
+"""Match statically linked Xbox 360 SDK functions from a MAP-labelled image.
+
+The matcher deliberately ignores relocated immediates and branch displacements
+while retaining instruction opcodes and register fields. This makes a function
+compiled from the same XDK library comparable after the linker moves code and
+data to different guest addresses.
+"""
+
+from __future__ import annotations
+
+import argparse
+import pathlib
+import re
+import struct
+
+BASE = 0x82000000
+MAP_FUNCTION = re.compile(
+    r"^\s*\S+\s+(?P<name>\S+)\s+(?P<address>[0-9A-Fa-f]{8})\s+f\s+(?P<object>\S+)"
+)
+
+
+def normalize(word: int) -> int:
+    opcode = word >> 26
+    # Relative branches: target displacement is link-position dependent.
+    if opcode == 18:
+        return word & 0xFC000003
+    if opcode == 16:
+        return word & 0xFFFF0003
+    # Immediate arithmetic, compare, logical and D-form load/store operations.
+    if opcode in {
+        7, 8, 10, 11, 12, 13, 14, 15,
+        24, 25, 26, 27, 28, 29,
+        *range(32, 56),
+    }:
+        return word & 0xFFFF0000
+    return word
+
+
+def normalized_words(image: bytes) -> list[int]:
+    usable = len(image) & ~3
+    return [normalize(word[0]) for word in struct.iter_unpack(">I", image[:usable])]
+
+
+def read_sdk_symbols(path: pathlib.Path) -> list[tuple[str, int, str]]:
+    result = []
+    with path.open("r", encoding="latin-1", errors="replace") as stream:
+        for line in stream:
+            match = MAP_FUNCTION.match(line)
+            if match and match.group("object").lower().startswith("d3d9:"):
+                result.append(
+                    (match.group("name"), int(match.group("address"), 16), match.group("object"))
+                )
+    return result
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--map", required=True, type=pathlib.Path)
+    parser.add_argument("--reference", required=True, type=pathlib.Path)
+    parser.add_argument("--target", required=True, type=pathlib.Path)
+    parser.add_argument("--words", type=int, default=16)
+    parser.add_argument("--filter", default="")
+    parser.add_argument("--output", type=pathlib.Path)
+    args = parser.parse_args()
+
+    reference = normalized_words(args.reference.read_bytes())
+    target = normalized_words(args.target.read_bytes())
+    width = args.words
+    if width < 4:
+        parser.error("--words must be at least 4")
+
+    selected = re.compile(args.filter, re.IGNORECASE) if args.filter else None
+    symbols = []
+    signatures: dict[int, dict[tuple[int, ...], list[int]]] = {}
+    for name, address, obj in read_sdk_symbols(args.map):
+        if selected and not selected.search(name):
+            continue
+        offset = (address - BASE) // 4
+        if offset < 0 or offset + width > len(reference):
+            continue
+        signature = tuple(reference[offset : offset + width])
+        symbol_index = len(symbols)
+        symbols.append((name, address, obj, signature))
+        by_signature = signatures.setdefault(signature[0], {})
+        by_signature.setdefault(signature, []).append(symbol_index)
+
+    hits: list[list[int]] = [[] for _ in symbols]
+    for i in range(len(target) - width + 1):
+        by_signature = signatures.get(target[i])
+        if not by_signature:
+            continue
+        signature = tuple(target[i : i + width])
+        for symbol_index in by_signature.get(signature, ()):
+            hits[symbol_index].append(BASE + i * 4)
+
+    lines = []
+    matched = ambiguous = missing = 0
+    for (name, address, obj, _), symbol_hits in zip(symbols, hits):
+        if len(symbol_hits) == 1:
+            matched += 1
+            lines.append(f"{name}\t0x{address:08X}\t0x{symbol_hits[0]:08X}\t{obj}")
+        elif len(symbol_hits) > 1:
+            ambiguous += 1
+            joined = ",".join(f"0x{hit:08X}" for hit in symbol_hits[:8])
+            lines.append(f"# ambiguous {name}\t0x{address:08X}\t{joined}\t{obj}")
+        else:
+            missing += 1
+
+    lines.append(f"# matched={matched} ambiguous={ambiguous} missing={missing}")
+    text = "\n".join(lines) + "\n"
+    if args.output:
+        args.output.write_text(text, encoding="utf-8")
+    else:
+        print(text, end="")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From 879eaa635d21cbaba7ab9b89d073dfd0baf3f9b0 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 14:27:55 +0300
Subject: [PATCH 06/44] gpu: add native shader resource lifecycle

---
 rexlego/CMakeLists.txt                  |  41 +++++
 rexlego/src/gpu_native/build_info.h.in  |   6 +
 rexlego/src/gpu_native/d3d.h            |  32 +++-
 rexlego/src/gpu_native/device.cpp       |   9 +-
 rexlego/src/gpu_native/hooks_shader.cpp |  63 +++++++
 rexlego/src/gpu_native/shaders.cpp      | 215 ++++++++++++++++++++++++
 rexlego/src/gpu_native/shaders.h        |  37 ++++
 7 files changed, 400 insertions(+), 3 deletions(-)
 create mode 100644 rexlego/src/gpu_native/build_info.h.in
 create mode 100644 rexlego/src/gpu_native/hooks_shader.cpp
 create mode 100644 rexlego/src/gpu_native/shaders.cpp
 create mode 100644 rexlego/src/gpu_native/shaders.h

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index 728d76e..ac5c082 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -57,7 +57,9 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     list(APPEND LEGODIMENSIONS_SOURCES
         src/gpu_native/device.cpp
         src/gpu_native/hooks_device.cpp
+        src/gpu_native/hooks_shader.cpp
         src/gpu_native/shader_archive.cpp
+        src/gpu_native/shaders.cpp
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c"
     )
@@ -148,6 +150,45 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     target_sources(legodimensions PRIVATE "${LEGO_NATIVE_LINKED_SHADER_CACHE}")
     set_source_files_properties("${LEGO_NATIVE_LINKED_SHADER_CACHE}"
         PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+
+    # Fingerprint the exact native renderer inputs. CMAKE_CONFIGURE_DEPENDS
+    # makes a header-only edit trigger reconfigure as well as recompilation,
+    # and the fingerprint printed at startup proves which executable is live.
+    set(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/d3d.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_device.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_shader.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shader_archive.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shader_archive.h"
+        "${LEGO_NATIVE_SHADER_CACHE}"
+        "${LEGO_NATIVE_LINKED_SHADER_CACHE}")
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
+        ${LEGO_NATIVE_GPU_FINGERPRINT_INPUTS})
+    set(LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL "")
+    foreach(LEGO_NATIVE_GPU_INPUT IN LISTS LEGO_NATIVE_GPU_FINGERPRINT_INPUTS)
+        file(SHA256 "${LEGO_NATIVE_GPU_INPUT}" LEGO_NATIVE_GPU_INPUT_HASH)
+        string(APPEND LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL
+            "${LEGO_NATIVE_GPU_INPUT_HASH}")
+    endforeach()
+    string(SHA256 LEGO_NATIVE_GPU_FINGERPRINT_FULL
+        "${LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL}")
+    string(SUBSTRING "${LEGO_NATIVE_GPU_FINGERPRINT_FULL}" 0 16
+        LEGO_NATIVE_GPU_FINGERPRINT)
+    configure_file(src/gpu_native/build_info.h.in
+        "${CMAKE_CURRENT_BINARY_DIR}/generated/native_gpu_build_info.h" @ONLY)
+    target_include_directories(legodimensions PRIVATE
+        "${CMAKE_CURRENT_BINARY_DIR}/generated")
+    file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/native-gpu-build.txt"
+        "LEGO Dimensions native GPU ${LEGO_NATIVE_GPU_FINGERPRINT}\n")
+    add_custom_command(TARGET legodimensions POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "${CMAKE_CURRENT_BINARY_DIR}/native-gpu-build.txt"
+            "$<TARGET_FILE_DIR:legodimensions>/native-gpu-build.txt"
+        VERBATIM)
 endif()
 
 # The achievement unlock sound, converted from the user-supplied MP3 to a quiet
diff --git a/rexlego/src/gpu_native/build_info.h.in b/rexlego/src/gpu_native/build_info.h.in
new file mode 100644
index 0000000..a6503e2
--- /dev/null
+++ b/rexlego/src/gpu_native/build_info.h.in
@@ -0,0 +1,6 @@
+// Generated by CMake from the native renderer source and AOT archive hashes.
+#pragma once
+
+namespace legodimensions::gpu_native {
+inline constexpr char kNativeGpuBuildFingerprint[] = "@LEGO_NATIVE_GPU_FINGERPRINT@";
+}
diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h
index ce1abd3..9de8808 100644
--- a/rexlego/src/gpu_native/d3d.h
+++ b/rexlego/src/gpu_native/d3d.h
@@ -35,6 +35,32 @@ struct FetchConstant {
 };
 static_assert(sizeof(FetchConstant) == 0x18);
 
+// Header shared by Xbox 360 D3D resources. The shader create routines in the
+// Sep'13 runtime initialize shaders with type 6 (vertex) / 7 (pixel), a
+// reference count of one, and 0xFFFF0000 in BaseFlush.
+struct D3DResource {
+  be_u32 common;
+  be_u32 reference_count;
+  be_u32 fence;
+  be_u32 read_fence;
+  be_u32 identifier;
+  be_u32 base_flush;
+};
+static_assert(sizeof(D3DResource) == 0x18);
+
+struct ShaderContainer {
+  be_u32 flags;
+  be_u32 virtual_size;
+  be_u32 physical_size;
+  be_u32 field_0c;
+  be_u32 constant_table_offset;
+  be_u32 definition_table_offset;
+  be_u32 shader_offset;
+  be_u32 field_1c;
+  be_u32 field_20;
+};
+static_assert(sizeof(ShaderContainer) == 0x24);
+
 // Only fields proven from TU23 code are named. Keep unknown spans opaque until
 // an accessor or state writer establishes their meaning.
 struct D3DDevice {
@@ -49,7 +75,9 @@ struct D3DDevice {
   D3DViewport9 viewport;
   be_u32 viewport_reserved;
   D3DRect scissor;
-  u8 unknown_330C[kGuestDeviceSize - 0x330C];
+  be_u32 pixel_shader;
+  be_u32 vertex_shader;
+  u8 unknown_3314[kGuestDeviceSize - 0x3314];
 };
 
 static_assert(sizeof(D3DDevice) == kGuestDeviceSize);
@@ -60,5 +88,7 @@ static_assert(offsetof(D3DDevice, get_sampler_state) == 0x3B8);
 static_assert(offsetof(D3DDevice, fetch_constants) == 0x480);
 static_assert(offsetof(D3DDevice, viewport) == 0x32E0);
 static_assert(offsetof(D3DDevice, scissor) == 0x32FC);
+static_assert(offsetof(D3DDevice, pixel_shader) == 0x330C);
+static_assert(offsetof(D3DDevice, vertex_shader) == 0x3310);
 
 }  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 8e2dc50..b10f76b 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -1,5 +1,8 @@
 #include "gpu_native/device.h"
 
+#include "gpu_native/shaders.h"
+#include "native_gpu_build_info.h"
+
 #include <mutex>
 #include <string>
 
@@ -83,8 +86,9 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
   }
 
   const auto& description = state->device->getDescription();
-  REXLOG_INFO("Native GPU: {} ready on '{}' ({}x{}, {} images)", NameOf(backend),
-              description.name, state->swap_chain->getWidth(), state->swap_chain->getHeight(),
+  REXLOG_INFO("Native GPU [{}]: {} ready on '{}' ({}x{}, {} images)",
+              kNativeGpuBuildFingerprint, NameOf(backend), description.name,
+              state->swap_chain->getWidth(), state->swap_chain->getHeight(),
               state->swap_chain->getTextureCount());
   g_state = std::move(state);
   return true;
@@ -95,6 +99,7 @@ void HostDevice::Shutdown() {
   if (!g_state) {
     return;
   }
+  ResetShaderResources();
   if (g_state->swap_chain) {
     g_state->swap_chain->wait();
   }
diff --git a/rexlego/src/gpu_native/hooks_shader.cpp b/rexlego/src/gpu_native/hooks_shader.cpp
new file mode 100644
index 0000000..3f74579
--- /dev/null
+++ b/rexlego/src/gpu_native/hooks_shader.cpp
@@ -0,0 +1,63 @@
+#include <rex/hook.h>
+#include <rex/ppc/context.h>
+#include <rex/types.h>
+
+#include "gpu_native/d3d.h"
+#include "gpu_native/shaders.h"
+
+extern "C" void __imp__sub_83FC4CE0(PPCContext& __restrict ctx, u8* base);
+extern "C" void __imp__sub_83FC59D8(PPCContext& __restrict ctx, u8* base);
+
+namespace legodimensions::gpu_native {
+namespace {
+
+u32 CreatePixelShaderHook(mapped_u32 function) {
+  return CreateShaderResource(function, ShaderStage::kPixel);
+}
+
+u32 CreateVertexShaderHook(mapped_u32 function) {
+  return CreateShaderResource(function, ShaderStage::kVertex);
+}
+
+void SetPixelShaderHook(D3DDevice* device, u32 shader) {
+  if (device) {
+    device->pixel_shader = shader;
+  }
+  BindShader(ShaderStage::kPixel, shader);
+}
+
+void SetVertexShaderHook(D3DDevice* device, u32 shader) {
+  if (device) {
+    device->vertex_shader = shader;
+  }
+  BindShader(ShaderStage::kVertex, shader);
+}
+
+}  // namespace
+}  // namespace legodimensions::gpu_native
+
+REX_HOOK(sub_83FB7528, legodimensions::gpu_native::CreatePixelShaderHook);
+REX_HOOK(sub_83FB7750, legodimensions::gpu_native::CreateVertexShaderHook);
+REX_HOOK(sub_83FB6828, legodimensions::gpu_native::SetPixelShaderHook);
+REX_HOOK(sub_83FB6A30, legodimensions::gpu_native::SetVertexShaderHook);
+
+// Resource AddRef/Release serve all D3D resource types. Only intercept objects
+// created by the native shader hooks; every other pointer retains the exact
+// Sep'13 implementation.
+REX_HOOK_RAW(sub_83FC4CE0) {
+  const u32 resource = ctx.r3.u32;
+  if (!legodimensions::gpu_native::IsNativeShader(resource)) {
+    __imp__sub_83FC4CE0(ctx, base);
+    return;
+  }
+  ctx.r3.u64 = legodimensions::gpu_native::AddRefNativeShader(resource);
+}
+
+REX_HOOK_RAW(sub_83FC59D8) {
+  const u32 resource = ctx.r3.u32;
+  if (!legodimensions::gpu_native::IsNativeShader(resource)) {
+    __imp__sub_83FC59D8(ctx, base);
+    return;
+  }
+  ctx.r3.u64 = legodimensions::gpu_native::ReleaseNativeShader(resource);
+}
diff --git a/rexlego/src/gpu_native/shaders.cpp b/rexlego/src/gpu_native/shaders.cpp
new file mode 100644
index 0000000..658802e
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders.cpp
@@ -0,0 +1,215 @@
+#include "gpu_native/shaders.h"
+
+#include <array>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <plume_render_interface.h>
+#include <rex/logging.h>
+#include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
+
+#include "gpu_native/d3d.h"
+#include "gpu_native/device.h"
+#include "gpu_native/shader_archive.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+constexpr u32 kPixelShaderGuestSize = 40;
+constexpr u32 kVertexShaderGuestSize = 872;
+constexpr u32 kShaderFlag = 0x00100000;
+
+struct ShaderResource {
+  u32 guest_address = 0;
+  ShaderStage stage = ShaderStage::kVertex;
+  u64 hash = 0;
+  const ShaderCacheEntry* cache_entry = nullptr;
+  std::mutex shader_mutex;
+  std::unordered_map<u32, std::unique_ptr<plume::RenderShader>> variants;
+};
+
+std::mutex g_registry_mutex;
+std::unordered_map<u32, std::shared_ptr<ShaderResource>> g_registry;
+std::array<std::shared_ptr<ShaderResource>, 2> g_bound_shaders;
+
+size_t StageIndex(ShaderStage stage) {
+  return stage == ShaderStage::kVertex ? 0 : 1;
+}
+
+std::shared_ptr<ShaderResource> FindResource(u32 guest_address) {
+  std::lock_guard lock(g_registry_mutex);
+  const auto it = g_registry.find(guest_address);
+  return it == g_registry.end() ? nullptr : it->second;
+}
+
+D3DResource* GuestHeader(u32 guest_address) {
+  auto* memory = REX_KERNEL_MEMORY();
+  return reinterpret_cast<D3DResource*>(memory->virtual_membase() + guest_address);
+}
+
+}  // namespace
+
+u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) {
+  if (!function || !HostDevice::IsReady()) {
+    return 0;
+  }
+
+  const auto* container = reinterpret_cast<const ShaderContainer*>(function.host_address());
+  const size_t byte_length = ShaderContainerByteLength(container);
+  // Corrupt sizes must never turn a guest pointer into an unbounded host read.
+  if (byte_length < sizeof(ShaderContainer) || byte_length > 4 * 1024 * 1024) {
+    REXLOG_ERROR("Native GPU: rejected {} shader container of {} bytes",
+                 stage == ShaderStage::kVertex ? "vertex" : "pixel", byte_length);
+    return 0;
+  }
+
+  const u64 hash = HashShaderContainer(container);
+  const ShaderCacheEntry* cache_entry = FindShader(hash);
+  if (!cache_entry) {
+    REXLOG_WARN("Native GPU: shader not present in AOT cache, hash=0x{:016X}", hash);
+  }
+
+  auto* memory = REX_KERNEL_MEMORY();
+  const u32 allocation_size =
+      stage == ShaderStage::kVertex ? kVertexShaderGuestSize : kPixelShaderGuestSize;
+  const u32 guest_address = memory->SystemHeapAlloc(allocation_size, 0x10);
+  if (!guest_address) {
+    return 0;
+  }
+  memory->Zero(guest_address, allocation_size);
+
+  auto* header = GuestHeader(guest_address);
+  header->common = kShaderFlag | static_cast<u32>(stage);
+  header->reference_count = 1;
+  header->base_flush = 0xFFFF0000u;
+
+  // Preserve the two SDK-private backing-pointer fields observed in TU23.
+  auto* bytes = memory->virtual_membase() + guest_address;
+  if (stage == ShaderStage::kVertex) {
+    *reinterpret_cast<be_u32*>(bytes + 0x20) = function.guest_address();
+  } else {
+    *reinterpret_cast<be_u32*>(bytes + 0x18) = function.guest_address();
+  }
+
+  auto resource = std::make_shared<ShaderResource>();
+  resource->guest_address = guest_address;
+  resource->stage = stage;
+  resource->hash = hash;
+  resource->cache_entry = cache_entry;
+  {
+    std::lock_guard lock(g_registry_mutex);
+    g_registry.emplace(guest_address, std::move(resource));
+  }
+  return guest_address;
+}
+
+bool BindShader(ShaderStage stage, u32 guest_address) {
+  std::shared_ptr<ShaderResource> resource;
+  if (guest_address) {
+    resource = FindResource(guest_address);
+    if (!resource || resource->stage != stage) {
+      REXLOG_WARN("Native GPU: unrecognized {} shader resource 0x{:08X}",
+                  stage == ShaderStage::kVertex ? "vertex" : "pixel", guest_address);
+      return false;
+    }
+  }
+  std::lock_guard lock(g_registry_mutex);
+  g_bound_shaders[StageIndex(stage)] = std::move(resource);
+  return true;
+}
+
+plume::RenderShader* ResolveBoundShader(ShaderStage stage, u32 spec_constants) {
+  std::shared_ptr<ShaderResource> resource;
+  {
+    std::lock_guard lock(g_registry_mutex);
+    resource = g_bound_shaders[StageIndex(stage)];
+  }
+  if (!resource || !resource->cache_entry) {
+    return nullptr;
+  }
+
+  const u32 masked = spec_constants & resource->cache_entry->specConstantsMask;
+  std::lock_guard shader_lock(resource->shader_mutex);
+  const auto existing = resource->variants.find(masked);
+  if (existing != resource->variants.end()) {
+    return existing->second.get();
+  }
+
+  const ShaderBytecode bytecode = FindDxil(resource->hash, masked);
+  auto* device = HostDevice::Device();
+  if (!bytecode || !device) {
+    return nullptr;
+  }
+  auto shader = device->createShader(bytecode.data, bytecode.size, "main",
+                                     plume::RenderShaderFormat::DXIL);
+  if (!shader) {
+    REXLOG_ERROR("Native GPU: D3D12 shader creation failed, hash=0x{:016X}",
+                 resource->hash);
+    return nullptr;
+  }
+  shader->setName(stage == ShaderStage::kVertex ? "LEGO guest vertex shader"
+                                                : "LEGO guest pixel shader");
+  auto [it, inserted] = resource->variants.emplace(masked, std::move(shader));
+  return it->second.get();
+}
+
+bool IsNativeShader(u32 guest_address) {
+  std::lock_guard lock(g_registry_mutex);
+  return g_registry.contains(guest_address);
+}
+
+u32 AddRefNativeShader(u32 guest_address) {
+  std::lock_guard lock(g_registry_mutex);
+  if (!g_registry.contains(guest_address)) {
+    return 0;
+  }
+  auto* header = GuestHeader(guest_address);
+  const u32 count = static_cast<u32>(header->reference_count) + 1;
+  header->reference_count = count;
+  return count;
+}
+
+u32 ReleaseNativeShader(u32 guest_address) {
+  std::shared_ptr<ShaderResource> released;
+  {
+    std::lock_guard lock(g_registry_mutex);
+    const auto it = g_registry.find(guest_address);
+    if (it == g_registry.end()) {
+      return ~u32{0};
+    }
+    auto* header = GuestHeader(guest_address);
+    const u32 old_count = static_cast<u32>(header->reference_count);
+    const u32 count = old_count > 0 ? old_count - 1 : 0;
+    header->reference_count = count;
+    if (count != 0) {
+      return count;
+    }
+    released = std::move(it->second);
+    g_registry.erase(it);
+  }
+  REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
+  return 0;
+}
+
+void ResetShaderResources() {
+  std::vector<u32> guest_addresses;
+  {
+    std::lock_guard lock(g_registry_mutex);
+    g_bound_shaders = {};
+    guest_addresses.reserve(g_registry.size());
+    for (const auto& [guest_address, resource] : g_registry) {
+      guest_addresses.push_back(guest_address);
+    }
+    g_registry.clear();
+  }
+  for (const u32 guest_address : guest_addresses) {
+    REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
+  }
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/shaders.h b/rexlego/src/gpu_native/shaders.h
new file mode 100644
index 0000000..3631fba
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders.h
@@ -0,0 +1,37 @@
+// Guest shader lifetime and AOT host-shader resolution.
+#pragma once
+
+#include <memory>
+
+#include <rex/types.h>
+
+namespace plume {
+struct RenderShader;
+}
+
+namespace legodimensions::gpu_native {
+
+enum class ShaderStage : u32 {
+  kVertex = 6,
+  kPixel = 7,
+};
+
+// Creates the guest-visible resource and records the corresponding AOT hash.
+// The returned value is a guest virtual address, as expected by D3D9.
+u32 CreateShaderResource(mapped_u32 function, ShaderStage stage);
+
+// Binding retains host metadata independently of the guest resource lifetime.
+bool BindShader(ShaderStage stage, u32 guest_address);
+
+// Materializes the correct prelinked specialization at pipeline creation time.
+// A null result marks the containing render pass as unsupported by native GPU.
+plume::RenderShader* ResolveBoundShader(ShaderStage stage, u32 spec_constants);
+
+bool IsNativeShader(u32 guest_address);
+u32 AddRefNativeShader(u32 guest_address);
+u32 ReleaseNativeShader(u32 guest_address);
+
+// Must run before the Plume device is destroyed.
+void ResetShaderResources();
+
+}  // namespace legodimensions::gpu_native

From 0911a43fe7ebad89fa99c619517d66cba8373d8e Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 14:33:54 +0300
Subject: [PATCH 07/44] gpu: add native vertex and index buffers

---
 rexlego/CMakeLists.txt                  |   5 +
 rexlego/src/gpu_native/buffers.cpp      | 219 ++++++++++++++++++++++++
 rexlego/src/gpu_native/buffers.h        |  33 ++++
 rexlego/src/gpu_native/d3d.h            |   7 +
 rexlego/src/gpu_native/device.cpp       |   5 +-
 rexlego/src/gpu_native/hooks_buffer.cpp |  44 +++++
 rexlego/src/gpu_native/hooks_shader.cpp |   9 +
 7 files changed, 321 insertions(+), 1 deletion(-)
 create mode 100644 rexlego/src/gpu_native/buffers.cpp
 create mode 100644 rexlego/src/gpu_native/buffers.h
 create mode 100644 rexlego/src/gpu_native/hooks_buffer.cpp

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index ac5c082..ef880b3 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -55,7 +55,9 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     set_property(TARGET plume PROPERTY SOURCES "${LEGO_PLUME_SOURCES}")
 
     list(APPEND LEGODIMENSIONS_SOURCES
+        src/gpu_native/buffers.cpp
         src/gpu_native/device.cpp
+        src/gpu_native/hooks_buffer.cpp
         src/gpu_native/hooks_device.cpp
         src/gpu_native/hooks_shader.cpp
         src/gpu_native/shader_archive.cpp
@@ -158,6 +160,9 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/d3d.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_buffer.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_device.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_shader.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders.cpp"
diff --git a/rexlego/src/gpu_native/buffers.cpp b/rexlego/src/gpu_native/buffers.cpp
new file mode 100644
index 0000000..a57cf1f
--- /dev/null
+++ b/rexlego/src/gpu_native/buffers.cpp
@@ -0,0 +1,219 @@
+#include "gpu_native/buffers.h"
+
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <plume_render_interface.h>
+#include <rex/logging.h>
+#include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
+
+#include "gpu_native/d3d.h"
+#include "gpu_native/device.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+struct BufferResource {
+  u32 guest_address = 0;
+  u32 mirror_address = 0;
+  u32 length = 0;
+  u32 format = 0;
+  BufferKind kind = BufferKind::kVertex;
+  std::mutex upload_mutex;
+  std::unique_ptr<plume::RenderBuffer> buffer;
+};
+
+std::mutex g_buffers_mutex;
+std::unordered_map<u32, std::shared_ptr<BufferResource>> g_buffers;
+
+D3DBuffer* GuestBuffer(u32 guest_address) {
+  auto* memory = REX_KERNEL_MEMORY();
+  return reinterpret_cast<D3DBuffer*>(memory->virtual_membase() + guest_address);
+}
+
+std::shared_ptr<BufferResource> FindBuffer(u32 guest_address) {
+  std::lock_guard lock(g_buffers_mutex);
+  const auto it = g_buffers.find(guest_address);
+  return it == g_buffers.end() ? nullptr : it->second;
+}
+
+void ByteSwapElements(void* destination, const void* source, size_t size,
+                      u32 element_size) {
+  auto* dst = static_cast<u8*>(destination);
+  const auto* src = static_cast<const u8*>(source);
+  size_t offset = 0;
+  if (element_size == 2) {
+    for (; offset + 2 <= size; offset += 2) {
+      dst[offset + 0] = src[offset + 1];
+      dst[offset + 1] = src[offset + 0];
+    }
+  } else {
+    for (; offset + 4 <= size; offset += 4) {
+      dst[offset + 0] = src[offset + 3];
+      dst[offset + 1] = src[offset + 2];
+      dst[offset + 2] = src[offset + 1];
+      dst[offset + 3] = src[offset + 0];
+    }
+  }
+  if (offset != size) {
+    std::memcpy(dst + offset, src + offset, size - offset);
+  }
+}
+
+}  // namespace
+
+u32 CreateBufferResource(u32 length, u32 /*usage*/, u32 format, u32 /*pool*/,
+                         BufferKind kind) {
+  if (!length || !HostDevice::IsReady()) {
+    return 0;
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  const u32 guest_address = memory->SystemHeapAlloc(sizeof(D3DBuffer), 0x10);
+  if (!guest_address) {
+    return 0;
+  }
+  const u32 mirror_address = memory->SystemHeapAlloc(length, 0x20);
+  if (!mirror_address) {
+    memory->SystemHeapFree(guest_address);
+    return 0;
+  }
+  memory->Zero(guest_address, sizeof(D3DBuffer));
+  memory->Zero(mirror_address, length);
+
+  auto* guest = GuestBuffer(guest_address);
+  guest->resource.common = static_cast<u32>(kind);
+  guest->resource.reference_count = 1;
+  guest->fetch_lo = mirror_address;
+  guest->fetch_hi = length;
+
+  auto resource = std::make_shared<BufferResource>();
+  resource->guest_address = guest_address;
+  resource->mirror_address = mirror_address;
+  resource->length = length;
+  resource->format = format;
+  resource->kind = kind;
+  {
+    std::lock_guard lock(g_buffers_mutex);
+    g_buffers.emplace(guest_address, std::move(resource));
+  }
+  return guest_address;
+}
+
+u32 LockBufferResource(u32 guest_address, u32 offset, u32 size, u32 /*flags*/,
+                       BufferKind kind) {
+  const auto resource = FindBuffer(guest_address);
+  if (!resource || resource->kind != kind || offset > resource->length ||
+      (size != 0 && size > resource->length - offset)) {
+    return 0;
+  }
+  return resource->mirror_address + offset;
+}
+
+plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind) {
+  const auto resource = FindBuffer(guest_address);
+  if (!resource || resource->kind != kind) {
+    return nullptr;
+  }
+  std::lock_guard upload_lock(resource->upload_mutex);
+  if (!resource->buffer) {
+    auto* device = HostDevice::Device();
+    if (!device) {
+      return nullptr;
+    }
+    const auto desc = kind == BufferKind::kIndex
+                          ? plume::RenderBufferDesc::IndexBuffer(
+                                resource->length, plume::RenderHeapType::UPLOAD)
+                          : plume::RenderBufferDesc::VertexBuffer(
+                                resource->length, plume::RenderHeapType::UPLOAD);
+    resource->buffer = device->createBuffer(desc);
+    if (!resource->buffer) {
+      REXLOG_ERROR("Native GPU: failed to allocate {} buffer ({} bytes)",
+                   kind == BufferKind::kIndex ? "index" : "vertex",
+                   resource->length);
+      return nullptr;
+    }
+    resource->buffer->setName(kind == BufferKind::kIndex ? "LEGO guest index buffer"
+                                                          : "LEGO guest vertex buffer");
+  }
+
+  void* mapped = resource->buffer->map();
+  if (!mapped) {
+    return nullptr;
+  }
+  const auto* source =
+      REX_KERNEL_MEMORY()->virtual_membase() + resource->mirror_address;
+  const u32 element_size =
+      kind == BufferKind::kIndex && resource->format == 1 ? 2 : 4;
+  ByteSwapElements(mapped, source, resource->length, element_size);
+  resource->buffer->unmap();
+  return resource->buffer.get();
+}
+
+bool IsNativeBuffer(u32 guest_address) {
+  std::lock_guard lock(g_buffers_mutex);
+  return g_buffers.contains(guest_address);
+}
+
+u32 NativeBufferType(u32 guest_address) {
+  const auto resource = FindBuffer(guest_address);
+  return resource ? static_cast<u32>(resource->kind) : 0;
+}
+
+u32 AddRefNativeBuffer(u32 guest_address) {
+  std::lock_guard lock(g_buffers_mutex);
+  if (!g_buffers.contains(guest_address)) {
+    return 0;
+  }
+  auto* header = &GuestBuffer(guest_address)->resource;
+  const u32 count = static_cast<u32>(header->reference_count) + 1;
+  header->reference_count = count;
+  return count;
+}
+
+u32 ReleaseNativeBuffer(u32 guest_address) {
+  std::shared_ptr<BufferResource> released;
+  {
+    std::lock_guard lock(g_buffers_mutex);
+    const auto it = g_buffers.find(guest_address);
+    if (it == g_buffers.end()) {
+      return ~u32{0};
+    }
+    auto* header = &GuestBuffer(guest_address)->resource;
+    const u32 old_count = static_cast<u32>(header->reference_count);
+    const u32 count = old_count > 0 ? old_count - 1 : 0;
+    header->reference_count = count;
+    if (count != 0) {
+      return count;
+    }
+    released = std::move(it->second);
+    g_buffers.erase(it);
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  memory->SystemHeapFree(released->mirror_address);
+  memory->SystemHeapFree(guest_address);
+  return 0;
+}
+
+void ResetBufferResources() {
+  std::vector<std::pair<u32, u32>> allocations;
+  {
+    std::lock_guard lock(g_buffers_mutex);
+    allocations.reserve(g_buffers.size());
+    for (const auto& [guest_address, resource] : g_buffers) {
+      allocations.emplace_back(guest_address, resource->mirror_address);
+    }
+    g_buffers.clear();
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  for (const auto [guest_address, mirror_address] : allocations) {
+    memory->SystemHeapFree(mirror_address);
+    memory->SystemHeapFree(guest_address);
+  }
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/buffers.h b/rexlego/src/gpu_native/buffers.h
new file mode 100644
index 0000000..e11d1bf
--- /dev/null
+++ b/rexlego/src/gpu_native/buffers.h
@@ -0,0 +1,33 @@
+// Guest-visible vertex/index buffers and their CPU mirrors.
+#pragma once
+
+#include <rex/types.h>
+
+namespace plume {
+struct RenderBuffer;
+}
+
+namespace legodimensions::gpu_native {
+
+enum class BufferKind : u32 {
+  kVertex = 6,
+  kIndex = 7,
+};
+
+u32 CreateBufferResource(u32 length, u32 usage, u32 format, u32 pool,
+                         BufferKind kind);
+u32 LockBufferResource(u32 guest_address, u32 offset, u32 size, u32 flags,
+                       BufferKind kind);
+
+// Uploads the current guest mirror on every call. This deliberately favors
+// correctness over dirty-range guesses: CPU writes made without Lock/Unlock
+// and later CPU readback remain visible.
+plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind);
+
+bool IsNativeBuffer(u32 guest_address);
+u32 NativeBufferType(u32 guest_address);
+u32 AddRefNativeBuffer(u32 guest_address);
+u32 ReleaseNativeBuffer(u32 guest_address);
+void ResetBufferResources();
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h
index 9de8808..4c08e57 100644
--- a/rexlego/src/gpu_native/d3d.h
+++ b/rexlego/src/gpu_native/d3d.h
@@ -48,6 +48,13 @@ struct D3DResource {
 };
 static_assert(sizeof(D3DResource) == 0x18);
 
+struct D3DBuffer {
+  D3DResource resource;
+  be_u32 fetch_lo;
+  be_u32 fetch_hi;
+};
+static_assert(sizeof(D3DBuffer) == 0x20);
+
 struct ShaderContainer {
   be_u32 flags;
   be_u32 virtual_size;
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index b10f76b..b7909cb 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -1,5 +1,6 @@
 #include "gpu_native/device.h"
 
+#include "gpu_native/buffers.h"
 #include "gpu_native/shaders.h"
 #include "native_gpu_build_info.h"
 
@@ -99,10 +100,12 @@ void HostDevice::Shutdown() {
   if (!g_state) {
     return;
   }
-  ResetShaderResources();
   if (g_state->swap_chain) {
     g_state->swap_chain->wait();
   }
+  // All GPU users must be idle before Plume resources are released.
+  ResetShaderResources();
+  ResetBufferResources();
   g_state.reset();
 }
 
diff --git a/rexlego/src/gpu_native/hooks_buffer.cpp b/rexlego/src/gpu_native/hooks_buffer.cpp
new file mode 100644
index 0000000..fbd7ce2
--- /dev/null
+++ b/rexlego/src/gpu_native/hooks_buffer.cpp
@@ -0,0 +1,44 @@
+#include <rex/hook.h>
+#include <rex/ppc/context.h>
+#include <rex/types.h>
+
+#include "gpu_native/buffers.h"
+
+extern "C" void __imp__sub_83FC49E8(PPCContext& __restrict ctx, u8* base);
+
+namespace legodimensions::gpu_native {
+namespace {
+
+u32 CreateVertexBufferHook(u32 length, u32 usage, u32 fvf, u32 pool) {
+  return CreateBufferResource(length, usage, fvf, pool, BufferKind::kVertex);
+}
+
+u32 CreateIndexBufferHook(u32 length, u32 usage, u32 format, u32 pool) {
+  return CreateBufferResource(length, usage, format, pool, BufferKind::kIndex);
+}
+
+u32 LockVertexBufferHook(u32 buffer, u32 offset, u32 size, u32 flags) {
+  return LockBufferResource(buffer, offset, size, flags, BufferKind::kVertex);
+}
+
+u32 LockIndexBufferHook(u32 buffer, u32 offset, u32 size, u32 flags) {
+  return LockBufferResource(buffer, offset, size, flags, BufferKind::kIndex);
+}
+
+}  // namespace
+}  // namespace legodimensions::gpu_native
+
+REX_HOOK(sub_83FC4D58, legodimensions::gpu_native::CreateVertexBufferHook);
+REX_HOOK(sub_83FC4E30, legodimensions::gpu_native::CreateIndexBufferHook);
+REX_HOOK(sub_83FC60D8, legodimensions::gpu_native::LockVertexBufferHook);
+REX_HOOK(sub_83FC6128, legodimensions::gpu_native::LockIndexBufferHook);
+
+REX_HOOK_RAW(sub_83FC49E8) {
+  const u32 resource = ctx.r3.u32;
+  const u32 type = legodimensions::gpu_native::NativeBufferType(resource);
+  if (type) {
+    ctx.r3.u64 = type;
+    return;
+  }
+  __imp__sub_83FC49E8(ctx, base);
+}
diff --git a/rexlego/src/gpu_native/hooks_shader.cpp b/rexlego/src/gpu_native/hooks_shader.cpp
index 3f74579..bb2417b 100644
--- a/rexlego/src/gpu_native/hooks_shader.cpp
+++ b/rexlego/src/gpu_native/hooks_shader.cpp
@@ -3,6 +3,7 @@
 #include <rex/types.h>
 
 #include "gpu_native/d3d.h"
+#include "gpu_native/buffers.h"
 #include "gpu_native/shaders.h"
 
 extern "C" void __imp__sub_83FC4CE0(PPCContext& __restrict ctx, u8* base);
@@ -47,6 +48,10 @@ REX_HOOK(sub_83FB6A30, legodimensions::gpu_native::SetVertexShaderHook);
 REX_HOOK_RAW(sub_83FC4CE0) {
   const u32 resource = ctx.r3.u32;
   if (!legodimensions::gpu_native::IsNativeShader(resource)) {
+    if (legodimensions::gpu_native::IsNativeBuffer(resource)) {
+      ctx.r3.u64 = legodimensions::gpu_native::AddRefNativeBuffer(resource);
+      return;
+    }
     __imp__sub_83FC4CE0(ctx, base);
     return;
   }
@@ -56,6 +61,10 @@ REX_HOOK_RAW(sub_83FC4CE0) {
 REX_HOOK_RAW(sub_83FC59D8) {
   const u32 resource = ctx.r3.u32;
   if (!legodimensions::gpu_native::IsNativeShader(resource)) {
+    if (legodimensions::gpu_native::IsNativeBuffer(resource)) {
+      ctx.r3.u64 = legodimensions::gpu_native::ReleaseNativeBuffer(resource);
+      return;
+    }
     __imp__sub_83FC59D8(ctx, base);
     return;
   }

From 7e80a23586de2459f78c7a196ad6e5a0ce8b9383 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 14:49:12 +0300
Subject: [PATCH 08/44] gpu: add native texture resource lifecycle

---
 rexlego/CMakeLists.txt                   |   8 +
 rexlego/src/gpu_native/d3d.h             |  44 +++
 rexlego/src/gpu_native/device.cpp        |   2 +
 rexlego/src/gpu_native/format.cpp        | 186 +++++++++++
 rexlego/src/gpu_native/format.h          |  16 +
 rexlego/src/gpu_native/hooks_buffer.cpp  |   7 +-
 rexlego/src/gpu_native/hooks_shader.cpp  |  10 +
 rexlego/src/gpu_native/hooks_texture.cpp |  64 ++++
 rexlego/src/gpu_native/textures.cpp      | 373 +++++++++++++++++++++++
 rexlego/src/gpu_native/textures.h        |  29 ++
 10 files changed, 737 insertions(+), 2 deletions(-)
 create mode 100644 rexlego/src/gpu_native/format.cpp
 create mode 100644 rexlego/src/gpu_native/format.h
 create mode 100644 rexlego/src/gpu_native/hooks_texture.cpp
 create mode 100644 rexlego/src/gpu_native/textures.cpp
 create mode 100644 rexlego/src/gpu_native/textures.h

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index ef880b3..f32bea7 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -57,11 +57,14 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     list(APPEND LEGODIMENSIONS_SOURCES
         src/gpu_native/buffers.cpp
         src/gpu_native/device.cpp
+        src/gpu_native/format.cpp
         src/gpu_native/hooks_buffer.cpp
         src/gpu_native/hooks_device.cpp
         src/gpu_native/hooks_shader.cpp
+        src/gpu_native/hooks_texture.cpp
         src/gpu_native/shader_archive.cpp
         src/gpu_native/shaders.cpp
+        src/gpu_native/textures.cpp
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c"
     )
@@ -160,6 +163,11 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/d3d.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/format.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/format.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/textures.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/textures.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_texture.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_buffer.cpp"
diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h
index 4c08e57..21e3b13 100644
--- a/rexlego/src/gpu_native/d3d.h
+++ b/rexlego/src/gpu_native/d3d.h
@@ -55,6 +55,50 @@ struct D3DBuffer {
 };
 static_assert(sizeof(D3DBuffer) == 0x20);
 
+enum class D3DResourceType : u32 {
+  kSurface = 1,
+  kTexture = 3,
+  kVolumeTexture = 17,
+  kCubeTexture = 18,
+};
+
+struct D3DTexture {
+  D3DResource resource;
+  be_u32 mip_flush;
+  FetchConstant format;
+};
+static_assert(sizeof(D3DTexture) == 0x34);
+static_assert(offsetof(D3DTexture, format) == 0x1C);
+
+struct D3DSurface {
+  D3DResource resource;
+  be_u32 surface_info;
+  be_u32 depth_info;
+  be_u32 hi_control;
+  be_u32 size_bits;
+  be_u32 format;
+  be_u32 size;
+};
+static_assert(sizeof(D3DSurface) == 0x30);
+
+struct D3DSurfaceDesc {
+  be_u32 format;
+  be_u32 type;
+  be_u32 usage;
+  be_u32 pool;
+  be_u32 multi_sample_type;
+  be_u32 multi_sample_quality;
+  be_u32 width;
+  be_u32 height;
+};
+static_assert(sizeof(D3DSurfaceDesc) == 0x20);
+
+struct D3DLockedRect {
+  be_u32 pitch;
+  be_u32 bits;
+};
+static_assert(sizeof(D3DLockedRect) == 8);
+
 struct ShaderContainer {
   be_u32 flags;
   be_u32 virtual_size;
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index b7909cb..5dc361c 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -2,6 +2,7 @@
 
 #include "gpu_native/buffers.h"
 #include "gpu_native/shaders.h"
+#include "gpu_native/textures.h"
 #include "native_gpu_build_info.h"
 
 #include <mutex>
@@ -106,6 +107,7 @@ void HostDevice::Shutdown() {
   // All GPU users must be idle before Plume resources are released.
   ResetShaderResources();
   ResetBufferResources();
+  ResetTextureResources();
   g_state.reset();
 }
 
diff --git a/rexlego/src/gpu_native/format.cpp b/rexlego/src/gpu_native/format.cpp
new file mode 100644
index 0000000..0601344
--- /dev/null
+++ b/rexlego/src/gpu_native/format.cpp
@@ -0,0 +1,186 @@
+#include "gpu_native/format.h"
+
+#include <atomic>
+
+#include <rex/graphics/xenos.h>
+#include <rex/logging.h>
+
+namespace legodimensions::gpu_native {
+namespace {
+
+// Full X360 D3DFMT words observed in LEGO/Unleashed-class renderers.
+enum class D3DFormat : u32 {
+  kA16B16G16R16F = 0x1A22AB60,
+  kA16B16G16R16FAlt = 0x1A2201BF,
+  kA8B8G8R8 = 0x1A200186,
+  kA8R8G8B8 = 0x18280186,
+  kX8R8G8B8 = 0x28280086,
+  kTT8888 = 0x28280106,
+  kTT8888Alt = 0x28280186,
+  kTT2101010As16161616 = 0x182801B6,
+  kD24FS8 = 0x1A220197,
+  kD24S8 = 0x2D200196,
+  kR32F = 0x2DA2ABA4,
+  kG16R16F = 0x2D22AB9F,
+  kG16R16FAlt = 0x2D20AB8D,
+  kIndex16 = 1,
+  kIndex32 = 6,
+  kL8 = 0x28000102,
+  kL8Alt = 0x28000002,
+  kA8 = 0x04900102,
+};
+
+void WarnUnknown(u32 value) {
+  static std::atomic<u32> count{0};
+  if (count.fetch_add(1, std::memory_order_relaxed) < 16) {
+    REXLOG_WARN("Native GPU: unknown guest texture format 0x{:08X}", value);
+  }
+}
+
+}  // namespace
+
+plume::RenderFormat ConvertGuestTextureFormat(u32 guest_format) {
+  using RF = plume::RenderFormat;
+
+  switch (static_cast<D3DFormat>(guest_format)) {
+    case D3DFormat::kA16B16G16R16F:
+    case D3DFormat::kA16B16G16R16FAlt:
+    case D3DFormat::kTT2101010As16161616:
+      return RF::R16G16B16A16_FLOAT;
+    case D3DFormat::kA8B8G8R8:
+    case D3DFormat::kA8R8G8B8:
+    case D3DFormat::kX8R8G8B8:
+    case D3DFormat::kTT8888:
+    case D3DFormat::kTT8888Alt:
+      return RF::R8G8B8A8_UNORM;
+    case D3DFormat::kD24FS8:
+    case D3DFormat::kD24S8:
+      // LEGO uses stencil shadows, so retaining the stencil plane is required.
+      return RF::D32_FLOAT_S8_UINT;
+    case D3DFormat::kR32F:
+      return RF::R32_FLOAT;
+    case D3DFormat::kG16R16F:
+    case D3DFormat::kG16R16FAlt:
+      return RF::R16G16_FLOAT;
+    case D3DFormat::kIndex16:
+      return RF::R16_UINT;
+    case D3DFormat::kIndex32:
+      return RF::R32_UINT;
+    case D3DFormat::kL8:
+    case D3DFormat::kL8Alt:
+    case D3DFormat::kA8:
+      return RF::R8_UNORM;
+    default:
+      WarnUnknown(guest_format);
+      return RF::UNKNOWN;
+  }
+}
+
+plume::RenderFormat ConvertXenosTextureFormat(u32 xenos_format) {
+  using TF = rex::graphics::xenos::TextureFormat;
+  using RF = plume::RenderFormat;
+
+  // Some TT asset paths pass the six-bit Xenos texture format rather than a
+  // full D3DFMT word. Connor's Diorama parser confirms Dimensions assets use
+  // DDS DXT1/DXT3/DXT5; DXN/DXT5A cover its normal/mask variants.
+  switch (static_cast<TF>(xenos_format)) {
+    case TF::k_8:
+    case TF::k_8_A:
+    case TF::k_8_B:
+      return RF::R8_UNORM;
+    case TF::k_8_8:
+      return RF::R8G8_UNORM;
+    case TF::k_8_8_8_8:
+    case TF::k_8_8_8_8_A:
+      return RF::R8G8B8A8_UNORM;
+    case TF::k_DXT1:
+      return RF::BC1_UNORM;
+    case TF::k_DXT2_3:
+      return RF::BC2_UNORM;
+    case TF::k_DXT4_5:
+      return RF::BC3_UNORM;
+    case TF::k_DXN:
+      return RF::BC5_UNORM;
+    case TF::k_DXT3A:
+    case TF::k_DXT5A:
+      return RF::BC4_UNORM;
+    case TF::k_16_FLOAT:
+      return RF::R16_FLOAT;
+    case TF::k_16_16_FLOAT:
+      return RF::R16G16_FLOAT;
+    case TF::k_16_16_16_16_FLOAT:
+      return RF::R16G16B16A16_FLOAT;
+    case TF::k_32_FLOAT:
+      return RF::R32_FLOAT;
+    case TF::k_32_32_FLOAT:
+      return RF::R32G32_FLOAT;
+    case TF::k_32_32_32_32_FLOAT:
+      return RF::R32G32B32A32_FLOAT;
+    case TF::k_24_8:
+    case TF::k_24_8_FLOAT:
+      return RF::D32_FLOAT_S8_UINT;
+    case TF::k_2_10_10_10_AS_16_16_16_16:
+      return RF::R16G16B16A16_FLOAT;
+    default:
+      WarnUnknown(xenos_format);
+      return RF::UNKNOWN;
+  }
+}
+
+bool IsDepthFormat(plume::RenderFormat format) {
+  return format == plume::RenderFormat::D32_FLOAT ||
+         format == plume::RenderFormat::D32_FLOAT_S8_UINT;
+}
+
+bool IsRenderTargetFormat(plume::RenderFormat format) {
+  using RF = plume::RenderFormat;
+  switch (format) {
+    case RF::R8_UNORM:
+    case RF::R8G8_UNORM:
+    case RF::R8G8B8A8_UNORM:
+    case RF::B8G8R8A8_UNORM:
+    case RF::R16_FLOAT:
+    case RF::R16G16_FLOAT:
+    case RF::R16G16B16A16_FLOAT:
+    case RF::R16G16B16A16_UNORM:
+    case RF::R32_FLOAT:
+    case RF::R32G32_FLOAT:
+    case RF::R32G32B32A32_FLOAT:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool IsBlockCompressedFormat(plume::RenderFormat format) {
+  using RF = plume::RenderFormat;
+  switch (format) {
+    case RF::BC1_UNORM:
+    case RF::BC2_UNORM:
+    case RF::BC3_UNORM:
+    case RF::BC4_UNORM:
+    case RF::BC5_UNORM:
+    case RF::BC7_UNORM:
+      return true;
+    default:
+      return false;
+  }
+}
+
+u32 FormatBlockBytes(plume::RenderFormat format) {
+  using RF = plume::RenderFormat;
+  switch (format) {
+    case RF::BC1_UNORM:
+    case RF::BC4_UNORM:
+      return 8;
+    case RF::BC2_UNORM:
+    case RF::BC3_UNORM:
+    case RF::BC5_UNORM:
+    case RF::BC7_UNORM:
+      return 16;
+    default:
+      return 0;
+  }
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/format.h b/rexlego/src/gpu_native/format.h
new file mode 100644
index 0000000..ebec0c6
--- /dev/null
+++ b/rexlego/src/gpu_native/format.h
@@ -0,0 +1,16 @@
+// Xbox 360 D3D/Xenos format conversion for the native renderer.
+#pragma once
+
+#include <plume_render_interface.h>
+#include <rex/types.h>
+
+namespace legodimensions::gpu_native {
+
+plume::RenderFormat ConvertGuestTextureFormat(u32 guest_format);
+plume::RenderFormat ConvertXenosTextureFormat(u32 xenos_format);
+bool IsDepthFormat(plume::RenderFormat format);
+bool IsRenderTargetFormat(plume::RenderFormat format);
+bool IsBlockCompressedFormat(plume::RenderFormat format);
+u32 FormatBlockBytes(plume::RenderFormat format);
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/hooks_buffer.cpp b/rexlego/src/gpu_native/hooks_buffer.cpp
index fbd7ce2..77fd488 100644
--- a/rexlego/src/gpu_native/hooks_buffer.cpp
+++ b/rexlego/src/gpu_native/hooks_buffer.cpp
@@ -3,6 +3,7 @@
 #include <rex/types.h>
 
 #include "gpu_native/buffers.h"
+#include "gpu_native/textures.h"
 
 extern "C" void __imp__sub_83FC49E8(PPCContext& __restrict ctx, u8* base);
 
@@ -36,8 +37,10 @@ REX_HOOK(sub_83FC6128, legodimensions::gpu_native::LockIndexBufferHook);
 REX_HOOK_RAW(sub_83FC49E8) {
   const u32 resource = ctx.r3.u32;
   const u32 type = legodimensions::gpu_native::NativeBufferType(resource);
-  if (type) {
-    ctx.r3.u64 = type;
+  const u32 texture_type =
+      legodimensions::gpu_native::NativeTextureType(resource);
+  if (type || texture_type) {
+    ctx.r3.u64 = type ? type : texture_type;
     return;
   }
   __imp__sub_83FC49E8(ctx, base);
diff --git a/rexlego/src/gpu_native/hooks_shader.cpp b/rexlego/src/gpu_native/hooks_shader.cpp
index bb2417b..5f9b295 100644
--- a/rexlego/src/gpu_native/hooks_shader.cpp
+++ b/rexlego/src/gpu_native/hooks_shader.cpp
@@ -5,6 +5,7 @@
 #include "gpu_native/d3d.h"
 #include "gpu_native/buffers.h"
 #include "gpu_native/shaders.h"
+#include "gpu_native/textures.h"
 
 extern "C" void __imp__sub_83FC4CE0(PPCContext& __restrict ctx, u8* base);
 extern "C" void __imp__sub_83FC59D8(PPCContext& __restrict ctx, u8* base);
@@ -52,6 +53,10 @@ REX_HOOK_RAW(sub_83FC4CE0) {
       ctx.r3.u64 = legodimensions::gpu_native::AddRefNativeBuffer(resource);
       return;
     }
+    if (legodimensions::gpu_native::IsNativeTexture(resource)) {
+      ctx.r3.u64 = legodimensions::gpu_native::AddRefNativeTexture(resource);
+      return;
+    }
     __imp__sub_83FC4CE0(ctx, base);
     return;
   }
@@ -65,6 +70,11 @@ REX_HOOK_RAW(sub_83FC59D8) {
       ctx.r3.u64 = legodimensions::gpu_native::ReleaseNativeBuffer(resource);
       return;
     }
+    if (legodimensions::gpu_native::IsNativeTexture(resource)) {
+      ctx.r3.u64 =
+          legodimensions::gpu_native::ReleaseNativeTexture(resource);
+      return;
+    }
     __imp__sub_83FC59D8(ctx, base);
     return;
   }
diff --git a/rexlego/src/gpu_native/hooks_texture.cpp b/rexlego/src/gpu_native/hooks_texture.cpp
new file mode 100644
index 0000000..2b74d1f
--- /dev/null
+++ b/rexlego/src/gpu_native/hooks_texture.cpp
@@ -0,0 +1,64 @@
+#include <rex/hook.h>
+#include <rex/ppc/context.h>
+#include <rex/types.h>
+
+#include "gpu_native/textures.h"
+
+extern "C" void __imp__sub_83FB5540(PPCContext& __restrict ctx, u8* base);
+extern "C" void __imp__sub_83FB5560(PPCContext& __restrict ctx, u8* base);
+extern "C" void __imp__sub_83FB57C8(PPCContext& __restrict ctx, u8* base);
+
+namespace legodimensions::gpu_native {
+namespace {
+
+u32 CreateTextureHook(u32 width, u32 height, u32 depth, u32 levels, u32 usage,
+                      u32 format, u32 pool, u32 type) {
+  return CreateTextureResource(width, height, depth, levels, usage, format,
+                               pool, type);
+}
+
+u32 CreateSurfaceHook(u32 width, u32 height, u32 format, u32 multi_sample,
+                      u32 parameters) {
+  return CreateSurfaceResource(width, height, format, multi_sample, parameters);
+}
+
+}  // namespace
+}  // namespace legodimensions::gpu_native
+
+REX_HOOK(sub_83FB5570, legodimensions::gpu_native::CreateTextureHook);
+REX_HOOK(sub_83FB5690, legodimensions::gpu_native::CreateSurfaceHook);
+
+REX_HOOK_RAW(sub_83FB5540) {
+  if (legodimensions::gpu_native::LockTextureResource(
+          ctx.r3.u32, ctx.r4.u32, ctx.r5.u32, ctx.r6.u32, ctx.r7.u32)) {
+    return;
+  }
+  // A malformed request for one of our zero-fetch-constant textures must not
+  // enter the SDK body: it decodes that absent constant and can divide by zero.
+  if (legodimensions::gpu_native::IsNativeTexture(ctx.r3.u32)) {
+    return;
+  }
+  __imp__sub_83FB5540(ctx, base);
+}
+
+REX_HOOK_RAW(sub_83FB5560) {
+  if (legodimensions::gpu_native::DescribeTextureResource(
+          ctx.r3.u32, ctx.r4.u32, ctx.r5.u32)) {
+    return;
+  }
+  if (legodimensions::gpu_native::IsNativeTexture(ctx.r3.u32)) {
+    return;
+  }
+  __imp__sub_83FB5560(ctx, base);
+}
+
+REX_HOOK_RAW(sub_83FB57C8) {
+  if (legodimensions::gpu_native::DescribeSurfaceResource(ctx.r3.u32,
+                                                          ctx.r4.u32)) {
+    return;
+  }
+  if (legodimensions::gpu_native::IsNativeTexture(ctx.r3.u32)) {
+    return;
+  }
+  __imp__sub_83FB57C8(ctx, base);
+}
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
new file mode 100644
index 0000000..e5b536d
--- /dev/null
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -0,0 +1,373 @@
+#include "gpu_native/textures.h"
+
+#include <algorithm>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <plume_render_interface.h>
+#include <rex/logging.h>
+#include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
+
+#include "gpu_native/d3d.h"
+#include "gpu_native/device.h"
+#include "gpu_native/format.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+constexpr u32 kPitchAlignment = 0x100;
+
+struct TextureResource {
+  u32 guest_address = 0;
+  u32 mirror_address = 0;
+  u32 mirror_size = 0;
+  u32 width = 0;
+  u32 height = 0;
+  u32 depth = 1;
+  u32 levels = 1;
+  u32 guest_format = 0;
+  u32 d3d_type = 3;
+  plume::RenderFormat format = plume::RenderFormat::UNKNOWN;
+  bool surface = false;
+  std::mutex mutex;
+  std::unique_ptr<plume::RenderTexture> texture;
+  std::unique_ptr<plume::RenderTextureView> view;
+};
+
+std::mutex g_textures_mutex;
+std::unordered_map<u32, std::shared_ptr<TextureResource>> g_textures;
+
+template <typename T>
+T* GuestAt(u32 guest_address) {
+  return reinterpret_cast<T*>(REX_KERNEL_MEMORY()->virtual_membase() +
+                              guest_address);
+}
+
+std::shared_ptr<TextureResource> FindTexture(u32 guest_address) {
+  std::lock_guard lock(g_textures_mutex);
+  const auto it = g_textures.find(guest_address);
+  return it == g_textures.end() ? nullptr : it->second;
+}
+
+u32 AlignUp(u32 value, u32 alignment) {
+  return (value + alignment - 1) & ~(alignment - 1);
+}
+
+u32 MipDimension(u32 value, u32 level) {
+  return std::max(1u, value >> std::min(level, 31u));
+}
+
+u32 FullMipCount(u32 width, u32 height, u32 depth) {
+  u32 largest = std::max({width, height, depth});
+  u32 count = 0;
+  do {
+    ++count;
+    largest >>= 1;
+  } while (largest);
+  return count;
+}
+
+u32 RowPitch(plume::RenderFormat format, u32 width) {
+  if (IsBlockCompressedFormat(format)) {
+    return AlignUp(std::max(1u, (width + 3) / 4) * FormatBlockBytes(format),
+                   kPitchAlignment);
+  }
+  const u32 texel_size = plume::RenderFormatSize(format);
+  return texel_size ? AlignUp(width * texel_size, kPitchAlignment) : 0;
+}
+
+u32 RowCount(plume::RenderFormat format, u32 height) {
+  return IsBlockCompressedFormat(format) ? std::max(1u, (height + 3) / 4)
+                                         : height;
+}
+
+u32 LevelSize(const TextureResource& resource, u32 level) {
+  const u32 width = MipDimension(resource.width, level);
+  const u32 height = MipDimension(resource.height, level);
+  const u32 depth = resource.d3d_type ==
+                            static_cast<u32>(D3DResourceType::kVolumeTexture)
+                        ? MipDimension(resource.depth, level)
+                        : 1;
+  return RowPitch(resource.format, width) * RowCount(resource.format, height) *
+         depth;
+}
+
+u32 LevelOffset(const TextureResource& resource, u32 level) {
+  u32 offset = 0;
+  for (u32 i = 0; i < level; ++i) {
+    offset += LevelSize(resource, i);
+  }
+  return offset;
+}
+
+bool EnsureMirror(TextureResource& resource) {
+  if (resource.mirror_address) {
+    return true;
+  }
+  u32 size = 0;
+  for (u32 level = 0; level < resource.levels; ++level) {
+    size += LevelSize(resource, level);
+  }
+  if (!size) {
+    return false;
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  resource.mirror_address = memory->SystemHeapAlloc(size, 0x100);
+  if (!resource.mirror_address) {
+    return false;
+  }
+  resource.mirror_size = size;
+  memory->Zero(resource.mirror_address, size);
+  return true;
+}
+
+std::unique_ptr<plume::RenderTexture> CreateHostTexture(
+    const TextureResource& resource, u32 usage, u32 multi_sample) {
+  auto* device = HostDevice::Device();
+  if (!device || resource.format == plume::RenderFormat::UNKNOWN) {
+    return nullptr;
+  }
+
+  const bool volume = resource.d3d_type ==
+                      static_cast<u32>(D3DResourceType::kVolumeTexture);
+  const bool cube = resource.d3d_type ==
+                    static_cast<u32>(D3DResourceType::kCubeTexture);
+  plume::RenderTextureDesc desc;
+  desc.dimension = volume ? plume::RenderTextureDimension::TEXTURE_3D
+                          : plume::RenderTextureDimension::TEXTURE_2D;
+  desc.width = resource.width;
+  desc.height = resource.height;
+  desc.depth = volume ? resource.depth : 1;
+  desc.mipLevels = resource.levels;
+  desc.arraySize = cube ? 6 : 1;
+  desc.format = resource.format;
+  desc.multisampling.sampleCount = plume::RenderSampleCount::COUNT_1;
+  if (cube) {
+    desc.flags |= plume::RenderTextureFlag::CUBE;
+  }
+  if (IsDepthFormat(resource.format)) {
+    desc.flags |= plume::RenderTextureFlag::DEPTH_TARGET;
+  } else if (resource.surface || usage != 0 ||
+             IsRenderTargetFormat(resource.format)) {
+    desc.flags |= plume::RenderTextureFlag::RENDER_TARGET;
+  }
+  desc.committed = resource.surface || IsDepthFormat(resource.format) ||
+                   usage != 0 || multi_sample != 0;
+  return device->createTexture(desc);
+}
+
+u32 AllocateResource(const std::shared_ptr<TextureResource>& resource,
+                     u32 usage, u32 multi_sample) {
+  if (!resource->width || !resource->height || !resource->levels ||
+      resource->format == plume::RenderFormat::UNKNOWN) {
+    return 0;
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  const u32 guest_size = resource->surface ? sizeof(D3DSurface)
+                                           : sizeof(D3DTexture);
+  resource->guest_address = memory->SystemHeapAlloc(guest_size, 0x10);
+  if (!resource->guest_address) {
+    return 0;
+  }
+  memory->Zero(resource->guest_address, guest_size);
+  resource->texture = CreateHostTexture(*resource, usage, multi_sample);
+  if (!resource->texture) {
+    memory->SystemHeapFree(resource->guest_address);
+    resource->guest_address = 0;
+    return 0;
+  }
+  resource->texture->setName(resource->surface ? "LEGO guest surface"
+                                               : "LEGO guest texture");
+
+  if (resource->surface) {
+    auto* guest = GuestAt<D3DSurface>(resource->guest_address);
+    guest->resource.common = static_cast<u32>(D3DResourceType::kSurface);
+    guest->resource.reference_count = 1;
+    guest->format = resource->guest_format;
+    guest->size_bits = ((resource->width - 1) << 18) |
+                       ((resource->height - 1) << 3);
+  } else {
+    auto* guest = GuestAt<D3DTexture>(resource->guest_address);
+    guest->resource.common = static_cast<u32>(D3DResourceType::kTexture);
+    guest->resource.reference_count = 1;
+  }
+  {
+    std::lock_guard lock(g_textures_mutex);
+    g_textures.emplace(resource->guest_address, resource);
+  }
+  return resource->guest_address;
+}
+
+}  // namespace
+
+u32 CreateTextureResource(u32 width, u32 height, u32 depth, u32 levels,
+                          u32 usage, u32 guest_format, u32 /*pool*/,
+                          u32 d3d_type) {
+  if (!HostDevice::IsReady()) {
+    return 0;
+  }
+  auto resource = std::make_shared<TextureResource>();
+  resource->width = width;
+  resource->height = height;
+  resource->depth = std::max(1u, depth);
+  resource->levels = levels ? levels
+                            : FullMipCount(width, height, resource->depth);
+  resource->guest_format = guest_format;
+  resource->d3d_type = d3d_type == 17 || d3d_type == 18 ? d3d_type : 3;
+  resource->format = ConvertGuestTextureFormat(guest_format);
+  return AllocateResource(resource, usage, 0);
+}
+
+u32 CreateSurfaceResource(u32 width, u32 height, u32 guest_format,
+                          u32 multi_sample, u32 /*parameters*/) {
+  if (!HostDevice::IsReady()) {
+    return 0;
+  }
+  auto resource = std::make_shared<TextureResource>();
+  resource->width = width;
+  resource->height = height;
+  resource->depth = 1;
+  resource->levels = 1;
+  resource->guest_format = guest_format;
+  resource->d3d_type = static_cast<u32>(D3DResourceType::kSurface);
+  resource->format = ConvertGuestTextureFormat(guest_format);
+  resource->surface = true;
+  return AllocateResource(resource, 1, multi_sample);
+}
+
+bool LockTextureResource(u32 guest_address, u32 level, u32 locked_rect,
+                         u32 /*rect*/, u32 /*flags*/) {
+  const auto resource = FindTexture(guest_address);
+  if (!resource) {
+    return false;
+  }
+  if (locked_rect) {
+    auto* locked = GuestAt<D3DLockedRect>(locked_rect);
+    locked->pitch = 0;
+    locked->bits = 0;
+  }
+  if (resource->surface || level >= resource->levels || !locked_rect) {
+    return false;
+  }
+  std::lock_guard lock(resource->mutex);
+  if (!EnsureMirror(*resource)) {
+    return false;
+  }
+  auto* locked = GuestAt<D3DLockedRect>(locked_rect);
+  locked->pitch = RowPitch(resource->format,
+                           MipDimension(resource->width, level));
+  locked->bits = resource->mirror_address + LevelOffset(*resource, level);
+  return true;
+}
+
+bool DescribeTextureResource(u32 guest_address, u32 level, u32 desc_address) {
+  const auto resource = FindTexture(guest_address);
+  if (!resource) {
+    return false;
+  }
+  if (!desc_address || resource->surface || level >= resource->levels) {
+    return false;
+  }
+  auto* desc = GuestAt<D3DSurfaceDesc>(desc_address);
+  desc->format = resource->guest_format;
+  desc->type = resource->d3d_type;
+  desc->usage = 0;
+  desc->pool = 0;
+  desc->multi_sample_type = 0;
+  desc->multi_sample_quality = 0;
+  desc->width = MipDimension(resource->width, level);
+  desc->height = MipDimension(resource->height, level);
+  return true;
+}
+
+bool DescribeSurfaceResource(u32 guest_address, u32 desc_address) {
+  const auto resource = FindTexture(guest_address);
+  if (!resource) {
+    return false;
+  }
+  if (!resource->surface || !desc_address) {
+    return false;
+  }
+  auto* desc = GuestAt<D3DSurfaceDesc>(desc_address);
+  desc->format = resource->guest_format;
+  desc->type = 4;
+  desc->usage = 0;
+  desc->pool = 0;
+  desc->multi_sample_type = 0;
+  desc->multi_sample_quality = 0;
+  desc->width = resource->width;
+  desc->height = resource->height;
+  return true;
+}
+
+bool IsNativeTexture(u32 guest_address) {
+  std::lock_guard lock(g_textures_mutex);
+  return g_textures.contains(guest_address);
+}
+
+u32 NativeTextureType(u32 guest_address) {
+  const auto resource = FindTexture(guest_address);
+  return resource ? resource->d3d_type : 0;
+}
+
+u32 AddRefNativeTexture(u32 guest_address) {
+  const auto resource = FindTexture(guest_address);
+  if (!resource) {
+    return 0;
+  }
+  auto* header = GuestAt<D3DResource>(guest_address);
+  const u32 count = static_cast<u32>(header->reference_count) + 1;
+  header->reference_count = count;
+  return count;
+}
+
+u32 ReleaseNativeTexture(u32 guest_address) {
+  std::shared_ptr<TextureResource> released;
+  {
+    std::lock_guard lock(g_textures_mutex);
+    const auto it = g_textures.find(guest_address);
+    if (it == g_textures.end()) {
+      return ~u32{0};
+    }
+    auto* header = GuestAt<D3DResource>(guest_address);
+    const u32 old_count = static_cast<u32>(header->reference_count);
+    const u32 count = old_count ? old_count - 1 : 0;
+    header->reference_count = count;
+    if (count) {
+      return count;
+    }
+    released = std::move(it->second);
+    g_textures.erase(it);
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  if (released->mirror_address) {
+    memory->SystemHeapFree(released->mirror_address);
+  }
+  memory->SystemHeapFree(guest_address);
+  return 0;
+}
+
+void ResetTextureResources() {
+  std::vector<std::pair<u32, u32>> allocations;
+  {
+    std::lock_guard lock(g_textures_mutex);
+    allocations.reserve(g_textures.size());
+    for (const auto& [guest_address, resource] : g_textures) {
+      allocations.emplace_back(guest_address, resource->mirror_address);
+    }
+    g_textures.clear();
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  for (const auto [guest_address, mirror_address] : allocations) {
+    if (mirror_address) {
+      memory->SystemHeapFree(mirror_address);
+    }
+    memory->SystemHeapFree(guest_address);
+  }
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/textures.h b/rexlego/src/gpu_native/textures.h
new file mode 100644
index 0000000..21470cd
--- /dev/null
+++ b/rexlego/src/gpu_native/textures.h
@@ -0,0 +1,29 @@
+// Guest-visible texture/surface resources and their CPU mirrors.
+#pragma once
+
+#include <rex/types.h>
+
+namespace plume {
+struct RenderTexture;
+struct RenderTextureView;
+}
+
+namespace legodimensions::gpu_native {
+
+u32 CreateTextureResource(u32 width, u32 height, u32 depth, u32 levels,
+                          u32 usage, u32 guest_format, u32 pool, u32 d3d_type);
+u32 CreateSurfaceResource(u32 width, u32 height, u32 guest_format,
+                          u32 multi_sample, u32 parameters);
+
+bool LockTextureResource(u32 guest_address, u32 level, u32 locked_rect,
+                         u32 rect, u32 flags);
+bool DescribeTextureResource(u32 guest_address, u32 level, u32 desc_address);
+bool DescribeSurfaceResource(u32 guest_address, u32 desc_address);
+
+bool IsNativeTexture(u32 guest_address);
+u32 NativeTextureType(u32 guest_address);
+u32 AddRefNativeTexture(u32 guest_address);
+u32 ReleaseNativeTexture(u32 guest_address);
+void ResetTextureResources();
+
+}  // namespace legodimensions::gpu_native

From a206874d993af32d479df7251accc1373da73aa8 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 14:52:57 +0300
Subject: [PATCH 09/44] gpu: shadow native draw bindings

---
 rexlego/CMakeLists.txt                  |  5 ++
 rexlego/src/gpu_native/d3d.h            | 16 ++++-
 rexlego/src/gpu_native/device.cpp       |  2 +
 rexlego/src/gpu_native/hooks_device.cpp |  2 +
 rexlego/src/gpu_native/hooks_state.cpp  | 86 +++++++++++++++++++++++++
 rexlego/src/gpu_native/state.cpp        | 68 +++++++++++++++++++
 rexlego/src/gpu_native/state.h          | 39 +++++++++++
 7 files changed, 217 insertions(+), 1 deletion(-)
 create mode 100644 rexlego/src/gpu_native/hooks_state.cpp
 create mode 100644 rexlego/src/gpu_native/state.cpp
 create mode 100644 rexlego/src/gpu_native/state.h

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index f32bea7..b32ae92 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -61,9 +61,11 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         src/gpu_native/hooks_buffer.cpp
         src/gpu_native/hooks_device.cpp
         src/gpu_native/hooks_shader.cpp
+        src/gpu_native/hooks_state.cpp
         src/gpu_native/hooks_texture.cpp
         src/gpu_native/shader_archive.cpp
         src/gpu_native/shaders.cpp
+        src/gpu_native/state.cpp
         src/gpu_native/textures.cpp
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c"
@@ -168,6 +170,9 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/textures.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/textures.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_texture.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/state.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/state.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_state.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_buffer.cpp"
diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h
index 21e3b13..a5f5d7f 100644
--- a/rexlego/src/gpu_native/d3d.h
+++ b/rexlego/src/gpu_native/d3d.h
@@ -5,6 +5,8 @@
 
 #include <rex/types.h>
 
+#include "gpu_native/state.h"
+
 namespace legodimensions::gpu_native {
 
 constexpr u32 kGuestDeviceSize = 0x6080;
@@ -122,7 +124,14 @@ struct D3DDevice {
   be_u32 get_sampler_state[kSamplerStateCount];
   u8 unknown_0408[0x480 - 0x408];
   FetchConstant fetch_constants[kFetchConstantCount];
-  u8 unknown_06F0[0x32E0 - 0x6F0];
+  u8 unknown_06F0[0x320C - 0x6F0];
+  be_u32 index_buffer;
+  be_u32 render_targets[kNativeRenderTargets];
+  be_u32 depth_stencil;
+  be_u32 vertex_stream_buffers[18];
+  u8 unknown_326C[0x3278 - 0x326C];
+  be_u32 textures[kNativeTextureSlots];
+  u8 unknown_32B8[0x32E0 - 0x32B8];
   D3DViewport9 viewport;
   be_u32 viewport_reserved;
   D3DRect scissor;
@@ -138,6 +147,11 @@ static_assert(offsetof(D3DDevice, get_render_state) == 0x224);
 static_assert(offsetof(D3DDevice, get_sampler_state) == 0x3B8);
 static_assert(offsetof(D3DDevice, fetch_constants) == 0x480);
 static_assert(offsetof(D3DDevice, viewport) == 0x32E0);
+static_assert(offsetof(D3DDevice, index_buffer) == 0x320C);
+static_assert(offsetof(D3DDevice, render_targets) == 0x3210);
+static_assert(offsetof(D3DDevice, depth_stencil) == 0x3220);
+static_assert(offsetof(D3DDevice, vertex_stream_buffers) == 0x3224);
+static_assert(offsetof(D3DDevice, textures) == 0x3278);
 static_assert(offsetof(D3DDevice, scissor) == 0x32FC);
 static_assert(offsetof(D3DDevice, pixel_shader) == 0x330C);
 static_assert(offsetof(D3DDevice, vertex_shader) == 0x3310);
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 5dc361c..bc16f2a 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -3,6 +3,7 @@
 #include "gpu_native/buffers.h"
 #include "gpu_native/shaders.h"
 #include "gpu_native/textures.h"
+#include "gpu_native/state.h"
 #include "native_gpu_build_info.h"
 
 #include <mutex>
@@ -108,6 +109,7 @@ void HostDevice::Shutdown() {
   ResetShaderResources();
   ResetBufferResources();
   ResetTextureResources();
+  ResetDrawBindings();
   g_state.reset();
 }
 
diff --git a/rexlego/src/gpu_native/hooks_device.cpp b/rexlego/src/gpu_native/hooks_device.cpp
index f751313..bddf2ed 100644
--- a/rexlego/src/gpu_native/hooks_device.cpp
+++ b/rexlego/src/gpu_native/hooks_device.cpp
@@ -6,6 +6,7 @@
 
 #include "gpu_native/d3d.h"
 #include "gpu_native/device.h"
+#include "gpu_native/state.h"
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -40,6 +41,7 @@ u32 Direct3DCreateDeviceHook(u32 /*adapter*/, u32 /*device_type*/, u32 /*focus_w
     return 0x8007000Eu;
   }
   memory->Zero(guest_address, kGuestDeviceSize);
+  ResetDrawBindings();
 
   auto* device = reinterpret_cast<D3DDevice*>(memory->virtual_membase() + guest_address);
   CopyStateDispatch(device, memory->virtual_membase());
diff --git a/rexlego/src/gpu_native/hooks_state.cpp b/rexlego/src/gpu_native/hooks_state.cpp
new file mode 100644
index 0000000..9e83891
--- /dev/null
+++ b/rexlego/src/gpu_native/hooks_state.cpp
@@ -0,0 +1,86 @@
+#include <rex/hook.h>
+#include <rex/types.h>
+
+#include "gpu_native/d3d.h"
+#include "gpu_native/state.h"
+#include "gpu_native/textures.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+void SetTextureHook(D3DDevice* device, u32 sampler, u32 texture) {
+  if (device && sampler < kNativeTextureSlots) {
+    device->textures[sampler] = texture;
+  }
+  BindTexture(sampler, texture);
+}
+
+void SetStreamSourceHook(D3DDevice* device, u32 stream, u32 buffer,
+                         u32 offset, u32 stride) {
+  if (device && stream < 18) {
+    device->vertex_stream_buffers[stream] = buffer;
+  }
+  BindVertexStream(stream, buffer, offset, stride);
+}
+
+void SetIndicesHook(D3DDevice* device, u32 buffer) {
+  if (device) {
+    device->index_buffer = buffer;
+  }
+  BindIndexBuffer(buffer);
+}
+
+void SetRenderTargetHook(D3DDevice* device, u32 index, u32 surface) {
+  if (device && index < kNativeRenderTargets) {
+    device->render_targets[index] = surface;
+  }
+  BindRenderTarget(index, surface);
+}
+
+void SetDepthStencilHook(D3DDevice* device, u32 surface) {
+  if (device) {
+    device->depth_stencil = surface;
+  }
+  BindDepthStencil(surface);
+}
+
+u32 GetRenderTargetHook(D3DDevice* /*device*/, u32 index) {
+  const u32 surface = BoundRenderTarget(index);
+  if (surface && IsNativeTexture(surface)) {
+    AddRefNativeTexture(surface);
+  }
+  return surface;
+}
+
+u32 GetDepthStencilHook(D3DDevice* /*device*/) {
+  const u32 surface = BoundDepthStencil();
+  if (surface && IsNativeTexture(surface)) {
+    AddRefNativeTexture(surface);
+  }
+  return surface;
+}
+
+void SetViewportHook(D3DDevice* device, const D3DViewport9* viewport) {
+  if (device && viewport) {
+    device->viewport = *viewport;
+  }
+}
+
+void SetScissorRectHook(D3DDevice* device, const D3DRect* rect) {
+  if (device && rect) {
+    device->scissor = *rect;
+  }
+}
+
+}  // namespace
+}  // namespace legodimensions::gpu_native
+
+REX_HOOK(sub_83FB58A8, legodimensions::gpu_native::SetTextureHook);
+REX_HOOK(sub_83FBA160, legodimensions::gpu_native::SetStreamSourceHook);
+REX_HOOK(sub_83FBA308, legodimensions::gpu_native::SetIndicesHook);
+REX_HOOK(sub_83FBAAA8, legodimensions::gpu_native::SetRenderTargetHook);
+REX_HOOK(sub_83FBAE38, legodimensions::gpu_native::SetDepthStencilHook);
+REX_HOOK(sub_83FBA398, legodimensions::gpu_native::GetRenderTargetHook);
+REX_HOOK(sub_83FBA3E0, legodimensions::gpu_native::GetDepthStencilHook);
+REX_HOOK(sub_83FBA978, legodimensions::gpu_native::SetViewportHook);
+REX_HOOK(sub_83FBA068, legodimensions::gpu_native::SetScissorRectHook);
diff --git a/rexlego/src/gpu_native/state.cpp b/rexlego/src/gpu_native/state.cpp
new file mode 100644
index 0000000..6592d48
--- /dev/null
+++ b/rexlego/src/gpu_native/state.cpp
@@ -0,0 +1,68 @@
+#include "gpu_native/state.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+std::mutex g_bindings_mutex;
+DrawBindings g_bindings;
+
+}  // namespace
+
+void ResetDrawBindings() {
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings = {};
+}
+
+void BindTexture(u32 sampler, u32 texture) {
+  if (sampler >= kNativeTextureSlots) {
+    return;
+  }
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings.textures[sampler] = texture;
+}
+
+void BindVertexStream(u32 stream, u32 buffer, u32 offset, u32 stride) {
+  if (stream >= kNativeVertexStreams) {
+    return;
+  }
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings.vertex_streams[stream] = {buffer, offset, stride};
+}
+
+void BindIndexBuffer(u32 buffer) {
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings.index_buffer = buffer;
+}
+
+void BindRenderTarget(u32 index, u32 surface) {
+  if (index >= kNativeRenderTargets) {
+    return;
+  }
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings.render_targets[index] = surface;
+}
+
+void BindDepthStencil(u32 surface) {
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings.depth_stencil = surface;
+}
+
+u32 BoundRenderTarget(u32 index) {
+  if (index >= kNativeRenderTargets) {
+    return 0;
+  }
+  std::lock_guard lock(g_bindings_mutex);
+  return g_bindings.render_targets[index];
+}
+
+u32 BoundDepthStencil() {
+  std::lock_guard lock(g_bindings_mutex);
+  return g_bindings.depth_stencil;
+}
+
+DrawBindings SnapshotDrawBindings() {
+  std::lock_guard lock(g_bindings_mutex);
+  return g_bindings;
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/state.h b/rexlego/src/gpu_native/state.h
new file mode 100644
index 0000000..3626f22
--- /dev/null
+++ b/rexlego/src/gpu_native/state.h
@@ -0,0 +1,39 @@
+// CPU-side shadow of the guest draw bindings consumed by the native renderer.
+#pragma once
+
+#include <array>
+#include <mutex>
+
+#include <rex/types.h>
+
+namespace legodimensions::gpu_native {
+
+constexpr u32 kNativeTextureSlots = 16;
+constexpr u32 kNativeVertexStreams = 16;
+constexpr u32 kNativeRenderTargets = 4;
+
+struct VertexStreamBinding {
+  u32 buffer = 0;
+  u32 offset = 0;
+  u32 stride = 0;
+};
+
+struct DrawBindings {
+  std::array<u32, kNativeTextureSlots> textures{};
+  std::array<VertexStreamBinding, kNativeVertexStreams> vertex_streams{};
+  std::array<u32, kNativeRenderTargets> render_targets{};
+  u32 depth_stencil = 0;
+  u32 index_buffer = 0;
+};
+
+void ResetDrawBindings();
+void BindTexture(u32 sampler, u32 texture);
+void BindVertexStream(u32 stream, u32 buffer, u32 offset, u32 stride);
+void BindIndexBuffer(u32 buffer);
+void BindRenderTarget(u32 index, u32 surface);
+void BindDepthStencil(u32 surface);
+u32 BoundRenderTarget(u32 index);
+u32 BoundDepthStencil();
+DrawBindings SnapshotDrawBindings();
+
+}  // namespace legodimensions::gpu_native

From b0b7f252eff1f99aa8acb66ae4e02ab44c9fc3b2 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 14:58:40 +0300
Subject: [PATCH 10/44] gpu: compile native vertex declarations

---
 rexlego/CMakeLists.txt                        |   3 +
 rexlego/src/gpu_native/d3d.h                  |   4 +-
 rexlego/src/gpu_native/device.cpp             |   2 +
 rexlego/src/gpu_native/hooks_shader.cpp       |  11 +
 rexlego/src/gpu_native/hooks_state.cpp        |  16 +
 rexlego/src/gpu_native/state.cpp              |   5 +
 rexlego/src/gpu_native/state.h                |   2 +
 .../src/gpu_native/vertex_declarations.cpp    | 345 ++++++++++++++++++
 rexlego/src/gpu_native/vertex_declarations.h  |  33 ++
 9 files changed, 420 insertions(+), 1 deletion(-)
 create mode 100644 rexlego/src/gpu_native/vertex_declarations.cpp
 create mode 100644 rexlego/src/gpu_native/vertex_declarations.h

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index b32ae92..cea384b 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -67,6 +67,7 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         src/gpu_native/shaders.cpp
         src/gpu_native/state.cpp
         src/gpu_native/textures.cpp
+        src/gpu_native/vertex_declarations.cpp
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c"
     )
@@ -173,6 +174,8 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/state.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/state.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_state.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/vertex_declarations.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/vertex_declarations.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_buffer.cpp"
diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h
index a5f5d7f..5855523 100644
--- a/rexlego/src/gpu_native/d3d.h
+++ b/rexlego/src/gpu_native/d3d.h
@@ -124,7 +124,8 @@ struct D3DDevice {
   be_u32 get_sampler_state[kSamplerStateCount];
   u8 unknown_0408[0x480 - 0x408];
   FetchConstant fetch_constants[kFetchConstantCount];
-  u8 unknown_06F0[0x320C - 0x6F0];
+  u8 unknown_06F0[0x3208 - 0x6F0];
+  be_u32 vertex_declaration;
   be_u32 index_buffer;
   be_u32 render_targets[kNativeRenderTargets];
   be_u32 depth_stencil;
@@ -148,6 +149,7 @@ static_assert(offsetof(D3DDevice, get_sampler_state) == 0x3B8);
 static_assert(offsetof(D3DDevice, fetch_constants) == 0x480);
 static_assert(offsetof(D3DDevice, viewport) == 0x32E0);
 static_assert(offsetof(D3DDevice, index_buffer) == 0x320C);
+static_assert(offsetof(D3DDevice, vertex_declaration) == 0x3208);
 static_assert(offsetof(D3DDevice, render_targets) == 0x3210);
 static_assert(offsetof(D3DDevice, depth_stencil) == 0x3220);
 static_assert(offsetof(D3DDevice, vertex_stream_buffers) == 0x3224);
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index bc16f2a..9dd6782 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -4,6 +4,7 @@
 #include "gpu_native/shaders.h"
 #include "gpu_native/textures.h"
 #include "gpu_native/state.h"
+#include "gpu_native/vertex_declarations.h"
 #include "native_gpu_build_info.h"
 
 #include <mutex>
@@ -110,6 +111,7 @@ void HostDevice::Shutdown() {
   ResetBufferResources();
   ResetTextureResources();
   ResetDrawBindings();
+  ResetVertexDeclarations();
   g_state.reset();
 }
 
diff --git a/rexlego/src/gpu_native/hooks_shader.cpp b/rexlego/src/gpu_native/hooks_shader.cpp
index 5f9b295..a5df44f 100644
--- a/rexlego/src/gpu_native/hooks_shader.cpp
+++ b/rexlego/src/gpu_native/hooks_shader.cpp
@@ -6,6 +6,7 @@
 #include "gpu_native/buffers.h"
 #include "gpu_native/shaders.h"
 #include "gpu_native/textures.h"
+#include "gpu_native/vertex_declarations.h"
 
 extern "C" void __imp__sub_83FC4CE0(PPCContext& __restrict ctx, u8* base);
 extern "C" void __imp__sub_83FC59D8(PPCContext& __restrict ctx, u8* base);
@@ -57,6 +58,11 @@ REX_HOOK_RAW(sub_83FC4CE0) {
       ctx.r3.u64 = legodimensions::gpu_native::AddRefNativeTexture(resource);
       return;
     }
+    if (legodimensions::gpu_native::IsNativeVertexDeclaration(resource)) {
+      ctx.r3.u64 =
+          legodimensions::gpu_native::AddRefNativeVertexDeclaration(resource);
+      return;
+    }
     __imp__sub_83FC4CE0(ctx, base);
     return;
   }
@@ -75,6 +81,11 @@ REX_HOOK_RAW(sub_83FC59D8) {
           legodimensions::gpu_native::ReleaseNativeTexture(resource);
       return;
     }
+    if (legodimensions::gpu_native::IsNativeVertexDeclaration(resource)) {
+      ctx.r3.u64 = legodimensions::gpu_native::
+          ReleaseNativeVertexDeclaration(resource);
+      return;
+    }
     __imp__sub_83FC59D8(ctx, base);
     return;
   }
diff --git a/rexlego/src/gpu_native/hooks_state.cpp b/rexlego/src/gpu_native/hooks_state.cpp
index 9e83891..b90239f 100644
--- a/rexlego/src/gpu_native/hooks_state.cpp
+++ b/rexlego/src/gpu_native/hooks_state.cpp
@@ -4,6 +4,7 @@
 #include "gpu_native/d3d.h"
 #include "gpu_native/state.h"
 #include "gpu_native/textures.h"
+#include "gpu_native/vertex_declarations.h"
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -30,6 +31,17 @@ void SetIndicesHook(D3DDevice* device, u32 buffer) {
   BindIndexBuffer(buffer);
 }
 
+void SetVertexDeclarationHook(D3DDevice* device, u32 declaration) {
+  if (device) {
+    device->vertex_declaration = declaration;
+  }
+  BindVertexDeclaration(declaration);
+}
+
+u32 CreateVertexDeclarationHook(u32 elements) {
+  return CreateVertexDeclarationResource(elements);
+}
+
 void SetRenderTargetHook(D3DDevice* device, u32 index, u32 surface) {
   if (device && index < kNativeRenderTargets) {
     device->render_targets[index] = surface;
@@ -78,6 +90,10 @@ void SetScissorRectHook(D3DDevice* device, const D3DRect* rect) {
 REX_HOOK(sub_83FB58A8, legodimensions::gpu_native::SetTextureHook);
 REX_HOOK(sub_83FBA160, legodimensions::gpu_native::SetStreamSourceHook);
 REX_HOOK(sub_83FBA308, legodimensions::gpu_native::SetIndicesHook);
+REX_HOOK(sub_83FB6D50,
+         legodimensions::gpu_native::CreateVertexDeclarationHook);
+REX_HOOK(sub_83FB7840,
+         legodimensions::gpu_native::SetVertexDeclarationHook);
 REX_HOOK(sub_83FBAAA8, legodimensions::gpu_native::SetRenderTargetHook);
 REX_HOOK(sub_83FBAE38, legodimensions::gpu_native::SetDepthStencilHook);
 REX_HOOK(sub_83FBA398, legodimensions::gpu_native::GetRenderTargetHook);
diff --git a/rexlego/src/gpu_native/state.cpp b/rexlego/src/gpu_native/state.cpp
index 6592d48..2af61fd 100644
--- a/rexlego/src/gpu_native/state.cpp
+++ b/rexlego/src/gpu_native/state.cpp
@@ -34,6 +34,11 @@ void BindIndexBuffer(u32 buffer) {
   g_bindings.index_buffer = buffer;
 }
 
+void BindVertexDeclaration(u32 declaration) {
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings.vertex_declaration = declaration;
+}
+
 void BindRenderTarget(u32 index, u32 surface) {
   if (index >= kNativeRenderTargets) {
     return;
diff --git a/rexlego/src/gpu_native/state.h b/rexlego/src/gpu_native/state.h
index 3626f22..6385550 100644
--- a/rexlego/src/gpu_native/state.h
+++ b/rexlego/src/gpu_native/state.h
@@ -24,12 +24,14 @@ struct DrawBindings {
   std::array<u32, kNativeRenderTargets> render_targets{};
   u32 depth_stencil = 0;
   u32 index_buffer = 0;
+  u32 vertex_declaration = 0;
 };
 
 void ResetDrawBindings();
 void BindTexture(u32 sampler, u32 texture);
 void BindVertexStream(u32 stream, u32 buffer, u32 offset, u32 stride);
 void BindIndexBuffer(u32 buffer);
+void BindVertexDeclaration(u32 declaration);
 void BindRenderTarget(u32 index, u32 surface);
 void BindDepthStencil(u32 surface);
 u32 BoundRenderTarget(u32 index);
diff --git a/rexlego/src/gpu_native/vertex_declarations.cpp b/rexlego/src/gpu_native/vertex_declarations.cpp
new file mode 100644
index 0000000..4b6b14a
--- /dev/null
+++ b/rexlego/src/gpu_native/vertex_declarations.cpp
@@ -0,0 +1,345 @@
+#include "gpu_native/vertex_declarations.h"
+
+#include <algorithm>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+
+#include <plume_render_interface.h>
+#include <rex/logging.h>
+#include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
+
+#include "gpu_native/d3d.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+constexpr u32 kDeclarationFlag = 0x00100005;
+constexpr u32 kMaxElements = 64;
+
+enum class DeclUsage : u8 {
+  kPosition = 0,
+  kBlendWeight = 1,
+  kBlendIndices = 2,
+  kNormal = 3,
+  kPSize = 4,
+  kTexCoord = 5,
+  kTangent = 6,
+  kBinormal = 7,
+  kTessFactor = 8,
+  kPositionT = 9,
+  kColor = 10,
+  kFog = 11,
+  kDepth = 12,
+  kSample = 13,
+};
+
+struct GuestVertexElement {
+  be_u16 stream;
+  be_u16 offset;
+  be_u32 type;
+  u8 method;
+  u8 usage;
+  u8 usage_index;
+  u8 padding;
+};
+static_assert(sizeof(GuestVertexElement) == 12);
+
+struct VertexDeclarationResource {
+  u32 guest_address = 0;
+  std::vector<plume::RenderInputElement> inputs;
+  bool supported = true;
+  u32 swapped_texcoords = 0;
+  u32 swapped_normals = 0;
+  u32 swapped_binormals = 0;
+  u32 swapped_tangents = 0;
+  u32 swapped_blend_weights = 0;
+  u32 swapped_positions = 0;
+  u32 sint_texcoords = 0;
+  bool has_r11g11b10_normal = false;
+};
+
+std::mutex g_declarations_mutex;
+std::unordered_map<u32, std::shared_ptr<VertexDeclarationResource>>
+    g_declarations;
+
+const char* UsageName(DeclUsage usage) {
+  switch (usage) {
+    case DeclUsage::kPosition: return "POSITION";
+    case DeclUsage::kBlendWeight: return "BLENDWEIGHT";
+    case DeclUsage::kBlendIndices: return "BLENDINDICES";
+    case DeclUsage::kNormal: return "NORMAL";
+    case DeclUsage::kPSize: return "PSIZE";
+    case DeclUsage::kTexCoord: return "TEXCOORD";
+    case DeclUsage::kTangent: return "TANGENT";
+    case DeclUsage::kBinormal: return "BINORMAL";
+    case DeclUsage::kTessFactor: return "TESSFACTOR";
+    case DeclUsage::kPositionT: return "POSITIONT";
+    case DeclUsage::kColor: return "COLOR";
+    case DeclUsage::kFog: return "FOG";
+    case DeclUsage::kDepth: return "DEPTH";
+    case DeclUsage::kSample: return "SAMPLE";
+  }
+  return "UNKNOWN";
+}
+
+u32 InputLocation(DeclUsage usage, u32 index) {
+  if (usage == DeclUsage::kPosition && index <= 4) return index;
+  if (usage == DeclUsage::kNormal && index == 0) return 5;
+  if (usage == DeclUsage::kTangent && index == 0) return 6;
+  if (usage == DeclUsage::kTexCoord && index <= 2) return 7 + index;
+  if (usage == DeclUsage::kColor && index == 0) return 10;
+  return ~u32{0};
+}
+
+plume::RenderFormat ConvertDeclFormat(u32 type) {
+  using RF = plume::RenderFormat;
+  switch (type) {
+    case 0x002C83A4: return RF::R32_FLOAT;
+    case 0x002C23A5: return RF::R32G32_FLOAT;
+    case 0x002A23B9: return RF::R32G32B32_FLOAT;
+    case 0x001A23A6: return RF::R32G32B32A32_FLOAT;
+    case 0x00182886: return RF::B8G8R8A8_UNORM;
+    case 0x001A2286:
+    case 0x001A2386: return RF::R8G8B8A8_UINT;
+    case 0x002C2359: return RF::R16G16_SINT;
+    case 0x001A235A: return RF::R16G16B16A16_SNORM;
+    case 0x001A2086:
+    case 0x001A2186: return RF::R8G8B8A8_UNORM;
+    case 0x002C2159: return RF::R16G16_SNORM;
+    case 0x001A215A: return RF::R16G16B16A16_SNORM;
+    case 0x002C2059: return RF::R16G16_UNORM;
+    case 0x001A205A: return RF::R16G16B16A16_UNORM;
+    case 0x002C82A1: return RF::R32_UINT;
+    case 0x002A2190:
+    case 0x002A2390: return RF::R32_UINT;
+    case 0x002C235F: return RF::R16G16_FLOAT;
+    case 0x001A2360: return RF::R16G16B16A16_FLOAT;
+    default: return RF::UNKNOWN;
+  }
+}
+
+bool NeedsPairSwap(u32 type) {
+  switch (type) {
+    case 0x002C2359:
+    case 0x001A235A:
+    case 0x002C2159:
+    case 0x001A215A:
+    case 0x002C2059:
+    case 0x001A205A:
+    case 0x002C235F:
+    case 0x001A2360:
+      return true;
+    default:
+      return false;
+  }
+}
+
+std::shared_ptr<VertexDeclarationResource> FindDeclaration(u32 address) {
+  std::lock_guard lock(g_declarations_mutex);
+  const auto it = g_declarations.find(address);
+  return it == g_declarations.end() ? nullptr : it->second;
+}
+
+D3DResource* GuestHeader(u32 address) {
+  return reinterpret_cast<D3DResource*>(REX_KERNEL_MEMORY()->virtual_membase() +
+                                        address);
+}
+
+}  // namespace
+
+u32 CreateVertexDeclarationResource(u32 elements_address) {
+  if (!elements_address) {
+    return 0;
+  }
+  const auto* elements = reinterpret_cast<const GuestVertexElement*>(
+      REX_KERNEL_MEMORY()->virtual_membase() + elements_address);
+  u32 count = 0;
+  while (count < kMaxElements && static_cast<u16>(elements[count].stream) != 0xFF &&
+         static_cast<u32>(elements[count].type) != 0xFFFFFFFFu) {
+    ++count;
+  }
+  if (count == kMaxElements) {
+    REXLOG_ERROR("Native GPU: unterminated vertex declaration");
+    return 0;
+  }
+
+  auto resource = std::make_shared<VertexDeclarationResource>();
+  resource->inputs.reserve(count);
+  for (u32 i = 0; i < count; ++i) {
+    const auto& element = elements[i];
+    const auto usage = static_cast<DeclUsage>(element.usage);
+    plume::RenderInputElement input;
+    input.semanticName = UsageName(usage);
+    input.semanticIndex = element.usage_index;
+    input.location = InputLocation(usage, element.usage_index);
+    input.format = ConvertDeclFormat(element.type);
+    input.slotIndex = element.stream;
+    input.alignedByteOffset = element.offset;
+    if (input.format == plume::RenderFormat::UNKNOWN) {
+      resource->supported = false;
+      REXLOG_WARN("Native GPU: unsupported vertex element usage={} index={} "
+                  "type=0x{:08X}", element.usage, element.usage_index,
+                  static_cast<u32>(element.type));
+    }
+    const u32 usage_index_bit = 1u << std::min<u32>(element.usage_index, 31);
+    if (NeedsPairSwap(element.type)) {
+      switch (usage) {
+        case DeclUsage::kPosition:
+          resource->swapped_positions |= usage_index_bit;
+          break;
+        case DeclUsage::kNormal:
+          resource->swapped_normals |= usage_index_bit;
+          break;
+        case DeclUsage::kBinormal:
+          resource->swapped_binormals |= usage_index_bit;
+          break;
+        case DeclUsage::kTangent:
+          resource->swapped_tangents |= usage_index_bit;
+          break;
+        case DeclUsage::kBlendWeight:
+          resource->swapped_blend_weights |= usage_index_bit;
+          break;
+        case DeclUsage::kTexCoord:
+          resource->swapped_texcoords |= usage_index_bit;
+          break;
+        default:
+          break;
+      }
+    }
+    if ((usage == DeclUsage::kNormal || usage == DeclUsage::kTangent ||
+         usage == DeclUsage::kBinormal) && element.type == 0x002A23B9) {
+      input.format = plume::RenderFormat::R32G32B32_UINT;
+    }
+    if ((usage == DeclUsage::kNormal || usage == DeclUsage::kTangent ||
+         usage == DeclUsage::kBinormal) &&
+        (static_cast<u32>(element.type) == 0x002A2190 ||
+         static_cast<u32>(element.type) == 0x002A2390)) {
+      resource->has_r11g11b10_normal = true;
+    }
+    if (usage == DeclUsage::kTexCoord &&
+        (static_cast<u32>(element.type) == 0x002C2359 ||
+         static_cast<u32>(element.type) == 0x001A235A)) {
+      input.format = static_cast<u32>(element.type) == 0x002C2359
+                         ? plume::RenderFormat::R16G16_UINT
+                         : plume::RenderFormat::R16G16B16A16_UINT;
+      resource->sint_texcoords |= usage_index_bit;
+    }
+    resource->inputs.push_back(input);
+  }
+
+  const auto add_synthetic = [&](DeclUsage usage, u32 index, u32 location) {
+    const char* name = UsageName(usage);
+    for (const auto& input : resource->inputs) {
+      if (input.semanticIndex == index &&
+          std::strcmp(input.semanticName, name) == 0) {
+        return;
+      }
+    }
+    plume::RenderFormat format = plume::RenderFormat::R32_FLOAT;
+    if (usage == DeclUsage::kPosition) {
+      format = plume::RenderFormat::R32G32B32A32_FLOAT;
+    } else if (usage == DeclUsage::kNormal || usage == DeclUsage::kTangent) {
+      format = plume::RenderFormat::R32G32B32_FLOAT;
+    }
+    resource->inputs.emplace_back(name, index, location, format, 15, 0);
+  };
+  for (u32 i = 0; i <= 4; ++i) add_synthetic(DeclUsage::kPosition, i, i);
+  add_synthetic(DeclUsage::kNormal, 0, 5);
+  add_synthetic(DeclUsage::kTangent, 0, 6);
+  for (u32 i = 0; i <= 2; ++i) add_synthetic(DeclUsage::kTexCoord, i, 7 + i);
+  add_synthetic(DeclUsage::kColor, 0, 10);
+
+  const u32 allocation_size = 52 + (count + 1) * sizeof(GuestVertexElement);
+  auto* memory = REX_KERNEL_MEMORY();
+  const u32 guest_address = memory->SystemHeapAlloc(allocation_size, 0x10);
+  if (!guest_address) {
+    return 0;
+  }
+  memory->Zero(guest_address, allocation_size);
+  auto* header = GuestHeader(guest_address);
+  header->common = kDeclarationFlag;
+  header->reference_count = 1;
+  header->base_flush = 0xFFFF0000u;
+  auto* bytes = memory->virtual_membase() + guest_address;
+  *reinterpret_cast<be_u32*>(bytes + 24) = count;
+  std::memcpy(bytes + 52, elements,
+              (count + 1) * sizeof(GuestVertexElement));
+
+  resource->guest_address = guest_address;
+  {
+    std::lock_guard lock(g_declarations_mutex);
+    g_declarations.emplace(guest_address, resource);
+  }
+  return guest_address;
+}
+
+VertexDeclarationView ResolveVertexDeclaration(u32 guest_address) {
+  const auto resource = FindDeclaration(guest_address);
+  if (!resource) {
+    return {};
+  }
+  return {resource->inputs.data(),
+          static_cast<u32>(resource->inputs.size()),
+          resource->supported,
+          resource->swapped_texcoords,
+          resource->swapped_normals,
+          resource->swapped_binormals,
+          resource->swapped_tangents,
+          resource->swapped_blend_weights,
+          resource->swapped_positions,
+          resource->sint_texcoords,
+          resource->has_r11g11b10_normal};
+}
+
+bool IsNativeVertexDeclaration(u32 guest_address) {
+  std::lock_guard lock(g_declarations_mutex);
+  return g_declarations.contains(guest_address);
+}
+
+u32 AddRefNativeVertexDeclaration(u32 guest_address) {
+  const auto resource = FindDeclaration(guest_address);
+  if (!resource) return 0;
+  auto* header = GuestHeader(guest_address);
+  const u32 count = static_cast<u32>(header->reference_count) + 1;
+  header->reference_count = count;
+  return count;
+}
+
+u32 ReleaseNativeVertexDeclaration(u32 guest_address) {
+  std::shared_ptr<VertexDeclarationResource> released;
+  {
+    std::lock_guard lock(g_declarations_mutex);
+    const auto it = g_declarations.find(guest_address);
+    if (it == g_declarations.end()) return ~u32{0};
+    auto* header = GuestHeader(guest_address);
+    const u32 old_count = header->reference_count;
+    const u32 count = old_count ? old_count - 1 : 0;
+    header->reference_count = count;
+    if (count) return count;
+    released = std::move(it->second);
+    g_declarations.erase(it);
+  }
+  REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
+  return 0;
+}
+
+void ResetVertexDeclarations() {
+  std::vector<u32> addresses;
+  {
+    std::lock_guard lock(g_declarations_mutex);
+    addresses.reserve(g_declarations.size());
+    for (const auto& [address, resource] : g_declarations) {
+      addresses.push_back(address);
+    }
+    g_declarations.clear();
+  }
+  for (const u32 address : addresses) {
+    REX_KERNEL_MEMORY()->SystemHeapFree(address);
+  }
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/vertex_declarations.h b/rexlego/src/gpu_native/vertex_declarations.h
new file mode 100644
index 0000000..07b6479
--- /dev/null
+++ b/rexlego/src/gpu_native/vertex_declarations.h
@@ -0,0 +1,33 @@
+// Native input-layout metadata built from Xbox 360 D3D vertex elements.
+#pragma once
+
+#include <rex/types.h>
+
+namespace plume {
+struct RenderInputElement;
+}
+
+namespace legodimensions::gpu_native {
+
+struct VertexDeclarationView {
+  const plume::RenderInputElement* elements = nullptr;
+  u32 element_count = 0;
+  bool supported = false;
+  u32 swapped_texcoords = 0;
+  u32 swapped_normals = 0;
+  u32 swapped_binormals = 0;
+  u32 swapped_tangents = 0;
+  u32 swapped_blend_weights = 0;
+  u32 swapped_positions = 0;
+  u32 sint_texcoords = 0;
+  bool has_r11g11b10_normal = false;
+};
+
+u32 CreateVertexDeclarationResource(u32 elements_address);
+VertexDeclarationView ResolveVertexDeclaration(u32 guest_address);
+bool IsNativeVertexDeclaration(u32 guest_address);
+u32 AddRefNativeVertexDeclaration(u32 guest_address);
+u32 ReleaseNativeVertexDeclaration(u32 guest_address);
+void ResetVertexDeclarations();
+
+}  // namespace legodimensions::gpu_native

From 462c4301292e2e1110dd676623fd9c94f7423afa Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 15:02:05 +0300
Subject: [PATCH 11/44] gpu: add native bindless pipeline layout

---
 rexlego/src/gpu_native/device.cpp   | 119 ++++++++++++++++++++++++++++
 rexlego/src/gpu_native/device.h     |  12 +++
 rexlego/src/gpu_native/textures.cpp |  21 +++++
 3 files changed, 152 insertions(+)

diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 9dd6782..9c5ea4f 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -9,8 +9,10 @@
 
 #include <mutex>
 #include <string>
+#include <vector>
 
 #include <plume_render_interface.h>
+#include <plume_render_interface_builders.h>
 #include <rex/logging.h>
 #include <rex/ui/window.h>
 
@@ -27,9 +29,18 @@ struct State {
   std::unique_ptr<plume::RenderCommandQueue> queue;
   std::unique_ptr<plume::RenderSwapChain> swap_chain;
   std::unique_ptr<plume::RenderCommandFence> idle_fence;
+  std::unique_ptr<plume::RenderPipelineLayout> pipeline_layout;
+  std::unique_ptr<plume::RenderDescriptorSet> texture_descriptors;
+  std::unique_ptr<plume::RenderDescriptorSet> sampler_descriptors;
+  std::unique_ptr<plume::RenderSampler> default_sampler;
+  std::vector<bool> texture_slots;
   Backend backend = Backend::kD3D12;
 };
 
+constexpr u32 kBindlessTextureCount = 65536;
+constexpr u32 kBindlessSamplerCount = 2048;
+constexpr u32 kFirstTextureSlot = 3;
+
 std::mutex g_mutex;
 std::unique_ptr<State> g_state;
 
@@ -49,6 +60,63 @@ const char* NameOf(Backend backend) {
   return backend == Backend::kD3D12 ? "D3D12" : "Vulkan";
 }
 
+bool CreatePipelineLayout(State& state) {
+  plume::RenderPipelineLayoutBuilder layout;
+  layout.begin(false, true);
+
+  plume::RenderDescriptorSetBuilder textures;
+  textures.begin();
+  textures.addTexture(0, kBindlessTextureCount);
+  textures.end(true, kBindlessTextureCount);
+  state.texture_descriptors = textures.create(state.device.get());
+  if (!state.texture_descriptors) {
+    return false;
+  }
+  layout.addDescriptorSet(textures);
+  layout.addDescriptorSet(textures);
+  layout.addDescriptorSet(textures);
+
+  plume::RenderDescriptorSetBuilder samplers;
+  samplers.begin();
+  samplers.addSampler(0, kBindlessSamplerCount);
+  samplers.end(true, kBindlessSamplerCount);
+  state.sampler_descriptors = samplers.create(state.device.get());
+  if (!state.sampler_descriptors) {
+    return false;
+  }
+  layout.addDescriptorSet(samplers);
+
+  layout.addRootDescriptor(0, 4,
+                           plume::RenderRootDescriptorType::CONSTANT_BUFFER);
+  layout.addRootDescriptor(1, 4,
+                           plume::RenderRootDescriptorType::CONSTANT_BUFFER);
+  layout.addRootDescriptor(2, 4,
+                           plume::RenderRootDescriptorType::CONSTANT_BUFFER);
+  layout.end();
+  state.pipeline_layout = layout.create(state.device.get());
+  if (!state.pipeline_layout) {
+    return false;
+  }
+
+  plume::RenderSamplerDesc sampler_desc;
+  sampler_desc.minFilter = plume::RenderFilter::LINEAR;
+  sampler_desc.magFilter = plume::RenderFilter::LINEAR;
+  sampler_desc.mipmapMode = plume::RenderMipmapMode::LINEAR;
+  sampler_desc.addressU = plume::RenderTextureAddressMode::CLAMP;
+  sampler_desc.addressV = plume::RenderTextureAddressMode::CLAMP;
+  sampler_desc.addressW = plume::RenderTextureAddressMode::CLAMP;
+  state.default_sampler = state.device->createSampler(sampler_desc);
+  if (!state.default_sampler) {
+    return false;
+  }
+  state.sampler_descriptors->setSampler(0, state.default_sampler.get());
+  state.texture_slots.assign(kBindlessTextureCount, false);
+  for (u32 i = 0; i < kFirstTextureSlot; ++i) {
+    state.texture_slots[i] = true;
+  }
+  return true;
+}
+
 }  // namespace
 
 bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
@@ -74,6 +142,10 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
     REXLOG_ERROR("Native GPU: failed to create {} device", NameOf(backend));
     return false;
   }
+  if (!CreatePipelineLayout(*state)) {
+    REXLOG_ERROR("Native GPU: failed to create bindless pipeline layout");
+    return false;
+  }
   state->queue = state->device->createCommandQueue(plume::RenderCommandListType::DIRECT);
   state->idle_fence = state->device->createCommandFence();
   if (!state->queue || !state->idle_fence) {
@@ -135,6 +207,53 @@ plume::RenderSwapChain* HostDevice::SwapChain() {
   return g_state ? g_state->swap_chain.get() : nullptr;
 }
 
+plume::RenderPipelineLayout* HostDevice::PipelineLayout() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? g_state->pipeline_layout.get() : nullptr;
+}
+
+plume::RenderDescriptorSet* HostDevice::TextureDescriptorSet() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? g_state->texture_descriptors.get() : nullptr;
+}
+
+plume::RenderDescriptorSet* HostDevice::SamplerDescriptorSet() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? g_state->sampler_descriptors.get() : nullptr;
+}
+
+u32 HostDevice::RegisterTexture(plume::RenderTexture* texture,
+                                plume::RenderTextureView* view) {
+  if (!texture || !view) {
+    return ~u32{0};
+  }
+  std::lock_guard lock(g_mutex);
+  if (!g_state || !g_state->texture_descriptors) {
+    return ~u32{0};
+  }
+  for (u32 slot = kFirstTextureSlot; slot < g_state->texture_slots.size();
+       ++slot) {
+    if (g_state->texture_slots[slot]) {
+      continue;
+    }
+    g_state->texture_slots[slot] = true;
+    g_state->texture_descriptors->setTexture(
+        slot, texture, plume::RenderTextureLayout::SHADER_READ, view);
+    return slot;
+  }
+  REXLOG_ERROR("Native GPU: bindless texture heap exhausted");
+  return ~u32{0};
+}
+
+void HostDevice::UnregisterTexture(u32 descriptor_index) {
+  std::lock_guard lock(g_mutex);
+  if (!g_state || descriptor_index < kFirstTextureSlot ||
+      descriptor_index >= g_state->texture_slots.size()) {
+    return;
+  }
+  g_state->texture_slots[descriptor_index] = false;
+}
+
 std::string_view HostDevice::BackendName() {
   std::lock_guard lock(g_mutex);
   return g_state ? NameOf(g_state->backend) : "none";
diff --git a/rexlego/src/gpu_native/device.h b/rexlego/src/gpu_native/device.h
index b6f6661..e5e0098 100644
--- a/rexlego/src/gpu_native/device.h
+++ b/rexlego/src/gpu_native/device.h
@@ -4,12 +4,18 @@
 #include <memory>
 #include <string_view>
 
+#include <rex/types.h>
+
 namespace plume {
 struct RenderCommandFence;
 struct RenderCommandQueue;
+struct RenderDescriptorSet;
 struct RenderDevice;
 struct RenderInterface;
+struct RenderPipelineLayout;
 struct RenderSwapChain;
+struct RenderTexture;
+struct RenderTextureView;
 }  // namespace plume
 
 namespace rex::ui {
@@ -32,6 +38,12 @@ class HostDevice {
   static plume::RenderDevice* Device();
   static plume::RenderCommandQueue* Queue();
   static plume::RenderSwapChain* SwapChain();
+  static plume::RenderPipelineLayout* PipelineLayout();
+  static plume::RenderDescriptorSet* TextureDescriptorSet();
+  static plume::RenderDescriptorSet* SamplerDescriptorSet();
+  static u32 RegisterTexture(plume::RenderTexture* texture,
+                             plume::RenderTextureView* view);
+  static void UnregisterTexture(u32 descriptor_index);
   static std::string_view BackendName();
 
  private:
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index e5b536d..2ee5834 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -36,6 +36,7 @@ struct TextureResource {
   std::mutex mutex;
   std::unique_ptr<plume::RenderTexture> texture;
   std::unique_ptr<plume::RenderTextureView> view;
+  u32 descriptor_index = ~u32{0};
 };
 
 std::mutex g_textures_mutex;
@@ -182,6 +183,25 @@ u32 AllocateResource(const std::shared_ptr<TextureResource>& resource,
   }
   resource->texture->setName(resource->surface ? "LEGO guest surface"
                                                : "LEGO guest texture");
+  if (!IsDepthFormat(resource->format)) {
+    plume::RenderTextureViewDesc view_desc;
+    view_desc.format = resource->format;
+    view_desc.mipLevels = resource->levels;
+    if (resource->d3d_type ==
+        static_cast<u32>(D3DResourceType::kVolumeTexture)) {
+      view_desc.dimension = plume::RenderTextureViewDimension::TEXTURE_3D;
+    } else if (resource->d3d_type ==
+               static_cast<u32>(D3DResourceType::kCubeTexture)) {
+      view_desc.dimension = plume::RenderTextureViewDimension::TEXTURE_CUBE;
+    } else {
+      view_desc.dimension = plume::RenderTextureViewDimension::TEXTURE_2D;
+    }
+    resource->view = resource->texture->createTextureView(view_desc);
+    if (resource->view) {
+      resource->descriptor_index = HostDevice::RegisterTexture(
+          resource->texture.get(), resource->view.get());
+    }
+  }
 
   if (resource->surface) {
     auto* guest = GuestAt<D3DSurface>(resource->guest_address);
@@ -344,6 +364,7 @@ u32 ReleaseNativeTexture(u32 guest_address) {
     g_textures.erase(it);
   }
   auto* memory = REX_KERNEL_MEMORY();
+  HostDevice::UnregisterTexture(released->descriptor_index);
   if (released->mirror_address) {
     memory->SystemHeapFree(released->mirror_address);
   }

From 9513adbbf528b5704582b10369d2cebe9aeb3b94 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 15:03:39 +0300
Subject: [PATCH 12/44] gpu: cache native render framebuffers

---
 rexlego/src/gpu_native/textures.cpp | 79 +++++++++++++++++++++++++++++
 rexlego/src/gpu_native/textures.h   | 17 +++++++
 2 files changed, 96 insertions(+)

diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index 2ee5834..cf88384 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -36,6 +36,8 @@ struct TextureResource {
   std::mutex mutex;
   std::unique_ptr<plume::RenderTexture> texture;
   std::unique_ptr<plume::RenderTextureView> view;
+  std::unordered_map<u32, std::unique_ptr<plume::RenderFramebuffer>>
+      framebuffers;
   u32 descriptor_index = ~u32{0};
 };
 
@@ -54,6 +56,21 @@ std::shared_ptr<TextureResource> FindTexture(u32 guest_address) {
   return it == g_textures.end() ? nullptr : it->second;
 }
 
+void InvalidateFramebufferReferences(u32 guest_address) {
+  std::vector<std::shared_ptr<TextureResource>> resources;
+  {
+    std::lock_guard lock(g_textures_mutex);
+    resources.reserve(g_textures.size());
+    for (const auto& [address, resource] : g_textures) {
+      resources.push_back(resource);
+    }
+  }
+  for (const auto& resource : resources) {
+    std::lock_guard lock(resource->mutex);
+    resource->framebuffers.erase(guest_address);
+  }
+}
+
 u32 AlignUp(u32 value, u32 alignment) {
   return (value + alignment - 1) & ~(alignment - 1);
 }
@@ -324,6 +341,67 @@ bool DescribeSurfaceResource(u32 guest_address, u32 desc_address) {
   return true;
 }
 
+TextureResourceView ResolveTextureResource(u32 guest_address) {
+  const auto resource = FindTexture(guest_address);
+  if (!resource) {
+    return {};
+  }
+  return {resource->texture.get(),
+          resource->view.get(),
+          resource->format,
+          resource->descriptor_index,
+          resource->width,
+          resource->height,
+          resource->surface,
+          IsDepthFormat(resource->format)};
+}
+
+plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
+                                             u32 depth_stencil) {
+  const auto color = FindTexture(render_target);
+  const auto depth = FindTexture(depth_stencil);
+  if ((!color || !color->texture) && (!depth || !depth->texture)) {
+    return nullptr;
+  }
+  if (color && IsDepthFormat(color->format)) {
+    return nullptr;
+  }
+  if (depth && !IsDepthFormat(depth->format)) {
+    return nullptr;
+  }
+  const auto owner = depth ? depth : color;
+  const u32 key = depth ? render_target : depth_stencil;
+  std::lock_guard lock(owner->mutex);
+  const auto existing = owner->framebuffers.find(key);
+  if (existing != owner->framebuffers.end()) {
+    return existing->second.get();
+  }
+
+  plume::RenderFramebufferDesc desc;
+  const plume::RenderTexture* color_attachments[1];
+  if (color && color->texture) {
+    color_attachments[0] = color->texture.get();
+    desc.colorAttachments = color_attachments;
+    desc.colorAttachmentsCount = 1;
+  }
+  if (depth && depth->texture) {
+    desc.depthAttachment = depth->texture.get();
+  }
+  auto* device = HostDevice::Device();
+  if (!device) {
+    return nullptr;
+  }
+  auto framebuffer = device->createFramebuffer(desc);
+  if (!framebuffer) {
+    REXLOG_ERROR("Native GPU: failed to create framebuffer (rt=0x{:08X}, "
+                 "ds=0x{:08X})", render_target, depth_stencil);
+    return nullptr;
+  }
+  auto* result = framebuffer.get();
+  owner->framebuffers.emplace(key, std::move(framebuffer));
+  return result;
+}
+
 bool IsNativeTexture(u32 guest_address) {
   std::lock_guard lock(g_textures_mutex);
   return g_textures.contains(guest_address);
@@ -364,6 +442,7 @@ u32 ReleaseNativeTexture(u32 guest_address) {
     g_textures.erase(it);
   }
   auto* memory = REX_KERNEL_MEMORY();
+  InvalidateFramebufferReferences(guest_address);
   HostDevice::UnregisterTexture(released->descriptor_index);
   if (released->mirror_address) {
     memory->SystemHeapFree(released->mirror_address);
diff --git a/rexlego/src/gpu_native/textures.h b/rexlego/src/gpu_native/textures.h
index 21470cd..500fbcf 100644
--- a/rexlego/src/gpu_native/textures.h
+++ b/rexlego/src/gpu_native/textures.h
@@ -1,9 +1,11 @@
 // Guest-visible texture/surface resources and their CPU mirrors.
 #pragma once
 
+#include <plume_render_interface_types.h>
 #include <rex/types.h>
 
 namespace plume {
+struct RenderFramebuffer;
 struct RenderTexture;
 struct RenderTextureView;
 }
@@ -20,6 +22,21 @@ bool LockTextureResource(u32 guest_address, u32 level, u32 locked_rect,
 bool DescribeTextureResource(u32 guest_address, u32 level, u32 desc_address);
 bool DescribeSurfaceResource(u32 guest_address, u32 desc_address);
 
+struct TextureResourceView {
+  plume::RenderTexture* texture = nullptr;
+  plume::RenderTextureView* view = nullptr;
+  plume::RenderFormat format = plume::RenderFormat::UNKNOWN;
+  u32 descriptor_index = ~u32{0};
+  u32 width = 0;
+  u32 height = 0;
+  bool surface = false;
+  bool depth = false;
+};
+
+TextureResourceView ResolveTextureResource(u32 guest_address);
+plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
+                                             u32 depth_stencil);
+
 bool IsNativeTexture(u32 guest_address);
 u32 NativeTextureType(u32 guest_address);
 u32 AddRefNativeTexture(u32 guest_address);

From d629afbcf0f8f02077a94397622e14df0f8313ca Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 15:09:12 +0300
Subject: [PATCH 13/44] gpu: add native frame-ring present path

---
 rexlego/CMakeLists.txt                        |  38 ++++
 rexlego/src/gpu_native/device.cpp             | 191 ++++++++++++++++++
 rexlego/src/gpu_native/device.h               |   6 +
 .../src/gpu_native/shaders/copy_color_ps.hlsl |  12 ++
 .../src/gpu_native/shaders/copy_common.hlsli  |  11 +
 rexlego/src/gpu_native/shaders/copy_vs.hlsl   |   6 +
 6 files changed, 264 insertions(+)
 create mode 100644 rexlego/src/gpu_native/shaders/copy_color_ps.hlsl
 create mode 100644 rexlego/src/gpu_native/shaders/copy_common.hlsli
 create mode 100644 rexlego/src/gpu_native/shaders/copy_vs.hlsl

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index cea384b..af415ec 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -128,6 +128,41 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         set(LEGO_DXC_ARCH arm64)
     endif()
 
+    set(LEGO_NATIVE_HOST_SHADER_DIR
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders")
+    set(LEGO_NATIVE_GENERATED_SHADER_DIR
+        "${CMAKE_CURRENT_BINARY_DIR}/generated/gpu_native/shaders")
+    set(LEGO_NATIVE_DXC
+        "${LEGO_DXC_ROOT}/bin/${LEGO_DXC_ARCH}/dxc.exe")
+    foreach(LEGO_HOST_SHADER IN ITEMS copy_vs copy_color_ps)
+        if(LEGO_HOST_SHADER STREQUAL "copy_vs")
+            set(LEGO_HOST_SHADER_PROFILE vs_6_6)
+        else()
+            set(LEGO_HOST_SHADER_PROFILE ps_6_6)
+        endif()
+        set(LEGO_HOST_SHADER_HEADER
+            "${LEGO_NATIVE_GENERATED_SHADER_DIR}/${LEGO_HOST_SHADER}.h")
+        add_custom_command(
+            OUTPUT "${LEGO_HOST_SHADER_HEADER}"
+            COMMAND ${CMAKE_COMMAND} -E make_directory
+                    "${LEGO_NATIVE_GENERATED_SHADER_DIR}"
+            COMMAND "${LEGO_NATIVE_DXC}"
+                    -T ${LEGO_HOST_SHADER_PROFILE} -E main -HV 2021
+                    -all-resources-bound -Wno-ignored-attributes
+                    -I "${LEGO_NATIVE_HOST_SHADER_DIR}"
+                    -Fh "${LEGO_HOST_SHADER_HEADER}"
+                    -Vn g_${LEGO_HOST_SHADER}_dxil
+                    "${LEGO_NATIVE_HOST_SHADER_DIR}/${LEGO_HOST_SHADER}.hlsl"
+            DEPENDS
+                "${LEGO_NATIVE_HOST_SHADER_DIR}/${LEGO_HOST_SHADER}.hlsl"
+                "${LEGO_NATIVE_HOST_SHADER_DIR}/copy_common.hlsli"
+            COMMENT "Compiling ${LEGO_HOST_SHADER}.hlsl (DXIL)"
+            VERBATIM)
+        target_sources(legodimensions PRIVATE "${LEGO_HOST_SHADER_HEADER}")
+    endforeach()
+    target_include_directories(legodimensions PRIVATE
+        "${CMAKE_CURRENT_BINARY_DIR}/generated")
+
     add_executable(lego_gpu_prelink EXCLUDE_FROM_ALL
         "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/prelink_shader_cache.cpp"
         src/gpu_native/dxc_link.cpp
@@ -165,6 +200,9 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     set(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_common.hlsli"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_vs.hlsl"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_color_ps.hlsl"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/d3d.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/format.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/format.h"
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 9c5ea4f..49908e2 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -5,8 +5,12 @@
 #include "gpu_native/textures.h"
 #include "gpu_native/state.h"
 #include "gpu_native/vertex_declarations.h"
+#include "gpu_native/shaders/copy_color_ps.h"
+#include "gpu_native/shaders/copy_vs.h"
 #include "native_gpu_build_info.h"
 
+#include <array>
+#include <chrono>
 #include <mutex>
 #include <string>
 #include <vector>
@@ -24,6 +28,8 @@ namespace legodimensions::gpu_native {
 namespace {
 
 struct State {
+  static constexpr u32 kFramesInFlight = 3;
+
   std::unique_ptr<plume::RenderInterface> render_interface;
   std::unique_ptr<plume::RenderDevice> device;
   std::unique_ptr<plume::RenderCommandQueue> queue;
@@ -33,6 +39,21 @@ struct State {
   std::unique_ptr<plume::RenderDescriptorSet> texture_descriptors;
   std::unique_ptr<plume::RenderDescriptorSet> sampler_descriptors;
   std::unique_ptr<plume::RenderSampler> default_sampler;
+  std::unique_ptr<plume::RenderShader> copy_vertex_shader;
+  std::unique_ptr<plume::RenderShader> copy_pixel_shader;
+  std::unique_ptr<plume::RenderPipeline> copy_pipeline;
+  std::vector<std::unique_ptr<plume::RenderFramebuffer>> framebuffers;
+  std::array<std::unique_ptr<plume::RenderCommandList>, kFramesInFlight>
+      command_lists;
+  std::array<std::unique_ptr<plume::RenderCommandFence>, kFramesInFlight>
+      frame_fences;
+  std::array<std::unique_ptr<plume::RenderCommandSemaphore>, kFramesInFlight>
+      acquire_semaphores;
+  std::vector<std::unique_ptr<plume::RenderCommandSemaphore>>
+      render_semaphores;
+  std::array<bool, kFramesInFlight> frame_submitted{};
+  u32 frame_slot = 0;
+  u32 long_wait_log_count = 0;
   std::vector<bool> texture_slots;
   Backend backend = Backend::kD3D12;
 };
@@ -92,6 +113,8 @@ bool CreatePipelineLayout(State& state) {
                            plume::RenderRootDescriptorType::CONSTANT_BUFFER);
   layout.addRootDescriptor(2, 4,
                            plume::RenderRootDescriptorType::CONSTANT_BUFFER);
+  layout.addPushConstant(3, 4, sizeof(u32) * 4,
+                         plume::RenderShaderStageFlag::PIXEL);
   layout.end();
   state.pipeline_layout = layout.create(state.device.get());
   if (!state.pipeline_layout) {
@@ -117,6 +140,76 @@ bool CreatePipelineLayout(State& state) {
   return true;
 }
 
+bool CreatePresentPipeline(State& state) {
+  state.copy_vertex_shader = state.device->createShader(
+      g_copy_vs_dxil, sizeof(g_copy_vs_dxil), "main",
+      plume::RenderShaderFormat::DXIL);
+  state.copy_pixel_shader = state.device->createShader(
+      g_copy_color_ps_dxil, sizeof(g_copy_color_ps_dxil), "main",
+      plume::RenderShaderFormat::DXIL);
+  if (!state.copy_vertex_shader || !state.copy_pixel_shader) {
+    return false;
+  }
+
+  plume::RenderGraphicsPipelineDesc desc;
+  desc.pipelineLayout = state.pipeline_layout.get();
+  desc.vertexShader = state.copy_vertex_shader.get();
+  desc.pixelShader = state.copy_pixel_shader.get();
+  desc.depthFunction = plume::RenderComparisonFunction::ALWAYS;
+  desc.depthEnabled = false;
+  desc.depthWriteEnabled = false;
+  desc.primitiveTopology = plume::RenderPrimitiveTopology::TRIANGLE_LIST;
+  desc.cullMode = plume::RenderCullMode::NONE;
+  desc.fillMode = plume::RenderFillMode::SOLID;
+  desc.renderTargetCount = 1;
+  desc.renderTargetFormat[0] = plume::RenderFormat::B8G8R8A8_UNORM;
+  desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy();
+  state.copy_pipeline = state.device->createGraphicsPipeline(desc);
+  return state.copy_pipeline != nullptr;
+}
+
+bool CreateFrameRing(State& state) {
+  const u32 texture_count = state.swap_chain->getTextureCount();
+  state.framebuffers.clear();
+  state.render_semaphores.clear();
+  state.framebuffers.reserve(texture_count);
+  state.render_semaphores.reserve(texture_count);
+  for (u32 i = 0; i < texture_count; ++i) {
+    const plume::RenderTexture* attachment = state.swap_chain->getTexture(i);
+    state.framebuffers.emplace_back(state.device->createFramebuffer(
+        plume::RenderFramebufferDesc(&attachment, 1)));
+    state.render_semaphores.emplace_back(
+        state.device->createCommandSemaphore());
+    if (!state.framebuffers.back() || !state.render_semaphores.back()) {
+      return false;
+    }
+  }
+
+  for (u32 i = 0; i < State::kFramesInFlight; ++i) {
+    state.command_lists[i] = state.queue->createCommandList();
+    state.frame_fences[i] = state.device->createCommandFence();
+    state.acquire_semaphores[i] = state.device->createCommandSemaphore();
+    state.frame_submitted[i] = false;
+    if (!state.command_lists[i] || !state.frame_fences[i] ||
+        !state.acquire_semaphores[i]) {
+      return false;
+    }
+  }
+  state.frame_slot = 0;
+  return true;
+}
+
+bool RebuildSwapChain(State& state) {
+  state.swap_chain->wait();
+  state.framebuffers.clear();
+  state.render_semaphores.clear();
+  state.frame_submitted.fill(false);
+  if (!state.swap_chain->resize() || state.swap_chain->isEmpty()) {
+    return false;
+  }
+  return CreateFrameRing(state);
+}
+
 }  // namespace
 
 bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
@@ -160,6 +253,10 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
     REXLOG_ERROR("Native GPU: failed to create or size the swap chain");
     return false;
   }
+  if (!CreatePresentPipeline(*state) || !CreateFrameRing(*state)) {
+    REXLOG_ERROR("Native GPU: failed to create present pipeline or frame ring");
+    return false;
+  }
 
   const auto& description = state->device->getDescription();
   REXLOG_INFO("Native GPU [{}]: {} ready on '{}' ({}x{}, {} images)",
@@ -254,6 +351,100 @@ void HostDevice::UnregisterTexture(u32 descriptor_index) {
   g_state->texture_slots[descriptor_index] = false;
 }
 
+bool HostDevice::PresentTexture(plume::RenderTexture* texture,
+                                u32 descriptor_index) {
+  if (!texture || descriptor_index == ~u32{0}) {
+    return false;
+  }
+
+  std::lock_guard lock(g_mutex);
+  if (!g_state || !g_state->copy_pipeline ||
+      descriptor_index >= g_state->texture_slots.size() ||
+      !g_state->texture_slots[descriptor_index]) {
+    return false;
+  }
+  State& state = *g_state;
+  if (state.swap_chain->needsResize()) {
+    if (!RebuildSwapChain(state)) {
+      REXLOG_WARN("Native GPU: swap chain resize deferred");
+      return false;
+    }
+  }
+
+  const u32 slot = state.frame_slot;
+  if (state.frame_submitted[slot]) {
+    const auto wait_start = std::chrono::steady_clock::now();
+    state.queue->waitForCommandFence(state.frame_fences[slot].get());
+    const auto wait_ms = std::chrono::duration<double, std::milli>(
+                             std::chrono::steady_clock::now() - wait_start)
+                             .count();
+    if (wait_ms > 20.0 && state.long_wait_log_count++ < 20) {
+      REXLOG_WARN("Native GPU: frame-slot {} GPU wait took {:.2f} ms", slot,
+                  wait_ms);
+    }
+    state.frame_submitted[slot] = false;
+  }
+
+  u32 image_index = 0;
+  if (!state.swap_chain->acquireTexture(
+          state.acquire_semaphores[slot].get(), &image_index) ||
+      image_index >= state.framebuffers.size()) {
+    return false;
+  }
+
+  plume::RenderCommandList* commands = state.command_lists[slot].get();
+  plume::RenderTexture* back = state.swap_chain->getTexture(image_index);
+  commands->begin();
+  const plume::RenderTextureBarrier barriers[] = {
+      plume::RenderTextureBarrier(texture,
+                                  plume::RenderTextureLayout::SHADER_READ),
+      plume::RenderTextureBarrier(back,
+                                  plume::RenderTextureLayout::COLOR_WRITE),
+  };
+  commands->barriers(plume::RenderBarrierStage::GRAPHICS, barriers, 2);
+  commands->setFramebuffer(state.framebuffers[image_index].get());
+
+  const u32 width = state.swap_chain->getWidth();
+  const u32 height = state.swap_chain->getHeight();
+  commands->setViewports(
+      plume::RenderViewport(0.0f, 0.0f, float(width), float(height)));
+  commands->setScissors(plume::RenderRect(0, 0, width, height));
+  commands->setGraphicsPipelineLayout(state.pipeline_layout.get());
+  commands->setGraphicsDescriptorSet(state.texture_descriptors.get(), 0);
+  commands->setGraphicsDescriptorSet(state.texture_descriptors.get(), 1);
+  commands->setGraphicsDescriptorSet(state.texture_descriptors.get(), 2);
+  commands->setGraphicsDescriptorSet(state.sampler_descriptors.get(), 3);
+  commands->setPipeline(state.copy_pipeline.get());
+  const struct {
+    u32 descriptor_index;
+    u32 descriptor_index_2;
+    float multiplier;
+    float unused;
+  } push_constants = {descriptor_index, 0, 1.0f, 0.0f};
+  commands->setGraphicsPushConstants(0, &push_constants);
+  commands->drawInstanced(3, 1, 0, 0);
+  commands->setFramebuffer(nullptr);
+  commands->barriers(
+      plume::RenderBarrierStage::NONE,
+      plume::RenderTextureBarrier(back, plume::RenderTextureLayout::PRESENT));
+  commands->end();
+
+  const plume::RenderCommandList* lists[] = {commands};
+  plume::RenderCommandSemaphore* waits[] = {
+      state.acquire_semaphores[slot].get()};
+  plume::RenderCommandSemaphore* signals[] = {
+      state.render_semaphores[image_index].get()};
+  state.queue->executeCommandLists(lists, 1, waits, 1, signals, 1,
+                                   state.frame_fences[slot].get());
+  state.frame_submitted[slot] = true;
+  const bool presented = state.swap_chain->present(image_index, signals, 1);
+  state.frame_slot = (slot + 1) % State::kFramesInFlight;
+  if (!presented) {
+    REXLOG_ERROR("Native GPU: swap-chain present failed");
+  }
+  return presented;
+}
+
 std::string_view HostDevice::BackendName() {
   std::lock_guard lock(g_mutex);
   return g_state ? NameOf(g_state->backend) : "none";
diff --git a/rexlego/src/gpu_native/device.h b/rexlego/src/gpu_native/device.h
index e5e0098..39cdbdc 100644
--- a/rexlego/src/gpu_native/device.h
+++ b/rexlego/src/gpu_native/device.h
@@ -9,6 +9,7 @@
 namespace plume {
 struct RenderCommandFence;
 struct RenderCommandQueue;
+struct RenderCommandList;
 struct RenderDescriptorSet;
 struct RenderDevice;
 struct RenderInterface;
@@ -44,6 +45,11 @@ class HostDevice {
   static u32 RegisterTexture(plume::RenderTexture* texture,
                              plume::RenderTextureView* view);
   static void UnregisterTexture(u32 descriptor_index);
+  // Copies a single-sample native texture to the host swap chain. This is the
+  // first complete submission path and deliberately keeps command allocators
+  // in a frame ring so the CPU does not wait for the frame it just submitted.
+  static bool PresentTexture(plume::RenderTexture* texture,
+                             u32 descriptor_index);
   static std::string_view BackendName();
 
  private:
diff --git a/rexlego/src/gpu_native/shaders/copy_color_ps.hlsl b/rexlego/src/gpu_native/shaders/copy_color_ps.hlsl
new file mode 100644
index 0000000..171735d
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders/copy_color_ps.hlsl
@@ -0,0 +1,12 @@
+#include "copy_common.hlsli"
+
+Texture2D<float4> g_Texture2DDescriptorHeap[] : register(t0, space0);
+SamplerState g_SamplerDescriptorHeap[] : register(s0, space3);
+
+float4 main(in float4 position : SV_Position,
+            in float2 tex_coord : TEXCOORD) : SV_Target {
+  Texture2D<float4> texture =
+      g_Texture2DDescriptorHeap[g_PushConstants.ResourceDescriptorIndex];
+  return texture.Sample(g_SamplerDescriptorHeap[0], tex_coord) *
+         float4(g_PushConstants.Param0.xxx, 1.0);
+}
diff --git a/rexlego/src/gpu_native/shaders/copy_common.hlsli b/rexlego/src/gpu_native/shaders/copy_common.hlsli
new file mode 100644
index 0000000..24ef998
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders/copy_common.hlsli
@@ -0,0 +1,11 @@
+#pragma once
+
+struct PushConstants {
+  uint ResourceDescriptorIndex;
+  uint ResourceDescriptorIndex2;
+  float Param0;
+  float Param1;
+};
+
+[[vk::push_constant]] ConstantBuffer<PushConstants> g_PushConstants
+    : register(b3, space4);
diff --git a/rexlego/src/gpu_native/shaders/copy_vs.hlsl b/rexlego/src/gpu_native/shaders/copy_vs.hlsl
new file mode 100644
index 0000000..359e54f
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders/copy_vs.hlsl
@@ -0,0 +1,6 @@
+void main(in uint vertex_id : SV_VertexID, out float4 position : SV_Position,
+          out float2 tex_coord : TEXCOORD) {
+  tex_coord = float2((vertex_id << 1) & 2, vertex_id & 2);
+  position = float4(tex_coord * float2(2.0, -2.0) + float2(-1.0, 1.0),
+                    0.0, 1.0);
+}

From 686b40f83a79737530bf44a765f88b7b40c4f0c5 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 15:12:48 +0300
Subject: [PATCH 14/44] gpu: upload texture mirrors through frame ring

---
 rexlego/src/gpu_native/buffers.cpp            |  1 +
 rexlego/src/gpu_native/device.cpp             | 78 +++++++++++++---
 rexlego/src/gpu_native/device.h               |  6 ++
 rexlego/src/gpu_native/shaders.cpp            |  1 +
 rexlego/src/gpu_native/textures.cpp           | 88 +++++++++++++++++++
 rexlego/src/gpu_native/textures.h             |  5 ++
 .../src/gpu_native/vertex_declarations.cpp    |  2 +
 7 files changed, 167 insertions(+), 14 deletions(-)

diff --git a/rexlego/src/gpu_native/buffers.cpp b/rexlego/src/gpu_native/buffers.cpp
index a57cf1f..3d8f8d1 100644
--- a/rexlego/src/gpu_native/buffers.cpp
+++ b/rexlego/src/gpu_native/buffers.cpp
@@ -194,6 +194,7 @@ u32 ReleaseNativeBuffer(u32 guest_address) {
     g_buffers.erase(it);
   }
   auto* memory = REX_KERNEL_MEMORY();
+  HostDevice::RetireResource(released);
   memory->SystemHeapFree(released->mirror_address);
   memory->SystemHeapFree(guest_address);
   return 0;
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 49908e2..712ae2f 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -52,7 +52,10 @@ struct State {
   std::vector<std::unique_ptr<plume::RenderCommandSemaphore>>
       render_semaphores;
   std::array<bool, kFramesInFlight> frame_submitted{};
+  std::array<std::vector<std::shared_ptr<void>>, kFramesInFlight>
+      retired_resources;
   u32 frame_slot = 0;
+  bool command_list_open = false;
   u32 long_wait_log_count = 0;
   std::vector<bool> texture_slots;
   Backend backend = Backend::kD3D12;
@@ -190,6 +193,7 @@ bool CreateFrameRing(State& state) {
     state.frame_fences[i] = state.device->createCommandFence();
     state.acquire_semaphores[i] = state.device->createCommandSemaphore();
     state.frame_submitted[i] = false;
+    state.retired_resources[i].clear();
     if (!state.command_lists[i] || !state.frame_fences[i] ||
         !state.acquire_semaphores[i]) {
       return false;
@@ -210,6 +214,34 @@ bool RebuildSwapChain(State& state) {
   return CreateFrameRing(state);
 }
 
+plume::RenderCommandList* BeginFrameCommandsLocked(State& state) {
+  if (state.command_list_open) {
+    return state.command_lists[state.frame_slot].get();
+  }
+  if (state.swap_chain->needsResize() && !RebuildSwapChain(state)) {
+    return nullptr;
+  }
+
+  const u32 slot = state.frame_slot;
+  if (state.frame_submitted[slot]) {
+    const auto wait_start = std::chrono::steady_clock::now();
+    state.queue->waitForCommandFence(state.frame_fences[slot].get());
+    const auto wait_ms = std::chrono::duration<double, std::milli>(
+                             std::chrono::steady_clock::now() - wait_start)
+                             .count();
+    if (wait_ms > 20.0 && state.long_wait_log_count++ < 20) {
+      REXLOG_WARN("Native GPU: frame-slot {} GPU wait took {:.2f} ms", slot,
+                  wait_ms);
+    }
+    state.frame_submitted[slot] = false;
+  }
+  state.retired_resources[slot].clear();
+  auto* commands = state.command_lists[slot].get();
+  commands->begin();
+  state.command_list_open = true;
+  return commands;
+}
+
 }  // namespace
 
 bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
@@ -351,6 +383,26 @@ void HostDevice::UnregisterTexture(u32 descriptor_index) {
   g_state->texture_slots[descriptor_index] = false;
 }
 
+plume::RenderCommandList* HostDevice::BeginFrameCommands() {
+  std::lock_guard lock(g_mutex);
+  if (!g_state) {
+    return nullptr;
+  }
+  return BeginFrameCommandsLocked(*g_state);
+}
+
+void HostDevice::RetireResource(std::shared_ptr<void> resource) {
+  if (!resource) {
+    return;
+  }
+  std::lock_guard lock(g_mutex);
+  if (!g_state) {
+    return;
+  }
+  g_state->retired_resources[g_state->frame_slot].emplace_back(
+      std::move(resource));
+}
+
 bool HostDevice::PresentTexture(plume::RenderTexture* texture,
                                 u32 descriptor_index) {
   if (!texture || descriptor_index == ~u32{0}) {
@@ -364,7 +416,7 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
     return false;
   }
   State& state = *g_state;
-  if (state.swap_chain->needsResize()) {
+  if (!state.command_list_open && state.swap_chain->needsResize()) {
     if (!RebuildSwapChain(state)) {
       REXLOG_WARN("Native GPU: swap chain resize deferred");
       return false;
@@ -372,29 +424,26 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
   }
 
   const u32 slot = state.frame_slot;
-  if (state.frame_submitted[slot]) {
-    const auto wait_start = std::chrono::steady_clock::now();
-    state.queue->waitForCommandFence(state.frame_fences[slot].get());
-    const auto wait_ms = std::chrono::duration<double, std::milli>(
-                             std::chrono::steady_clock::now() - wait_start)
-                             .count();
-    if (wait_ms > 20.0 && state.long_wait_log_count++ < 20) {
-      REXLOG_WARN("Native GPU: frame-slot {} GPU wait took {:.2f} ms", slot,
-                  wait_ms);
-    }
-    state.frame_submitted[slot] = false;
+  plume::RenderCommandList* commands = BeginFrameCommandsLocked(state);
+  if (!commands) {
+    return false;
   }
 
   u32 image_index = 0;
   if (!state.swap_chain->acquireTexture(
           state.acquire_semaphores[slot].get(), &image_index) ||
       image_index >= state.framebuffers.size()) {
+    // Close and submit any recorded uploads/draws so the allocator remains in
+    // a valid state while the window is minimized or the swap chain changes.
+    commands->end();
+    state.command_list_open = false;
+    state.queue->executeCommandLists(commands, state.frame_fences[slot].get());
+    state.frame_submitted[slot] = true;
+    state.frame_slot = (slot + 1) % State::kFramesInFlight;
     return false;
   }
 
-  plume::RenderCommandList* commands = state.command_lists[slot].get();
   plume::RenderTexture* back = state.swap_chain->getTexture(image_index);
-  commands->begin();
   const plume::RenderTextureBarrier barriers[] = {
       plume::RenderTextureBarrier(texture,
                                   plume::RenderTextureLayout::SHADER_READ),
@@ -428,6 +477,7 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
       plume::RenderBarrierStage::NONE,
       plume::RenderTextureBarrier(back, plume::RenderTextureLayout::PRESENT));
   commands->end();
+  state.command_list_open = false;
 
   const plume::RenderCommandList* lists[] = {commands};
   plume::RenderCommandSemaphore* waits[] = {
diff --git a/rexlego/src/gpu_native/device.h b/rexlego/src/gpu_native/device.h
index 39cdbdc..d8d6d51 100644
--- a/rexlego/src/gpu_native/device.h
+++ b/rexlego/src/gpu_native/device.h
@@ -45,6 +45,12 @@ class HostDevice {
   static u32 RegisterTexture(plume::RenderTexture* texture,
                              plume::RenderTextureView* view);
   static void UnregisterTexture(u32 descriptor_index);
+  // Returns the current frame's direct command list, opening a new ring slot
+  // and waiting only when that slot is being reused.
+  static plume::RenderCommandList* BeginFrameCommands();
+  // Keeps a released resource alive through the fence of the command list
+  // which may still reference it.
+  static void RetireResource(std::shared_ptr<void> resource);
   // Copies a single-sample native texture to the host swap chain. This is the
   // first complete submission path and deliberately keeps command allocators
   // in a frame ring so the CPU does not wait for the frame it just submitted.
diff --git a/rexlego/src/gpu_native/shaders.cpp b/rexlego/src/gpu_native/shaders.cpp
index 658802e..d5f7d9c 100644
--- a/rexlego/src/gpu_native/shaders.cpp
+++ b/rexlego/src/gpu_native/shaders.cpp
@@ -192,6 +192,7 @@ u32 ReleaseNativeShader(u32 guest_address) {
     released = std::move(it->second);
     g_registry.erase(it);
   }
+  HostDevice::RetireResource(released);
   REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
   return 0;
 }
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index cf88384..7855be6 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -1,6 +1,7 @@
 #include "gpu_native/textures.h"
 
 #include <algorithm>
+#include <cstring>
 #include <memory>
 #include <mutex>
 #include <unordered_map>
@@ -356,6 +357,92 @@ TextureResourceView ResolveTextureResource(u32 guest_address) {
           IsDepthFormat(resource->format)};
 }
 
+bool UploadTextureResource(u32 guest_address,
+                           plume::RenderCommandList* commands) {
+  const auto resource = FindTexture(guest_address);
+  if (!resource || !commands || resource->surface ||
+      !resource->mirror_address || !resource->mirror_size ||
+      !resource->texture || IsDepthFormat(resource->format)) {
+    return false;
+  }
+  // Cube LockRect has a separate ABI and is not hooked yet. Do not upload the
+  // same 2D mirror into six faces and silently fabricate corrupt data.
+  if (resource->d3d_type ==
+      static_cast<u32>(D3DResourceType::kCubeTexture)) {
+    return false;
+  }
+
+  std::lock_guard resource_lock(resource->mutex);
+  u64 upload_size = 0;
+  for (u32 level = 0; level < resource->levels; ++level) {
+    upload_size = AlignUp(static_cast<u32>(upload_size), 0x200);
+    upload_size += LevelSize(*resource, level);
+  }
+  auto* device = HostDevice::Device();
+  if (!device || upload_size == 0 || upload_size > UINT32_MAX) {
+    return false;
+  }
+  auto upload = std::shared_ptr<plume::RenderBuffer>(
+      device->createBuffer(plume::RenderBufferDesc::UploadBuffer(upload_size))
+          .release());
+  if (!upload) {
+    REXLOG_ERROR("Native GPU: texture upload allocation failed ({} bytes)",
+                 upload_size);
+    return false;
+  }
+
+  auto* mapped = static_cast<u8*>(upload->map());
+  if (!mapped) {
+    return false;
+  }
+  const auto* source =
+      REX_KERNEL_MEMORY()->virtual_membase() + resource->mirror_address;
+  u64 destination_offset = 0;
+  for (u32 level = 0; level < resource->levels; ++level) {
+    destination_offset = AlignUp(static_cast<u32>(destination_offset), 0x200);
+    const u32 level_size = LevelSize(*resource, level);
+    std::memcpy(mapped + destination_offset,
+                source + LevelOffset(*resource, level), level_size);
+    destination_offset += level_size;
+  }
+  upload->unmap();
+
+  commands->barriers(
+      plume::RenderBarrierStage::COPY,
+      plume::RenderTextureBarrier(resource->texture.get(),
+                                  plume::RenderTextureLayout::COPY_DEST));
+  destination_offset = 0;
+  const u32 format_size = plume::RenderFormatSize(resource->format);
+  const u32 block_width = plume::RenderFormatBlockWidth(resource->format);
+  if (!format_size || !block_width) {
+    return false;
+  }
+  for (u32 level = 0; level < resource->levels; ++level) {
+    destination_offset = AlignUp(static_cast<u32>(destination_offset), 0x200);
+    const u32 width = MipDimension(resource->width, level);
+    const u32 height = MipDimension(resource->height, level);
+    const u32 depth = resource->d3d_type ==
+                              static_cast<u32>(D3DResourceType::kVolumeTexture)
+                          ? MipDimension(resource->depth, level)
+                          : 1;
+    const u32 pitch = RowPitch(resource->format, width);
+    const u32 row_width = (pitch / format_size) * block_width;
+    commands->copyTextureRegion(
+        plume::RenderTextureCopyLocation::Subresource(
+            resource->texture.get(), level),
+        plume::RenderTextureCopyLocation::PlacedFootprint(
+            upload.get(), resource->format, width, height, depth, row_width,
+            destination_offset));
+    destination_offset += LevelSize(*resource, level);
+  }
+  commands->barriers(
+      plume::RenderBarrierStage::GRAPHICS,
+      plume::RenderTextureBarrier(resource->texture.get(),
+                                  plume::RenderTextureLayout::SHADER_READ));
+  HostDevice::RetireResource(std::move(upload));
+  return true;
+}
+
 plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
                                              u32 depth_stencil) {
   const auto color = FindTexture(render_target);
@@ -444,6 +531,7 @@ u32 ReleaseNativeTexture(u32 guest_address) {
   auto* memory = REX_KERNEL_MEMORY();
   InvalidateFramebufferReferences(guest_address);
   HostDevice::UnregisterTexture(released->descriptor_index);
+  HostDevice::RetireResource(released);
   if (released->mirror_address) {
     memory->SystemHeapFree(released->mirror_address);
   }
diff --git a/rexlego/src/gpu_native/textures.h b/rexlego/src/gpu_native/textures.h
index 500fbcf..d57aebc 100644
--- a/rexlego/src/gpu_native/textures.h
+++ b/rexlego/src/gpu_native/textures.h
@@ -6,6 +6,7 @@
 
 namespace plume {
 struct RenderFramebuffer;
+struct RenderCommandList;
 struct RenderTexture;
 struct RenderTextureView;
 }
@@ -34,6 +35,10 @@ struct TextureResourceView {
 };
 
 TextureResourceView ResolveTextureResource(u32 guest_address);
+// Copies the complete CPU mirror into the host texture and leaves it in
+// SHADER_READ. The CPU mirror remains authoritative for readback/correctness.
+bool UploadTextureResource(u32 guest_address,
+                           plume::RenderCommandList* commands);
 plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
                                              u32 depth_stencil);
 
diff --git a/rexlego/src/gpu_native/vertex_declarations.cpp b/rexlego/src/gpu_native/vertex_declarations.cpp
index 4b6b14a..329faa9 100644
--- a/rexlego/src/gpu_native/vertex_declarations.cpp
+++ b/rexlego/src/gpu_native/vertex_declarations.cpp
@@ -13,6 +13,7 @@
 #include <rex/system/xmemory.h>
 
 #include "gpu_native/d3d.h"
+#include "gpu_native/device.h"
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -323,6 +324,7 @@ u32 ReleaseNativeVertexDeclaration(u32 guest_address) {
     released = std::move(it->second);
     g_declarations.erase(it);
   }
+  HostDevice::RetireResource(released);
   REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
   return 0;
 }

From b9643aaecb4b3054d8f000b8d5f3e6c9cf788bdb Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 29 Sep 2026 15:20:28 +0300
Subject: [PATCH 15/44] gpu: execute native draws and present resolved frames

---
 rexlego/CMakeLists.txt              |   3 +
 rexlego/src/gpu_native/buffers.cpp  |  10 +
 rexlego/src/gpu_native/buffers.h    |   8 +
 rexlego/src/gpu_native/device.cpp   | 126 +++++++-
 rexlego/src/gpu_native/device.h     |   2 +
 rexlego/src/gpu_native/draw.cpp     | 456 ++++++++++++++++++++++++++++
 rexlego/src/gpu_native/draw.h       |   8 +
 rexlego/src/gpu_native/shaders.cpp  |   6 +
 rexlego/src/gpu_native/shaders.h    |   1 +
 rexlego/src/gpu_native/textures.cpp |  43 +++
 rexlego/src/gpu_native/textures.h   |   4 +
 11 files changed, 666 insertions(+), 1 deletion(-)
 create mode 100644 rexlego/src/gpu_native/draw.cpp
 create mode 100644 rexlego/src/gpu_native/draw.h

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index af415ec..fdc542d 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -57,6 +57,7 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     list(APPEND LEGODIMENSIONS_SOURCES
         src/gpu_native/buffers.cpp
         src/gpu_native/device.cpp
+        src/gpu_native/draw.cpp
         src/gpu_native/format.cpp
         src/gpu_native/hooks_buffer.cpp
         src/gpu_native/hooks_device.cpp
@@ -200,6 +201,8 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     set(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/draw.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/draw.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_common.hlsli"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_vs.hlsl"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_color_ps.hlsl"
diff --git a/rexlego/src/gpu_native/buffers.cpp b/rexlego/src/gpu_native/buffers.cpp
index 3d8f8d1..294d279 100644
--- a/rexlego/src/gpu_native/buffers.cpp
+++ b/rexlego/src/gpu_native/buffers.cpp
@@ -154,6 +154,16 @@ plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind) {
   return resource->buffer.get();
 }
 
+BufferResourceView ResolveBufferResourceView(u32 guest_address,
+                                             BufferKind kind) {
+  const auto resource = FindBuffer(guest_address);
+  if (!resource || resource->kind != kind) {
+    return {};
+  }
+  auto* buffer = ResolveBufferResource(guest_address, kind);
+  return {buffer, resource->length, resource->format};
+}
+
 bool IsNativeBuffer(u32 guest_address) {
   std::lock_guard lock(g_buffers_mutex);
   return g_buffers.contains(guest_address);
diff --git a/rexlego/src/gpu_native/buffers.h b/rexlego/src/gpu_native/buffers.h
index e11d1bf..8afc52c 100644
--- a/rexlego/src/gpu_native/buffers.h
+++ b/rexlego/src/gpu_native/buffers.h
@@ -24,6 +24,14 @@ u32 LockBufferResource(u32 guest_address, u32 offset, u32 size, u32 flags,
 // and later CPU readback remain visible.
 plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind);
 
+struct BufferResourceView {
+  plume::RenderBuffer* buffer = nullptr;
+  u32 length = 0;
+  u32 guest_format = 0;
+};
+BufferResourceView ResolveBufferResourceView(u32 guest_address,
+                                             BufferKind kind);
+
 bool IsNativeBuffer(u32 guest_address);
 u32 NativeBufferType(u32 guest_address);
 u32 AddRefNativeBuffer(u32 guest_address);
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 712ae2f..5e083c8 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -1,6 +1,7 @@
 #include "gpu_native/device.h"
 
 #include "gpu_native/buffers.h"
+#include "gpu_native/draw.h"
 #include "gpu_native/shaders.h"
 #include "gpu_native/textures.h"
 #include "gpu_native/state.h"
@@ -11,6 +12,7 @@
 
 #include <array>
 #include <chrono>
+#include <cstring>
 #include <mutex>
 #include <string>
 #include <vector>
@@ -39,6 +41,11 @@ struct State {
   std::unique_ptr<plume::RenderDescriptorSet> texture_descriptors;
   std::unique_ptr<plume::RenderDescriptorSet> sampler_descriptors;
   std::unique_ptr<plume::RenderSampler> default_sampler;
+  std::unique_ptr<plume::RenderBuffer> null_vertex_buffer;
+  std::array<std::unique_ptr<plume::RenderTexture>, 3> null_textures;
+  std::array<std::unique_ptr<plume::RenderTextureView>, 3> null_texture_views;
+  std::unique_ptr<plume::RenderBuffer> null_texture_upload;
+  bool null_textures_initialized = false;
   std::unique_ptr<plume::RenderShader> copy_vertex_shader;
   std::unique_ptr<plume::RenderShader> copy_pixel_shader;
   std::unique_ptr<plume::RenderPipeline> copy_pipeline;
@@ -203,6 +210,115 @@ bool CreateFrameRing(State& state) {
   return true;
 }
 
+bool CreateNullVertexBuffer(State& state) {
+  state.null_vertex_buffer = state.device->createBuffer(
+      plume::RenderBufferDesc::VertexBuffer(256,
+                                             plume::RenderHeapType::UPLOAD));
+  if (!state.null_vertex_buffer) {
+    return false;
+  }
+  void* mapped = state.null_vertex_buffer->map();
+  if (!mapped) {
+    return false;
+  }
+  std::memset(mapped, 0, 256);
+  state.null_vertex_buffer->unmap();
+  return true;
+}
+
+bool CreateNullTextures(State& state) {
+  const plume::RenderTextureDimension dimensions[] = {
+      plume::RenderTextureDimension::TEXTURE_2D,
+      plume::RenderTextureDimension::TEXTURE_3D,
+      plume::RenderTextureDimension::TEXTURE_2D};
+  const plume::RenderTextureViewDimension view_dimensions[] = {
+      plume::RenderTextureViewDimension::TEXTURE_2D,
+      plume::RenderTextureViewDimension::TEXTURE_3D,
+      plume::RenderTextureViewDimension::TEXTURE_CUBE};
+  for (u32 i = 0; i < 3; ++i) {
+    plume::RenderTextureDesc desc;
+    desc.dimension = dimensions[i];
+    desc.width = desc.height = desc.depth = desc.mipLevels = 1;
+    desc.arraySize = i == 2 ? 6 : 1;
+    desc.format = plume::RenderFormat::R8G8B8A8_UNORM;
+    if (i == 2) {
+      desc.flags = plume::RenderTextureFlag::CUBE;
+    }
+    state.null_textures[i] = state.device->createTexture(desc);
+    if (!state.null_textures[i]) {
+      return false;
+    }
+    plume::RenderTextureViewDesc view_desc;
+    view_desc.format = desc.format;
+    view_desc.dimension = view_dimensions[i];
+    view_desc.mipLevels = 1;
+    state.null_texture_views[i] =
+        state.null_textures[i]->createTextureView(view_desc);
+    if (!state.null_texture_views[i]) {
+      return false;
+    }
+    state.texture_descriptors->setTexture(
+        i, state.null_textures[i].get(),
+        plume::RenderTextureLayout::SHADER_READ,
+        state.null_texture_views[i].get());
+  }
+  // Six cube faces plus the 2D and 3D resources, each with a 512-byte aligned
+  // placed footprint.
+  state.null_texture_upload = state.device->createBuffer(
+      plume::RenderBufferDesc::UploadBuffer(8 * 0x200));
+  if (!state.null_texture_upload) {
+    return false;
+  }
+  void* mapped = state.null_texture_upload->map();
+  if (!mapped) {
+    return false;
+  }
+  std::memset(mapped, 0, 8 * 0x200);
+  state.null_texture_upload->unmap();
+  return true;
+}
+
+void InitializeNullTextures(State& state, plume::RenderCommandList* commands) {
+  if (state.null_textures_initialized) {
+    return;
+  }
+  plume::RenderTextureBarrier pre[3];
+  for (u32 i = 0; i < 3; ++i) {
+    pre[i] = plume::RenderTextureBarrier(
+        state.null_textures[i].get(), plume::RenderTextureLayout::COPY_DEST);
+  }
+  commands->barriers(plume::RenderBarrierStage::COPY, pre, 3);
+  u32 footprint = 0;
+  commands->copyTextureRegion(
+      plume::RenderTextureCopyLocation::Subresource(
+          state.null_textures[0].get()),
+      plume::RenderTextureCopyLocation::PlacedFootprint(
+          state.null_texture_upload.get(), plume::RenderFormat::R8G8B8A8_UNORM,
+          1, 1, 1, 64, footprint++ * 0x200));
+  commands->copyTextureRegion(
+      plume::RenderTextureCopyLocation::Subresource(
+          state.null_textures[1].get()),
+      plume::RenderTextureCopyLocation::PlacedFootprint(
+          state.null_texture_upload.get(), plume::RenderFormat::R8G8B8A8_UNORM,
+          1, 1, 1, 64, footprint++ * 0x200));
+  for (u32 face = 0; face < 6; ++face) {
+    commands->copyTextureRegion(
+        plume::RenderTextureCopyLocation::Subresource(
+            state.null_textures[2].get(), 0, face),
+        plume::RenderTextureCopyLocation::PlacedFootprint(
+            state.null_texture_upload.get(),
+            plume::RenderFormat::R8G8B8A8_UNORM, 1, 1, 1, 64,
+            footprint++ * 0x200));
+  }
+  plume::RenderTextureBarrier post[3];
+  for (u32 i = 0; i < 3; ++i) {
+    post[i] = plume::RenderTextureBarrier(
+        state.null_textures[i].get(), plume::RenderTextureLayout::SHADER_READ);
+  }
+  commands->barriers(plume::RenderBarrierStage::GRAPHICS, post, 3);
+  state.null_textures_initialized = true;
+}
+
 bool RebuildSwapChain(State& state) {
   state.swap_chain->wait();
   state.framebuffers.clear();
@@ -238,6 +354,7 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) {
   state.retired_resources[slot].clear();
   auto* commands = state.command_lists[slot].get();
   commands->begin();
+  InitializeNullTextures(state, commands);
   state.command_list_open = true;
   return commands;
 }
@@ -285,7 +402,8 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
     REXLOG_ERROR("Native GPU: failed to create or size the swap chain");
     return false;
   }
-  if (!CreatePresentPipeline(*state) || !CreateFrameRing(*state)) {
+  if (!CreatePresentPipeline(*state) || !CreateNullVertexBuffer(*state) ||
+      !CreateNullTextures(*state) || !CreateFrameRing(*state)) {
     REXLOG_ERROR("Native GPU: failed to create present pipeline or frame ring");
     return false;
   }
@@ -308,6 +426,7 @@ void HostDevice::Shutdown() {
     g_state->swap_chain->wait();
   }
   // All GPU users must be idle before Plume resources are released.
+  ResetDrawResources();
   ResetShaderResources();
   ResetBufferResources();
   ResetTextureResources();
@@ -351,6 +470,11 @@ plume::RenderDescriptorSet* HostDevice::SamplerDescriptorSet() {
   return g_state ? g_state->sampler_descriptors.get() : nullptr;
 }
 
+plume::RenderBuffer* HostDevice::NullVertexBuffer() {
+  std::lock_guard lock(g_mutex);
+  return g_state ? g_state->null_vertex_buffer.get() : nullptr;
+}
+
 u32 HostDevice::RegisterTexture(plume::RenderTexture* texture,
                                 plume::RenderTextureView* view) {
   if (!texture || !view) {
diff --git a/rexlego/src/gpu_native/device.h b/rexlego/src/gpu_native/device.h
index d8d6d51..b3a79f6 100644
--- a/rexlego/src/gpu_native/device.h
+++ b/rexlego/src/gpu_native/device.h
@@ -7,6 +7,7 @@
 #include <rex/types.h>
 
 namespace plume {
+struct RenderBuffer;
 struct RenderCommandFence;
 struct RenderCommandQueue;
 struct RenderCommandList;
@@ -42,6 +43,7 @@ class HostDevice {
   static plume::RenderPipelineLayout* PipelineLayout();
   static plume::RenderDescriptorSet* TextureDescriptorSet();
   static plume::RenderDescriptorSet* SamplerDescriptorSet();
+  static plume::RenderBuffer* NullVertexBuffer();
   static u32 RegisterTexture(plume::RenderTexture* texture,
                              plume::RenderTextureView* view);
   static void UnregisterTexture(u32 descriptor_index);
diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
new file mode 100644
index 0000000..1c3de5f
--- /dev/null
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -0,0 +1,456 @@
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+
+#include <plume_render_interface.h>
+#include <rex/graphics/xenos.h>
+#include <rex/hook.h>
+#include <rex/logging.h>
+#include <rex/system/kernel_state.h>
+#include <rex/types.h>
+
+#include "gpu_native/buffers.h"
+#include "gpu_native/d3d.h"
+#include "gpu_native/device.h"
+#include "gpu_native/shaders.h"
+#include "gpu_native/state.h"
+#include "gpu_native/textures.h"
+#include "gpu_native/vertex_declarations.h"
+
+namespace legodimensions::gpu_native {
+namespace {
+
+struct SharedConstants {
+  u32 texture_2d[16]{};
+  u32 texture_3d[16]{};
+  u32 texture_cube[16]{};
+  u32 samplers[16]{};
+  u32 booleans[8]{};
+  u32 swapped_texcoords = 0;
+  float half_pixel_x = 0.0f;
+  float half_pixel_y = 0.0f;
+  float alpha_threshold = 0.0f;
+  u32 swapped_normals = 0;
+  u32 swapped_binormals = 0;
+  u32 swapped_tangents = 0;
+  u32 swapped_blend_weights = 0;
+  u32 swapped_positions = 0;
+  u32 sint_texcoords = 0;
+  float shadow_pcf_scale = 1.0f;
+  u32 padding = 0;
+  float blit_half_pixel_x = 0.0f;
+  float blit_half_pixel_y = 0.0f;
+  u32 padding_2[2]{};
+};
+static_assert(sizeof(SharedConstants) == 352);
+
+struct PipelineKey {
+  u32 vertex_shader = 0;
+  u32 pixel_shader = 0;
+  u32 declaration = 0;
+  u32 render_target_format = 0;
+  u32 depth_format = 0;
+  u32 topology = 0;
+  u32 spec_constants = 0;
+  std::array<u32, kNativeVertexStreams> strides{};
+  bool operator==(const PipelineKey&) const = default;
+};
+
+struct PipelineKeyHash {
+  size_t operator()(const PipelineKey& key) const {
+    size_t hash = 1469598103934665603ull;
+    const auto* bytes = reinterpret_cast<const u8*>(&key);
+    for (size_t i = 0; i < sizeof(key); ++i) {
+      hash = (hash ^ bytes[i]) * 1099511628211ull;
+    }
+    return hash;
+  }
+};
+
+std::mutex g_pipeline_mutex;
+std::unordered_map<PipelineKey, std::unique_ptr<plume::RenderPipeline>,
+                   PipelineKeyHash>
+    g_pipelines;
+u32 g_unsupported_draw_logs = 0;
+
+bool MapTopology(u32 primitive_type, plume::RenderPrimitiveTopology& result) {
+  using rex::graphics::xenos::PrimitiveType;
+  switch (static_cast<PrimitiveType>(primitive_type)) {
+    case PrimitiveType::kPointList:
+      result = plume::RenderPrimitiveTopology::POINT_LIST;
+      return true;
+    case PrimitiveType::kLineList:
+      result = plume::RenderPrimitiveTopology::LINE_LIST;
+      return true;
+    case PrimitiveType::kLineStrip:
+      result = plume::RenderPrimitiveTopology::LINE_STRIP;
+      return true;
+    case PrimitiveType::kTriangleList:
+      result = plume::RenderPrimitiveTopology::TRIANGLE_LIST;
+      return true;
+    case PrimitiveType::kTriangleStrip:
+      result = plume::RenderPrimitiveTopology::TRIANGLE_STRIP;
+      return true;
+    default:
+      return false;
+  }
+}
+
+void CopyBigEndianDwords(void* destination, const void* source, u32 size) {
+  auto* dst = static_cast<u32*>(destination);
+  const auto* src = static_cast<const be_u32*>(source);
+  for (u32 i = 0; i < size / sizeof(u32); ++i) {
+    dst[i] = src[i];
+  }
+}
+
+std::shared_ptr<plume::RenderBuffer> BindConstants(
+    plume::RenderCommandList* commands, const D3DDevice* device,
+    const DrawBindings& bindings, const VertexDeclarationView& declaration) {
+  constexpr u32 kConstantsSize = 0x1000;
+  constexpr u32 kSharedOffset = kConstantsSize * 2;
+  auto* host = HostDevice::Device();
+  if (!host) {
+    return {};
+  }
+  auto upload = std::shared_ptr<plume::RenderBuffer>(
+      host->createBuffer(plume::RenderBufferDesc::UploadBuffer(
+                             kSharedOffset + sizeof(SharedConstants)))
+          .release());
+  if (!upload) {
+    return {};
+  }
+  auto* mapped = static_cast<u8*>(upload->map());
+  if (!mapped) {
+    return {};
+  }
+  const auto* bytes = reinterpret_cast<const u8*>(device);
+  CopyBigEndianDwords(mapped, bytes + 0x700, kConstantsSize);
+  CopyBigEndianDwords(mapped + kConstantsSize, bytes + 0x1700,
+                      kConstantsSize);
+
+  SharedConstants shared;
+  std::fill(std::begin(shared.texture_3d), std::end(shared.texture_3d), 1u);
+  std::fill(std::begin(shared.texture_cube), std::end(shared.texture_cube), 2u);
+  const auto* bools = reinterpret_cast<const be_u32*>(bytes + 0x2700);
+  for (u32 i = 0; i < 8; ++i) {
+    shared.booleans[i] = bools[i];
+  }
+  shared.swapped_texcoords = declaration.swapped_texcoords;
+  shared.swapped_normals = declaration.swapped_normals;
+  shared.swapped_binormals = declaration.swapped_binormals;
+  shared.swapped_tangents = declaration.swapped_tangents;
+  shared.swapped_blend_weights = declaration.swapped_blend_weights;
+  shared.swapped_positions = declaration.swapped_positions;
+  shared.sint_texcoords = declaration.sint_texcoords;
+  for (u32 i = 0; i < kNativeTextureSlots; ++i) {
+    const auto texture = ResolveTextureResource(bindings.textures[i]);
+    if (!texture.texture || texture.descriptor_index == ~u32{0}) {
+      continue;
+    }
+    UploadTextureResource(bindings.textures[i], commands);
+    if (texture.d3d_type ==
+        static_cast<u32>(D3DResourceType::kVolumeTexture)) {
+      shared.texture_3d[i] = texture.descriptor_index;
+    } else if (texture.d3d_type ==
+               static_cast<u32>(D3DResourceType::kCubeTexture)) {
+      shared.texture_cube[i] = texture.descriptor_index;
+    } else {
+      shared.texture_2d[i] = texture.descriptor_index;
+    }
+  }
+  std::memcpy(mapped + kSharedOffset, &shared, sizeof(shared));
+  upload->unmap();
+  commands->setGraphicsRootDescriptor(upload->at(0), 0);
+  commands->setGraphicsRootDescriptor(upload->at(kConstantsSize), 1);
+  commands->setGraphicsRootDescriptor(upload->at(kSharedOffset), 2);
+  return upload;
+}
+
+plume::RenderPipeline* GetPipeline(
+    const PipelineKey& key, const VertexDeclarationView& declaration,
+    plume::RenderPrimitiveTopology topology, plume::RenderFormat color_format,
+    plume::RenderFormat depth_format) {
+  std::lock_guard lock(g_pipeline_mutex);
+  if (const auto it = g_pipelines.find(key); it != g_pipelines.end()) {
+    return it->second.get();
+  }
+  auto* vs = ResolveBoundShader(ShaderStage::kVertex, key.spec_constants);
+  auto* ps = ResolveBoundShader(ShaderStage::kPixel, key.spec_constants);
+  if (!vs || (key.pixel_shader && !ps)) {
+    return nullptr;
+  }
+  std::array<plume::RenderInputSlot, kNativeVertexStreams> slots;
+  for (u32 i = 0; i < slots.size(); ++i) {
+    slots[i] = plume::RenderInputSlot(i, key.strides[i] ? key.strides[i] : 16);
+  }
+  plume::RenderGraphicsPipelineDesc desc;
+  desc.pipelineLayout = HostDevice::PipelineLayout();
+  desc.vertexShader = vs;
+  desc.pixelShader = ps;
+  desc.depthFunction = plume::RenderComparisonFunction::LESS_EQUAL;
+  desc.depthEnabled = depth_format != plume::RenderFormat::UNKNOWN;
+  desc.depthWriteEnabled = desc.depthEnabled;
+  desc.depthClipEnabled = true;
+  desc.primitiveTopology = topology;
+  desc.cullMode = plume::RenderCullMode::NONE;
+  desc.fillMode = plume::RenderFillMode::SOLID;
+  desc.renderTargetCount = color_format == plume::RenderFormat::UNKNOWN ? 0 : 1;
+  desc.renderTargetFormat[0] = color_format;
+  desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy();
+  desc.depthTargetFormat = depth_format;
+  desc.inputElements = declaration.elements;
+  desc.inputElementsCount = declaration.element_count;
+  desc.inputSlots = slots.data();
+  desc.inputSlotsCount = static_cast<u32>(slots.size());
+  auto pipeline = HostDevice::Device()->createGraphicsPipeline(desc);
+  if (!pipeline) {
+    return nullptr;
+  }
+  auto* result = pipeline.get();
+  g_pipelines.emplace(key, std::move(pipeline));
+  return result;
+}
+
+bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
+                  u32 start, u32 count, i32 base_vertex) {
+  if (!device || !count) {
+    return false;
+  }
+  plume::RenderPrimitiveTopology topology;
+  if (!MapTopology(primitive_type, topology)) {
+    if (g_unsupported_draw_logs++ < 20) {
+      REXLOG_WARN("Native GPU: skipped unsupported primitive type {}",
+                  primitive_type);
+    }
+    return false;
+  }
+  const DrawBindings bindings = SnapshotDrawBindings();
+  const auto declaration = ResolveVertexDeclaration(bindings.vertex_declaration);
+  const auto color = ResolveTextureResource(bindings.render_targets[0]);
+  const auto depth = ResolveTextureResource(bindings.depth_stencil);
+  auto* framebuffer = ResolveFramebuffer(bindings.render_targets[0],
+                                         bindings.depth_stencil);
+  if (!declaration.supported || !declaration.elements || !framebuffer ||
+      (!color.texture && !depth.texture)) {
+    if (g_unsupported_draw_logs++ < 20) {
+      REXLOG_WARN("Native GPU: skipped draw with incomplete bindings "
+                  "(decl=0x{:08X}, rt=0x{:08X}, ds=0x{:08X})",
+                  bindings.vertex_declaration, bindings.render_targets[0],
+                  bindings.depth_stencil);
+    }
+    return false;
+  }
+  auto* commands = HostDevice::BeginFrameCommands();
+  if (!commands) {
+    return false;
+  }
+
+  std::array<plume::RenderTextureBarrier, 2> texture_barriers;
+  u32 barrier_count = 0;
+  if (color.texture) {
+    texture_barriers[barrier_count++] = plume::RenderTextureBarrier(
+        color.texture, plume::RenderTextureLayout::COLOR_WRITE);
+  }
+  if (depth.texture) {
+    texture_barriers[barrier_count++] = plume::RenderTextureBarrier(
+        depth.texture, plume::RenderTextureLayout::DEPTH_WRITE);
+  }
+  commands->barriers(plume::RenderBarrierStage::GRAPHICS,
+                     texture_barriers.data(), barrier_count);
+  commands->setFramebuffer(framebuffer);
+
+  PipelineKey key;
+  key.vertex_shader = BoundShaderAddress(ShaderStage::kVertex);
+  key.pixel_shader = BoundShaderAddress(ShaderStage::kPixel);
+  key.declaration = bindings.vertex_declaration;
+  key.render_target_format = static_cast<u32>(color.format);
+  key.depth_format = static_cast<u32>(depth.format);
+  key.topology = static_cast<u32>(topology);
+  key.spec_constants = declaration.has_r11g11b10_normal ? 1u : 0u;
+  for (u32 i = 0; i < kNativeVertexStreams; ++i) {
+    key.strides[i] = bindings.vertex_streams[i].stride;
+  }
+  auto* pipeline = GetPipeline(key, declaration, topology, color.format,
+                               depth.format);
+  if (!pipeline) {
+    return false;
+  }
+
+  commands->setGraphicsPipelineLayout(HostDevice::PipelineLayout());
+  commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 0);
+  commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 1);
+  commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 2);
+  commands->setGraphicsDescriptorSet(HostDevice::SamplerDescriptorSet(), 3);
+  auto constants = BindConstants(commands, device, bindings, declaration);
+  if (!constants) {
+    return false;
+  }
+  commands->setPipeline(pipeline);
+
+  const float viewport_width = static_cast<u32>(device->viewport.width);
+  const float viewport_height = static_cast<u32>(device->viewport.height);
+  commands->setViewports(plume::RenderViewport(
+      static_cast<u32>(device->viewport.x), static_cast<u32>(device->viewport.y),
+      viewport_width > 0 ? viewport_width : float(color.width),
+      viewport_height > 0 ? viewport_height : float(color.height),
+      static_cast<float>(device->viewport.min_z),
+      static_cast<float>(device->viewport.max_z)));
+  const i32 scissor_right = static_cast<i32>(device->scissor.right);
+  const i32 scissor_bottom = static_cast<i32>(device->scissor.bottom);
+  commands->setScissors(plume::RenderRect(
+      static_cast<i32>(device->scissor.left),
+      static_cast<i32>(device->scissor.top),
+      scissor_right > 0 ? scissor_right : static_cast<i32>(color.width),
+      scissor_bottom > 0 ? scissor_bottom : static_cast<i32>(color.height)));
+
+  std::array<plume::RenderVertexBufferView, kNativeVertexStreams> views;
+  std::array<plume::RenderInputSlot, kNativeVertexStreams> slots;
+  auto* null_buffer = HostDevice::NullVertexBuffer();
+  for (u32 i = 0; i < kNativeVertexStreams; ++i) {
+    const auto& binding = bindings.vertex_streams[i];
+    const auto buffer = ResolveBufferResourceView(binding.buffer,
+                                                  BufferKind::kVertex);
+    const u32 offset = std::min(binding.offset, buffer.length);
+    views[i] = buffer.buffer
+                   ? plume::RenderVertexBufferView(buffer.buffer->at(offset),
+                                                   buffer.length - offset)
+                   : plume::RenderVertexBufferView(null_buffer->at(0), 256);
+    slots[i] = plume::RenderInputSlot(i, binding.stride ? binding.stride : 16);
+  }
+  commands->setVertexBuffers(0, views.data(), views.size(), slots.data());
+
+  if (indexed) {
+    const auto index = ResolveBufferResourceView(bindings.index_buffer,
+                                                 BufferKind::kIndex);
+    if (!index.buffer) {
+      return false;
+    }
+    const plume::RenderFormat index_format =
+        index.guest_format == 1 ? plume::RenderFormat::R16_UINT
+                                : plume::RenderFormat::R32_UINT;
+    const plume::RenderIndexBufferView index_view(index.buffer->at(0),
+                                                  index.length, index_format);
+    commands->setIndexBuffer(&index_view);
+    commands->drawIndexedInstanced(count, 1, start, base_vertex, 0);
+  } else {
+    commands->drawInstanced(count, 1, start, 0);
+  }
+  HostDevice::RetireResource(std::move(constants));
+  return true;
+}
+
+u32 ClearHook(D3DDevice* /*device*/, u32 rectangle_count,
+              u32 rectangles_address, u32 flags, u32 color, f64 depth,
+              u32 /*depth_gpr_slot*/, u32 stencil, u32 /*edram_clear*/) {
+  const DrawBindings bindings = SnapshotDrawBindings();
+  const auto target = ResolveTextureResource(bindings.render_targets[0]);
+  const auto depth_target = ResolveTextureResource(bindings.depth_stencil);
+  auto* framebuffer = ResolveFramebuffer(bindings.render_targets[0],
+                                         bindings.depth_stencil);
+  auto* commands = HostDevice::BeginFrameCommands();
+  if (!framebuffer || !commands) {
+    return 0;
+  }
+  std::array<plume::RenderTextureBarrier, 2> barriers;
+  u32 barrier_count = 0;
+  if (target.texture) {
+    barriers[barrier_count++] = plume::RenderTextureBarrier(
+        target.texture, plume::RenderTextureLayout::COLOR_WRITE);
+  }
+  if (depth_target.texture) {
+    barriers[barrier_count++] = plume::RenderTextureBarrier(
+        depth_target.texture, plume::RenderTextureLayout::DEPTH_WRITE);
+  }
+  commands->barriers(plume::RenderBarrierStage::GRAPHICS, barriers.data(),
+                     barrier_count);
+  commands->setFramebuffer(framebuffer);
+
+  std::vector<plume::RenderRect> rectangles;
+  if (rectangle_count && rectangles_address) {
+    const auto* source = reinterpret_cast<const D3DRect*>(
+        REX_KERNEL_MEMORY()->virtual_membase() + rectangles_address);
+    rectangles.reserve(rectangle_count);
+    for (u32 i = 0; i < rectangle_count; ++i) {
+      rectangles.emplace_back(static_cast<i32>(source[i].left),
+                              static_cast<i32>(source[i].top),
+                              static_cast<i32>(source[i].right),
+                              static_cast<i32>(source[i].bottom));
+    }
+  }
+  const plume::RenderRect* clear_rectangles =
+      rectangles.empty() ? nullptr : rectangles.data();
+  const u32 clear_rectangle_count = static_cast<u32>(rectangles.size());
+  if ((flags & 0x1) && target.texture) {
+    const float scale = 1.0f / 255.0f;
+    commands->clearColor(
+        0,
+        plume::RenderColor(float((color >> 16) & 0xFF) * scale,
+                           float((color >> 8) & 0xFF) * scale,
+                           float(color & 0xFF) * scale,
+                           float((color >> 24) & 0xFF) * scale),
+        clear_rectangles, clear_rectangle_count);
+  }
+  if ((flags & 0x30) && depth_target.texture) {
+    commands->clearDepthStencil((flags & 0x10) != 0, (flags & 0x20) != 0,
+                                static_cast<float>(depth), stencil,
+                                clear_rectangles, clear_rectangle_count);
+  }
+  return 0;
+}
+
+u32 DrawVerticesHook(D3DDevice* device, u32 primitive_type, u32 start_vertex,
+                     u32 vertex_count) {
+  DispatchDraw(device, primitive_type, false, start_vertex, vertex_count, 0);
+  return 0;
+}
+
+u32 DrawIndexedVerticesHook(D3DDevice* device, u32 primitive_type,
+                            u32 base_vertex, u32 start_index, u32 index_count) {
+  DispatchDraw(device, primitive_type, true, start_index, index_count,
+               static_cast<i32>(base_vertex));
+  return 0;
+}
+
+u32 ResolveHook(D3DDevice* /*device*/, u32 /*flags*/, u32 /*source_rect*/,
+                u32 destination_texture, u32 /*destination_point*/,
+                u32 destination_level, u32 destination_slice,
+                u32 /*clear_color*/, u32 /*clear_z_hi*/, u32 /*clear_z_lo*/,
+                u32 /*clear_stencil*/, u32 /*parameters*/) {
+  if (!ResolveTextureFromSurface(destination_texture, BoundRenderTarget(0),
+                                 destination_level, destination_slice) &&
+      g_unsupported_draw_logs++ < 20) {
+    REXLOG_WARN("Native GPU: skipped unsupported resolve to 0x{:08X}",
+                destination_texture);
+  }
+  return 0;
+}
+
+u32 SwapHook(D3DDevice* /*device*/, u32 front_buffer, u32 /*parameters*/) {
+  const auto texture = ResolveTextureResource(front_buffer);
+  if (!HostDevice::PresentTexture(texture.texture, texture.descriptor_index) &&
+      g_unsupported_draw_logs++ < 20) {
+    REXLOG_WARN("Native GPU: skipped present for front buffer 0x{:08X}",
+                front_buffer);
+  }
+  return 0;
+}
+
+}  // namespace
+
+void ResetDrawResources() {
+  std::lock_guard lock(g_pipeline_mutex);
+  g_pipelines.clear();
+}
+
+}  // namespace legodimensions::gpu_native
+
+REX_HOOK(sub_83FC6640, legodimensions::gpu_native::DrawVerticesHook);
+REX_HOOK(sub_83FC6A58, legodimensions::gpu_native::DrawIndexedVerticesHook);
+REX_HOOK(sub_83FBF0D8, legodimensions::gpu_native::ResolveHook);
+REX_HOOK(sub_83FB33E0, legodimensions::gpu_native::SwapHook);
+REX_HOOK(sub_83FBC9D8, legodimensions::gpu_native::ClearHook);
diff --git a/rexlego/src/gpu_native/draw.h b/rexlego/src/gpu_native/draw.h
new file mode 100644
index 0000000..efaed6d
--- /dev/null
+++ b/rexlego/src/gpu_native/draw.h
@@ -0,0 +1,8 @@
+#pragma once
+
+namespace legodimensions::gpu_native {
+
+// Must run before shader/device teardown because cached PSOs refer to both.
+void ResetDrawResources();
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/shaders.cpp b/rexlego/src/gpu_native/shaders.cpp
index d5f7d9c..d5c6fff 100644
--- a/rexlego/src/gpu_native/shaders.cpp
+++ b/rexlego/src/gpu_native/shaders.cpp
@@ -158,6 +158,12 @@ plume::RenderShader* ResolveBoundShader(ShaderStage stage, u32 spec_constants) {
   return it->second.get();
 }
 
+u32 BoundShaderAddress(ShaderStage stage) {
+  std::lock_guard lock(g_registry_mutex);
+  const auto& resource = g_bound_shaders[StageIndex(stage)];
+  return resource ? resource->guest_address : 0;
+}
+
 bool IsNativeShader(u32 guest_address) {
   std::lock_guard lock(g_registry_mutex);
   return g_registry.contains(guest_address);
diff --git a/rexlego/src/gpu_native/shaders.h b/rexlego/src/gpu_native/shaders.h
index 3631fba..10ee448 100644
--- a/rexlego/src/gpu_native/shaders.h
+++ b/rexlego/src/gpu_native/shaders.h
@@ -26,6 +26,7 @@ bool BindShader(ShaderStage stage, u32 guest_address);
 // Materializes the correct prelinked specialization at pipeline creation time.
 // A null result marks the containing render pass as unsupported by native GPU.
 plume::RenderShader* ResolveBoundShader(ShaderStage stage, u32 spec_constants);
+u32 BoundShaderAddress(ShaderStage stage);
 
 bool IsNativeShader(u32 guest_address);
 u32 AddRefNativeShader(u32 guest_address);
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index 7855be6..4fbcccc 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -353,6 +353,7 @@ TextureResourceView ResolveTextureResource(u32 guest_address) {
           resource->descriptor_index,
           resource->width,
           resource->height,
+          resource->d3d_type,
           resource->surface,
           IsDepthFormat(resource->format)};
 }
@@ -443,6 +444,48 @@ bool UploadTextureResource(u32 guest_address,
   return true;
 }
 
+bool ResolveTextureFromSurface(u32 destination_texture, u32 source_surface,
+                               u32 destination_level,
+                               u32 destination_slice) {
+  const auto destination = FindTexture(destination_texture);
+  const auto source = FindTexture(source_surface);
+  if (!destination || !source || !destination->texture || !source->texture ||
+      !source->surface || destination->surface ||
+      destination_level >= destination->levels ||
+      destination->format != source->format) {
+    return false;
+  }
+  const bool cube = destination->d3d_type ==
+                    static_cast<u32>(D3DResourceType::kCubeTexture);
+  if ((!cube && destination_slice != 0) || (cube && destination_slice >= 6)) {
+    return false;
+  }
+  auto* commands = HostDevice::BeginFrameCommands();
+  if (!commands) {
+    return false;
+  }
+  const plume::RenderTextureBarrier barriers[] = {
+      plume::RenderTextureBarrier(source->texture.get(),
+                                  plume::RenderTextureLayout::COPY_SOURCE),
+      plume::RenderTextureBarrier(destination->texture.get(),
+                                  plume::RenderTextureLayout::COPY_DEST),
+  };
+  commands->barriers(plume::RenderBarrierStage::COPY, barriers, 2);
+  commands->copyTextureRegion(
+      plume::RenderTextureCopyLocation::Subresource(
+          destination->texture.get(), destination_level, destination_slice),
+      plume::RenderTextureCopyLocation::Subresource(source->texture.get(), 0,
+                                                    0));
+  const plume::RenderTextureBarrier final_barriers[] = {
+      plume::RenderTextureBarrier(source->texture.get(),
+                                  plume::RenderTextureLayout::COLOR_WRITE),
+      plume::RenderTextureBarrier(destination->texture.get(),
+                                  plume::RenderTextureLayout::SHADER_READ),
+  };
+  commands->barriers(plume::RenderBarrierStage::GRAPHICS, final_barriers, 2);
+  return true;
+}
+
 plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
                                              u32 depth_stencil) {
   const auto color = FindTexture(render_target);
diff --git a/rexlego/src/gpu_native/textures.h b/rexlego/src/gpu_native/textures.h
index d57aebc..66b4086 100644
--- a/rexlego/src/gpu_native/textures.h
+++ b/rexlego/src/gpu_native/textures.h
@@ -30,6 +30,7 @@ struct TextureResourceView {
   u32 descriptor_index = ~u32{0};
   u32 width = 0;
   u32 height = 0;
+  u32 d3d_type = 0;
   bool surface = false;
   bool depth = false;
 };
@@ -39,6 +40,9 @@ TextureResourceView ResolveTextureResource(u32 guest_address);
 // SHADER_READ. The CPU mirror remains authoritative for readback/correctness.
 bool UploadTextureResource(u32 guest_address,
                            plume::RenderCommandList* commands);
+bool ResolveTextureFromSurface(u32 destination_texture, u32 source_surface,
+                               u32 destination_level,
+                               u32 destination_slice);
 plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
                                              u32 depth_stencil);
 

From 8fa8798480f99d6b18e8fab2d8e5a8398f5a1b4b Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 20:36:32 +0300
Subject: [PATCH 16/44] Preserve native GPU investigations, probes and
 regression harnesses

---
 tools/gpu-aot/README.md                       |  437 +++++-
 tools/gpu-aot/alpha_compare.hlsli             |   16 +
 tools/gpu-aot/audit_d3d_paths.py              |   77 ++
 tools/gpu-aot/audit_sampler_defaults.py       |  110 ++
 tools/gpu-aot/audit_shader_semantics.py       |  125 ++
 tools/gpu-aot/compare_placement_shader.py     |   53 +
 tools/gpu-aot/d3d-hook-map.md                 |   11 +
 tools/gpu-aot/extract_sampler_oracle.py       |   25 +
 tools/gpu-aot/generate_microcode_index.cpp    |  115 ++
 tools/gpu-aot/inspect_gpu_snapshot.py         |   53 +
 tools/gpu-aot/match_mesh_assets.py            |   64 +
 tools/gpu-aot/merge_microcode_index.py        |   58 +
 .../native-depth-alias-findings-20261001.md   |   59 +
 tools/gpu-aot/native-performance-20261001.md  |  159 +++
 tools/gpu-aot/patches/xenosrecomp-lego.patch  |  327 ++++-
 tools/gpu-aot/prepare_alpha_candidate.py      |   84 ++
 tools/gpu-aot/prepare_lod_candidate.py        |  156 +++
 tools/gpu-aot/prepare_viewport_candidate.py   |   53 +
 tools/gpu-aot/questions-pending-20260930.md   |  264 ++++
 tools/gpu-aot/run_native_probe.ps1            |  156 +++
 tools/gpu-aot/sample_native_process.ps1       |   43 +
 tools/gpu-aot/simpsons-adoption.md            |   98 ++
 tools/gpu-aot/summarize_brick_resources.py    |   36 +
 tools/gpu-aot/summarize_frame_metrics.py      |   48 +
 tools/gpu-aot/summarize_long_probe.py         |   36 +
 tools/gpu-aot/summarize_mesh_trace.py         |   33 +
 tools/gpu-aot/test_alpha_test.py              |  177 +++
 tools/gpu-aot/test_audit_d3d_paths.py         |   43 +
 tools/gpu-aot/test_blend_state.cpp            |   17 +
 tools/gpu-aot/test_buffer_header.cpp          |   14 +
 tools/gpu-aot/test_buffer_variants.py         |  214 +++
 tools/gpu-aot/test_bulk_attachments.py        |   73 +
 tools/gpu-aot/test_completion_callbacks.py    |  112 ++
 tools/gpu-aot/test_cpu_resource_wait.py       |   80 ++
 tools/gpu-aot/test_depth_alias.py             |  100 ++
 tools/gpu-aot/test_depth_export.py            |   72 +
 tools/gpu-aot/test_descriptor_retirement.cpp  |   26 +
 tools/gpu-aot/test_draw_upload.py             |  113 ++
 tools/gpu-aot/test_hash_throughput.cpp        |   31 +
 tools/gpu-aot/test_inline_hash.py             |   53 +
 tools/gpu-aot/test_long_probe.py              |   58 +
 tools/gpu-aot/test_memory_watch.py            |  129 ++
 tools/gpu-aot/test_memory_watch_sdk.py        |   70 +
 tools/gpu-aot/test_pixel_shader_filter.cpp    |   10 +
 tools/gpu-aot/test_placement_resource_type.py |   88 ++
 tools/gpu-aot/test_pool_copy.cpp              |   34 +
 tools/gpu-aot/test_query_retirement.py        |  150 ++
 tools/gpu-aot/test_readable_range.py          |   68 +
 tools/gpu-aot/test_renderer_route.py          |   83 ++
 tools/gpu-aot/test_resize_fences.py           |  172 +++
 tools/gpu-aot/test_resolve_region.cpp         |   23 +
 tools/gpu-aot/test_sampler_state.cpp          |  119 ++
 tools/gpu-aot/test_shader_lookup_hashes.py    |   55 +
 tools/gpu-aot/test_shader_texture_usage.cpp   |   35 +
 tools/gpu-aot/test_stencil_state.py           |  162 +++
 tools/gpu-aot/test_texture_header_refresh.py  |  120 ++
 tools/gpu-aot/test_texture_lod.py             |   98 ++
 tools/gpu-aot/test_texture_number.cpp         |   28 +
 tools/gpu-aot/test_texture_upload_cache.cpp   |   36 +
 tools/gpu-aot/test_texture_upload_capture.py  |   67 +
 tools/gpu-aot/test_texture_volume_upload.cpp  |   65 +
 tools/gpu-aot/test_tile_extent.cpp            |   26 +
 tools/gpu-aot/test_tonemap_dof.cpp            |   29 +
 tools/gpu-aot/test_vertex_byte_order.cpp      |   39 +
 tools/gpu-aot/test_viewport_state.py          |   96 ++
 tools/gpu-aot/tiling-answer-20260930.md       | 1212 +++++++++++++++++
 tools/gpu-aot/tiling-reference-20260929.md    |  106 ++
 tools/gpu-aot/verify_alpha_archives.py        |   44 +
 tools/gpu-aot/verify_fps_build.py             |   58 +
 tools/gpu-aot/watch_native_probe.ps1          |   70 +
 70 files changed, 7109 insertions(+), 62 deletions(-)
 create mode 100644 tools/gpu-aot/alpha_compare.hlsli
 create mode 100644 tools/gpu-aot/audit_d3d_paths.py
 create mode 100644 tools/gpu-aot/audit_sampler_defaults.py
 create mode 100644 tools/gpu-aot/audit_shader_semantics.py
 create mode 100644 tools/gpu-aot/compare_placement_shader.py
 create mode 100644 tools/gpu-aot/extract_sampler_oracle.py
 create mode 100644 tools/gpu-aot/generate_microcode_index.cpp
 create mode 100644 tools/gpu-aot/inspect_gpu_snapshot.py
 create mode 100644 tools/gpu-aot/match_mesh_assets.py
 create mode 100644 tools/gpu-aot/merge_microcode_index.py
 create mode 100644 tools/gpu-aot/native-depth-alias-findings-20261001.md
 create mode 100644 tools/gpu-aot/native-performance-20261001.md
 create mode 100644 tools/gpu-aot/prepare_alpha_candidate.py
 create mode 100644 tools/gpu-aot/prepare_lod_candidate.py
 create mode 100644 tools/gpu-aot/prepare_viewport_candidate.py
 create mode 100644 tools/gpu-aot/questions-pending-20260930.md
 create mode 100644 tools/gpu-aot/run_native_probe.ps1
 create mode 100644 tools/gpu-aot/sample_native_process.ps1
 create mode 100644 tools/gpu-aot/simpsons-adoption.md
 create mode 100644 tools/gpu-aot/summarize_brick_resources.py
 create mode 100644 tools/gpu-aot/summarize_frame_metrics.py
 create mode 100644 tools/gpu-aot/summarize_long_probe.py
 create mode 100644 tools/gpu-aot/summarize_mesh_trace.py
 create mode 100644 tools/gpu-aot/test_alpha_test.py
 create mode 100644 tools/gpu-aot/test_audit_d3d_paths.py
 create mode 100644 tools/gpu-aot/test_blend_state.cpp
 create mode 100644 tools/gpu-aot/test_buffer_header.cpp
 create mode 100644 tools/gpu-aot/test_buffer_variants.py
 create mode 100644 tools/gpu-aot/test_bulk_attachments.py
 create mode 100644 tools/gpu-aot/test_completion_callbacks.py
 create mode 100644 tools/gpu-aot/test_cpu_resource_wait.py
 create mode 100644 tools/gpu-aot/test_depth_alias.py
 create mode 100644 tools/gpu-aot/test_depth_export.py
 create mode 100644 tools/gpu-aot/test_descriptor_retirement.cpp
 create mode 100644 tools/gpu-aot/test_draw_upload.py
 create mode 100644 tools/gpu-aot/test_hash_throughput.cpp
 create mode 100644 tools/gpu-aot/test_inline_hash.py
 create mode 100644 tools/gpu-aot/test_long_probe.py
 create mode 100644 tools/gpu-aot/test_memory_watch.py
 create mode 100644 tools/gpu-aot/test_memory_watch_sdk.py
 create mode 100644 tools/gpu-aot/test_pixel_shader_filter.cpp
 create mode 100644 tools/gpu-aot/test_placement_resource_type.py
 create mode 100644 tools/gpu-aot/test_pool_copy.cpp
 create mode 100644 tools/gpu-aot/test_query_retirement.py
 create mode 100644 tools/gpu-aot/test_readable_range.py
 create mode 100644 tools/gpu-aot/test_renderer_route.py
 create mode 100644 tools/gpu-aot/test_resize_fences.py
 create mode 100644 tools/gpu-aot/test_resolve_region.cpp
 create mode 100644 tools/gpu-aot/test_sampler_state.cpp
 create mode 100644 tools/gpu-aot/test_shader_lookup_hashes.py
 create mode 100644 tools/gpu-aot/test_shader_texture_usage.cpp
 create mode 100644 tools/gpu-aot/test_stencil_state.py
 create mode 100644 tools/gpu-aot/test_texture_header_refresh.py
 create mode 100644 tools/gpu-aot/test_texture_lod.py
 create mode 100644 tools/gpu-aot/test_texture_number.cpp
 create mode 100644 tools/gpu-aot/test_texture_upload_cache.cpp
 create mode 100644 tools/gpu-aot/test_texture_upload_capture.py
 create mode 100644 tools/gpu-aot/test_texture_volume_upload.cpp
 create mode 100644 tools/gpu-aot/test_tile_extent.cpp
 create mode 100644 tools/gpu-aot/test_tonemap_dof.cpp
 create mode 100644 tools/gpu-aot/test_vertex_byte_order.cpp
 create mode 100644 tools/gpu-aot/test_viewport_state.py
 create mode 100644 tools/gpu-aot/tiling-answer-20260930.md
 create mode 100644 tools/gpu-aot/tiling-reference-20260929.md
 create mode 100644 tools/gpu-aot/verify_alpha_archives.py
 create mode 100644 tools/gpu-aot/verify_fps_build.py
 create mode 100644 tools/gpu-aot/watch_native_probe.ps1

diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index 1e63ff5..7a3199d 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -23,12 +23,157 @@ resource layout. The LEGO adaptation currently includes:
 - D3D12-only cache generation and partial-cache output on failures;
 - bounded diagnostics plus single-hash repro modes.
 
-The verified D3D12-only corpus result is **3,481 / 3,481 unique shader
-containers (100.00%)**. The generated `lego-dxil-cache.cpp` is about 13 MiB
-compressed C++ source in the current build tree.
+The byte-aligned scanner finds **9,291 unique shader containers**, including
+the captured runtime shaders. The 2026-09-29 expanded build compiled **9,290**;
+runtime vertex shader `A58EBA123EFB541C` is rejected because its memory export
+has no implemented writeback path. The earlier 3,481/3,481 figure counted only
+containers found by the incorrect four-byte-aligned scanner, not full coverage.
+Keep `lego-dxil-cache.cpp` for rollback; the expanded output is
+`lego-expanded-cache.cpp`. Select it with CMake's `LEGO_NATIVE_SHADER_CACHE`.
+The placement index now has 18,504 signatures.
+
+### Placement capture update (2026-09-29)
+
+The current local index has 18,559 signatures after adding 32 exact runtime
+placement containers. All 32 compile successfully. The first pair was not found in
+the extracted disc/TU23 corpus; do not bind a similar shader by prefix alone.
+With `LEGO_DUMP_MISSING_SHADERS=<output-directory>`, native diagnostics capture
+up to 256 distinct missing placement objects per process. XDK retains the
+virtual header at object+872 (VS) / object+40 (PS), while its physical section
+is a separate allocation. Both ranges are checked before reconstruction.
+Only the `placement-containers` child directory is a compiler/indexer input;
+`placement-raw` contains bounded diagnostic fragments, NOT containers.
+Address reuse and the capture cap mean this is not an exhaustive inventory.
+
+The merged local input is `rexlego/out/native-gpu/placement-input`. It currently
+generates `runtime-cache.cpp`, followed by CMake's runtime prelink target.
+The old two-entry runtime cache is backed up as `runtime-cache.pre-placement.cpp`:
+its valid PS already exists in the main cache, while its old memexport VS
+`A58EBA123EFB541C` is intentionally no longer shipped (current compiler rejects
+its unsupported path). Native memexport/writeback is still not implemented.
+
+### Intermediate image diagnostics
+
+The September 30 continuation merges every validated placement capture into
+`placement-input-all-captures`: 91 unique containers, 91/91 DXIL compilations
+with `XENOS_RECOMP_LEGO_NATIVE_SCALE=1`, 18,670 indexed signatures and 146
+runtime prelinked variants. The previous 73-entry and 32-entry caches are
+backed up in `session-20260930`. A zero physical prefix is valid when its
+instruction stream starts later in the section; inspect the declared offset.
+Use `run_native_probe.ps1 -Label <fresh-label> -CaptureMissing` to collect
+additional actual containers, then close the recorded process with `-Close`.
+
+Snapshot probes use wall-time windows at 0/8/16/24/32/48 seconds after the
+first tiled pass (`LEGO_GPU_SNAPSHOT_TIME_WINDOWS=1`). This includes later
+geometry passes when fixed frame indices would all sample the fast starfield.
+The setting remains opt-in; regular gameplay has no snapshot readback.
+Create `capture-next-frame` inside the active capture directory to capture a
+later menu or hub frame without restarting the game. At most 16 mesh draws are
+captured per frame, reserving room for the final resolve and postprocessing.
+
+`test_resolve_region.cpp` checks padded extents, four atlas placements, cropped
+source offsets and rejection of out-of-bounds rectangles. Resolve now consumes
+SourceRect/DestPoint; k_24_8 depth sampling uses an R32_FLOAT atlas mirror.
+Stencil and CPU readback are not supplied by this mirror.
+The native draw path now honors `skip_pixel_shaders`, including the existing
+Graphics depth-of-field toggle. It maps exact indexed microcode hashes to AOT
+containers and refreshes when the setting changes. `test_pixel_shader_filter.cpp`
+checks valid lists, duplicate hashes, whitespace, optional prefixes and errors.
+
+`test_descriptor_retirement.cpp` checks descriptor reuse against all three
+live frame fences, out-of-order completion, repeated release and reuse.
+Native texture Release now reserves its SRV slot through those fences as well
+as retaining the underlying texture/view; keeping just the texture alive does
+not protect recorded draws from a descriptor being overwritten.
+
+Set `LEGO_GPU_SNAPSHOT_DIR` only for a short native-renderer test. The opt-in
+readback records HDR draws, color resolve sources and the final present input
+on bounded frames relative to the first HDR pass. Completed fences gate DDS
+mapping; no intermediate images are read back during normal gameplay.
+Set `LEGO_GPU_SNAPSHOT_START_AT_TILING=1` to anchor these frames at the first
+successful BeginTiling instead of startup HDR work. Captures include expanded
+color resolves and float sampling views. Keep capture runs separate from FPS runs.
+
+The 2026-09-30 fixed16 bank additionally requires
+`XENOS_RECOMP_LEGO_NATIVE_SCALE=1` during both main and runtime DXIL generation.
+Its pixel-shader epilogue reads four output scales at SharedConstants byte 608
+(total host layout 624 bytes), after alpha testing. These scales come from
+each surface's RB_COLOR_INFO format and signed exponent, including the fixed16
+division by 32. Use `lego-fixed16-cache.cpp` with the matching runtime cache;
+the previous banks and executables are retained for rollback.
+
+The placement index now stores conservative texture-slot masks from all Xenos
+exec clauses. Unknown input retains all 32 slots. Native draws check uploads only
+for slots referenced by either shader; hashes and updates for those slots remain
+enabled. `LEGO_NATIVE_ALL_TEXTURES=1` restores the previous all-binding behavior
+for comparison. `test_shader_texture_usage.cpp` covers conditional END clauses,
+slot 31, vertex fetches, ALU-only programs and malformed input.
+
+```powershell
+E:/devtools/python/python.exe tools/gpu-aot/inspect_gpu_snapshot.py <capture-directory>
+E:/devtools/python/python.exe tools/gpu-aot/inspect_gpu_snapshot.py <capture.dds> --image
+```
+
+`--image` creates an RGB PPM, and optional `--scale` changes diagnostic exposure
+only. Inspect the PPM or convert it to PNG; avoid relying on another decoder's
+implicit channel interpretation for DX10 DDS. Native stage logs identify shader
+addresses, targets, viewport, scissor, textures, write mask and blend state.
+
+Current local user preference (2026-09-30): launch tests on `monitor=3` and
+scope `SDL_WINDOW_ACTIVATE_WHEN_SHOWN=0` to the child process. The assistant
+handles loading up to the initial cutscene; later gameplay tests are coordinated
+with the user. Do not use an automatic close timer during a user-assisted test.
+Close only the exact assistant-launched test process after a diagnostic check;
+do not interact with the user's other game.
+
+Standalone state regressions (compile with the existing C++20 toolchain):
+`test_vertex_byte_order.cpp`, `test_blend_state.cpp`, `test_texture_number.cpp`.
+These are correctness tests, not performance benchmarks or gameplay coverage.
+
+`test_texture_upload_cache.cpp` additionally needs
+`-I rexglue-sdk/thirdparty/xxHash`. It checks that source-cache keys change for
+base data, the last cube face, mip tails, fetch semantics and range sizes.
+The runtime hashes readable guest base/mip spans BEFORE allocating/untile;
+unreadable conservative spans fall back to the previous per-block path.
+This avoids repeated conversion, but it still scans bytes on every binding:
+it is not page-dirty tracking, and CPU texture edits are not ignored.
+
+For a bounded timing check, set only `LEGO_NATIVE_TIMING=1` (unset GPU snapshot
+and texture/missing-shader dump variables). Slow-frame lines report intervals
+between presents and total CPU time/calls/source-cache hits/conversion bytes
+in texture upload. This adds no GPU readback or extra fence wait. These are
+startup diagnostics, not standalone/comparable performance benchmarks while
+the user is running another game.
+
+Depth-output regression (requires the built XenosRecomp executable):
+
+```powershell
+E:/devtools/python/python.exe tools/gpu-aot/test_depth_export.py `
+  research/reblue/thirdparty/XenosRecomp/out/build/lego-release/XenosRecomp/XenosRecomp.exe `
+  research/reblue/thirdparty/XenosRecomp/XenosRecomp/shader_common.h `
+  rexlego/out/native-gpu/depth-regression
+```
+
+The six synthetic containers test color-only shaders, inaccurate reflection,
+and real vector/scalar/constant/predicated depth exports through HLSL and DXC.
+Only actual decoded depth exports add `SV_Depth`; declaring it unconditionally
+and initializing it to zero breaks ordinary raster depth. Both the main and
+32-entry runtime archives (and their prelinked variants) must be regenerated
+after changing this codegen, not just the executable or dumped HLSL.
+
+`lego_gpu_microcode_index` separately records the physical Xenos microcode
+identity needed to adopt placement shader objects. It does not run DXC or
+change the compiled archive.
 
 ## Reproduction
 
+**Current native alpha support needs the compiler produced by
+`prepare_alpha_candidate.py`, with its generated shader_common.h.** The older
+generic XenosRecomp executable below still emits only the legacy >= test.
+Regenerate both banks and all linked variants together; do not replace one
+installed alpha bank with output from that older compiler. See the offline
+alpha section and coverage proof in session-20260930/alpha-candidate.
+
 Build the extractor with its local project, then:
 
 ```powershell
@@ -47,6 +192,17 @@ XenosRecomp.exe `
   research/reblue/thirdparty/XenosRecomp/XenosRecomp/shader_common.h
 ```
 
+Then build and run the placement indexer over both the archive corpus and any
+captured runtime containers:
+
+```powershell
+cmake --build rexlego/out/build/win-amd64-release --target lego_gpu_microcode_index
+rexlego/out/build/win-amd64-release/lego_gpu_microcode_index.exe `
+  rexlego/out/native-gpu/lego-microcode-index.cpp `
+  rexlego/out/native-gpu/shader-assets `
+  rexlego/out/native-gpu/runtime-missing
+```
+
 For a failing shader, set `XENOS_RECOMP_ONLY_HASH` to its `0x...` hash and
 `XENOS_RECOMP_VERBOSE_DXC=1`. `XENOS_RECOMP_HLSL_DUMP` may name a directory
 for generated HLSL.
@@ -58,3 +214,278 @@ still requires the D3D12 host device, guest resource wrappers, draw/state
 hooks, constant and descriptor upload, render-target/resolve handling, and a
 safe fallback to the existing `xenos` plugin until native coverage is proven
 in gameplay.
+
+## TU23 sampler regression check
+
+`test_sampler_state.cpp` compares the native fetch-header merge with the actual
+generated `83FB58A8` body, including all 26 slots, NULL unbind, physical address
+fixups and dirty-mask writes. Extract the oracle into the session scratchpad
+before compiling (the test does not start the game):
+
+```powershell
+python -B tools/gpu-aot/extract_sampler_oracle.py --output rexlego/out/native-gpu/session-20260930/tu23_set_texture_oracle.inc
+clang++ -std=c++20 -O2 -DNOMINMAX -Irexlego/src -Ithirdparty/plume -Irexlego/out/native-gpu/session-20260930 tools/gpu-aot/test_sampler_state.cpp -o rexlego/out/native-gpu/session-20260930/test-sampler-state.exe
+rexlego/out/native-gpu/session-20260930/test-sampler-state.exe
+```
+
+This also checks host sampler address/filter/anisotropy/LOD modes and cache
+identity. It cannot establish visual correctness, actual guest default table
+contents or descriptor behavior on a live D3D12 device. Native startup logs the
+20 defaults and rejects initializer functions outside the audited TU23 set.
+Runtime sampler/VS-slot diagnostics are bounded; no descriptor is overwritten
+or recycled while the device is active. Explicit register LOD and unusual
+border colors remain separate shader/backend limitations.
+
+`audit_sampler_defaults.py --output <scratch.json>` reads the 20 getter/setter/
+default triples from the mapped TU23 image and checks their exact whitelist,
+generated leaf bodies and direct/conditional branch targets. The float
+constants are 32 for MipMapLodBias and -8 for AnisotropyBias. The September 30
+sampler-fetch run independently logged the same 20 defaults and initialization
+of all 26 slots. Static table checks alone cannot establish a live device state.
+
+Snapshot diagnostics also report changed borrowed texture headers and changed
+VS texture fetch words. A draw does not automatically replace a texture from
+its latest header: an existing draw uses the active device fetch shadow.
+At SetTexture and Resolve, the new binding comes from the header, so refresh
+borrowed resources there. An address-only change updates the raw CPU alias and
+invalidates the upload key; a layout change retires the old storage and SRVs at
+their GPU fences. Keep host-resolved contents authoritative during relocation.
+The manual `capture-next-frame` trigger selects the
+next frame so noticing the sentinel halfway through a pass cannot produce a
+misleading partial-frame capture. Existing time-window captures remain bounded.
+
+## Isolated explicit-LOD candidate
+
+The captured TU23 corpus contains 14 pixel shaders that set a register LOD
+and then fetch a cube texture on slot 3 with `use_register_lod`. The current
+compiler skips `SetTextureLod` and uses implicit sampling. Local Xenia's
+`dxbc_translator_fetch.cpp` preserves the scalar and adds the signed fetch
+bias divided by 32 plus the instruction bias divided by 16 before explicit
+sampling. This mismatch is established from code; its visual impact has not
+been tested.
+
+`prepare_lod_candidate.py` copies and patches the compiler into a scratch
+directory, leaving the production compiler, shader banks and native executable
+unchanged. It also writes an isolated `draw-candidate.cpp` showing the matching
+constant upload. Build with the existing local compiler dependencies:
+
+```powershell
+. E:/devtools/env.ps1
+python -B tools/gpu-aot/prepare_lod_candidate.py rexlego/out/native-gpu/session-20260930/lod-candidate
+cmake -S rexlego/out/native-gpu/session-20260930/lod-candidate/source -B rexlego/out/native-gpu/session-20260930/lod-candidate/build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++
+cmake --build rexlego/out/native-gpu/session-20260930/lod-candidate/build --target lod_compiler -j6
+python -B tools/gpu-aot/test_texture_lod.py rexlego/out/native-gpu/session-20260930/lod-candidate/build/lod_compiler.exe rexlego/out/native-gpu/session-20260930/lod-candidate/source/shader_common.h rexlego/out/native-gpu/session-20260930/lod-candidate/fixtures
+```
+
+The seven end-to-end fixtures cover register/zero LOD, instruction predication,
+positive/negative instruction bias, computed pixel LOD and vertex LOD behavior.
+DXC validates their output; relative `SetTextureLod` addressing is rejected.
+The isolated candidate also compiled all 14 actual containers to nonempty DXIL
+entries. Their generated HLSL preserves each decoded LOD source and uses
+`SampleLevel`; hashes and checks are saved in the session's
+`lod-candidate/verification.json`. Neither result establishes correct game
+pixels or frame timing.
+
+**Do not install these candidate shaders in the current runtime bank.** They
+require `XENOS_RECOMP_LEGO_NATIVE_SCALE=1` and a SharedConstants ABI of 752 bytes,
+with 32 fetch LOD biases beginning at byte 624. The current sampler build still
+uploads 624 bytes. A later integration must update both shader archives and
+their prelinked variants together with the host upload. Observe the sampler
+build before combining this separate change. Explicit gradients, instruction
+filter overrides, denormalized coordinates and 1D helper coverage remain
+separate limitations.
+
+## Bank118 and menu idle reproduction, September 30
+
+The production runtime bank now contains 118 validated containers, 18722
+microcode signatures and 192 linked runtime variants. It retains the current
+624-byte constants ABI. The six late containers had valid nonzero instruction
+streams despite leading zero words. `merge_microcode_index.py` preserves the
+previous index and rejects conflicting rows. Build proof and original inputs
+are in `session-20260930/bank118-candidate/`.
+
+The user confirmed substantially better late cutscene geometry and correct
+characters in build `cf7383e12e047c83`. The title screen and main menu initially
+looked correct, then idle reproduced corrupt logo/UI textures. Archive:
+`bank118-user-menu-good.png`, `bank118-user-menu-idle-corrupt.png`,
+`bank118-header-menu-idle-corrupt.png`, and the complete manual GPU frame2386.
+The game was closed normally at16:55:41. It is not a stable image baseline.
+
+The log contains 32 distinct stale-header pairs: 30 relocations and two layout
+changes. `test_texture_header_refresh.py` extracts the production refresh body
+and exercises those actual pairs with CPU and host-resolved authority, owned
+resources, unchanged headers, mip-allocation transitions and type reuse (131
+cases). The harness and descriptor-retirement test pass. With the refresh and
+opt-in diagnostics, the user confirmed stable main-menu image for 390.8 seconds
+in `cutscene-menu-long2`, then requested a restart for the full cutscene. This
+exceeds the previous >3-minute reproduction, but is not a completed 10-minute
+test. The cutscene was skipped in that run. Retain
+`before-texture-header-refresh/` as rollback.
+Next visual check: late cutscene, title-to-menu transition, then menu idle past
+the prior corruption point before settings/hub. Excessive late blur and FPS
+remain separate, unresolved checks.
+
+## Long cutscene and menu idle probe
+
+Launch with `run_native_probe.ps1 -Label <fresh-label> -Mode snapshot
+-CaptureMissing -LongProbe`. The game keeps its normal config and
+no-focus launch. The opt-in trace writes `trace-<label>/events.tsv`: every frame's
+CPU timings, texture upload hash changes, header relocations/layout changes,
+fetch changes, newly seen draw stages, resolve regions and failed bindings.
+Anomaly flags are candidates, including legitimate fades and resource reuse;
+they need screenshot review. They do not imply a diagnosed fault.
+
+The final image is captured every5seconds and full frames every30seconds,
+with extra next-frame captures on candidates (10second renderer cooldown).
+GPU readbacks are bounded at12GiB; reaching the budget emits an event while
+event/timing logging continues. Whole-frame captures include actual8816-byte
+constant uploads from draws with at most6vertices, capped at128files per frame:
+VS float constants at0, PS at4096, SharedConstants at8192
+(624bytes). `draw_constants` rows associate files with shader hashes. During
+long runs draw-stage snapshots focus on draws with at most6vertices; resolves
+still capture the HDR and intermediate images. `snapshot_saved` records the
+absolute captured frame because DDS filenames use the relative tiling frame.
+
+The hidden watcher verifies the game's PID/path/start time, records process
+memory and log liveness in heartbeat.json, and requests a frame on errors.
+It exits when the game exits; it never closes the game. To start the actual
+menu timer, write `trace-<label>/menu-start.json` with `{"utc":"<UTC ISO time>"}`
+only after main-menu entry is observed or confirmed by the user. At600seconds it logs the elapsed
+interval and requests another complete frame. Cutscene/title time is excluded.
+`summarize_long_probe.py trace-<label>` generates summary.json. FPS data include
+capture/logging overhead and are not a clean performance benchmark.
+
+`test_long_probe.py` checks disabled mode, a shared writer across translation
+units, concurrent event producers, anomaly requests and exact binary payloads.
+Build identity/config/bank proof: session-20260930/long-probe-build-proof.json.
+
+## FPS experiments and placement resource types (30 September)
+
+The launcher now defaults to primary monitor1 while the user is away; `-Monitor N`
+overrides it and records the actual number. `-Mode timing` writes lightweight
+per-present `LABEL-frames.csv`, without GPU readbacks. Summarize with
+`summarize_frame_metrics.py PATH`. Scene identity still needs a screenshot.
+Stable30 has not been reached; compare matching scenes rather than whole-run
+averages from different cutscene/menu segments.
+
+Full page-watch reuse accelerated rendering but regressed Vortech geometry.
+The watch is now off by default. `-StaticTextureWatch` enables experimental
+reuse for CPU textures sampled only by PS; VS animation textures and all
+buffers still hash current bytes on every use. `-NoMemoryWatch` disables it;
+`-NoDrawArena` uses separate constant uploads for a controlled comparison.
+Tests: `test_memory_watch.py`, `test_draw_upload.py`, `test_buffer_variants.py`.
+Their models cannot prove real page-fault/GPU behavior.
+
+Title idle also corrupted with the watch disabled, and with the draw arena
+disabled. Found a separate TU23 contract error: the GetResourceType hook
+returned host buffer kinds6/7 for borrowed placement headers of type1/2.
+XGOffsetResource83F9B640 consequently wrote a VB's header+32 instead of
+updating its base at+24. Registry overrides now apply only to owned simplified
+headers; borrowed buffers/textures use the original leaf GetResourceType.
+`test_placement_resource_type.py OUTPUT` executes the actual generated
+GetResourceType/XGOffsetResource bodies and native hook, reproduces the old
+wrong-field write, checks borrowed VB/IB and texture dimension distinctions,
+and preserves owned-header behavior. The user subsequently confirmed five
+minutes on the title before Enter without the old giant polygons. This is a
+narrow regression check; the cutscene still has missing arms and strong blur.
+
+`XXH_INLINE_ALL` uses the application's existing x86-64-v3 target for full
+content hashes. `test_inline_hash.py` compares 5000 seeded/unseeded, unaligned
+cases against the actual SDK library. The 1.87x hash benchmark is not a game
+FPS claim. Buffers and VS textures still hash every use.
+
+`-AsyncCpuResources` is an experimental opt-in for BlockUntilNotBusy on known
+CPU snapshot resources. Unknown/GPU-produced resources keep their waits;
+completion callbacks use the deferred queue described below.
+`test_cpu_resource_wait.py` verifies those
+exclusions and callback ordering; real GPU validation is separate. The first
+runtime probe skipped almost none of the expensive waits, so it is not a
+verified FPS improvement. Per-frame CSV now attributes sync calls and time
+to idle, fence, resource, callback, query begin/release, and other callers.
+This attribution changes no synchronization behavior.
+
+## Deferred completion callbacks
+
+Addendum14 identifies TU23 callbacks82BCDE68/82BCDF30 as profiler markers.
+Original InsertCallback records them without blocking the CPU. Native now
+enqueues all callbacks in FIFO order and executes them only after the containing
+host submission's fence completes. InsertCallback neither drains nor forces a
+submission. This also retains completion ordering for unknown/UI callbacks;
+there is no immediate profiler shortcut that assumes timing readers are absent.
+
+Completion is conservative at a submission boundary. Profiler timestamps have
+the granularity of the completion poll, not exact GPU marker positions. Polls
+run on guest hook threads at InsertCallback, KickOff, explicit waits and Swap.
+Guest callbacks execute outside the device and recording mutexes; a nonblocking
+execution gate preserves FIFO across reentrant/concurrent polls. Pending
+callbacks are cancelled at title shutdown instead of entering guest code on
+the host UI thread. D3D12 polls compare against Plume's post-signal
+`fenceValue - 1`; device-removal UINT64_MAX never counts as completion.
+
+`test_completion_callbacks.py OUTPUT` executes actual enqueue/poll and
+submission attribution functions against deterministic fences, including1000
+slot generations, FIFO, reentrancy, mutex release and removal. The CSV includes
+callbacks_enqueued/executed/pending. These tests and a successful native build
+do not establish gameplay correctness or a30FPS result; a controlled run is
+still required.
+
+## Query storage retirement
+
+Occlusion query BEGIN now creates a fresh host readback generation, retaining
+the preceding generation through every submitted/open frame slot that may
+reference it. Final Release removes the CPU lookup/header immediately and
+retires host query heaps, readbacks and their completion fence the same way.
+Neither operation drains all GPU work. GetData still requires completion of
+that query's fence and returns the real saturating sample sum; device removal
+and failed draws/readbacks remain failures. No visibility result is fabricated.
+
+`test_query_retirement.py OUTPUT` executes the actual IssueQuery, GetQueryData,
+ReleaseQuery and HostDevice::RetireResource bodies against fake fences. It checks
+reuse, release before END, all-live-slot retention, refcounts/header reuse,
+readiness, sums, and allocation/signal/readback failures. Real GPU validation
+and its effect on portal visibility/FPS remain separate.
+
+## Disabled DoF in the final tonemap
+
+Filtering CoC shader CFEAC7ADB912F8A9 alone did not disable the final sharp/mip
+mix. Translated PS3A47E5DDE66B42C6 blends slot0 with downsampled slot4 using
+`saturate(2*(mip.a*c12.x+c12.y)-1)`; slot3 contributes squared additive glow.
+When the existing depth_of_field setting is false, only that shader's immutable
+host upload changes c12.xy to zero. Guest constants, exposure, bloom, LUT and
+c12.zw are preserved, and enabling DoF uses the original values again.
+
+`test_tonemap_dof.cpp` checks shader/setting/bounds gating, untouched constants,
+and zero mixing over the finite fixed16 alpha range. This makes the disabled
+setting deterministic; it does not prove why mip alpha previously caused blur
+or repair absent character parts, portal draws, or alpha-test state.
+
+## Offline alpha/defaults and resize repair
+
+Native now uploads raw FLOAT RB_ALPHA_REF at shared offset556 and raw Xenos
+compare at offset600, keeping the shared ABI624. RB_COLORCONTROL bit3 selects
+alpha specialization bit1. The isolated compiler prepared by
+`prepare_alpha_candidate.py` emits all eight compares before EDRAM scaling,
+gating alpha test for shaders which actually export RT0. It preserves the
+upstream behavior outside LEGO_NATIVE_GPU. Both the9290-entry corpus and118
+runtime containers were regenerated, then every specialization was prelinked.
+The known memexport A58EBA123EFB541C remains excluded; hash/mask coverage is
+checked by `verify_alpha_archives.py`. The LOD752 candidate remains separate.
+
+CreateDevice applies15 audited CPU-only TU23 leaf setters for alpha, blending,
+and the four color masks, in descriptor-table order. Defaults come from the
+actual table, with setter-address validation before any writes. In particular,
+the alpha-reference default is raw0x3F800000 (1.0), not the peer's suggested0.0;
+blending defaults to copy and every color write mask to15. Full Xbox queue
+initialization is still bypassed. `test_alpha_test.py` executes those actual
+generated setters plus the native initializer, checks968 compare cases and
+five microcode/HLSL/DXIL fixtures. No claim that alpha fixes hands/portal is made.
+
+Resize/shutdown now drain every submitted frame fence before releasing the
+ring. DXGI's frame-latency wait is not GPU completion. Plume's auto-reset event
+is treated only as a wakeup, always checked against the current fence target;
+consumed or stale events cannot cause a hang or premature completion. Failed
+ResizeBuffers restores its old references and preserves dimensions; minimized
+or failed resize is retried even when native framebuffers are empty.
+`test_resize_fences.py` executes the actual wait/resize bodies with deterministic
+fences/events and checks all slots, removal, partial retries and backbuffer
+restoration. These offline checks do not confirm real window-mode recovery.
diff --git a/tools/gpu-aot/alpha_compare.hlsli b/tools/gpu-aot/alpha_compare.hlsli
new file mode 100644
index 0000000..4dc89b3
--- /dev/null
+++ b/tools/gpu-aot/alpha_compare.hlsli
@@ -0,0 +1,16 @@
+// Xenos RB_COLORCONTROL alpha_func. Test original shader alpha before EDRAM
+// exponent scaling. Ordered comparisons reject NaNs; NOTEQUAL accepts them.
+bool LegoAlphaPass(float alpha, float reference, uint function)
+{
+    switch (function & 7u)
+    {
+        case 0u: return false;
+        case 1u: return alpha < reference;
+        case 2u: return alpha == reference;
+        case 3u: return alpha <= reference;
+        case 4u: return alpha > reference;
+        case 5u: return alpha != reference;
+        case 6u: return alpha >= reference;
+        default: return true;
+    }
+}
diff --git a/tools/gpu-aot/audit_d3d_paths.py b/tools/gpu-aot/audit_d3d_paths.py
new file mode 100644
index 0000000..cdec314
--- /dev/null
+++ b/tools/gpu-aot/audit_d3d_paths.py
@@ -0,0 +1,77 @@
+"""Read-only direct-call audit of generated TU23 code and native hook coverage.
+
+This is a candidate map, not a proof of runtime reachability or correctness.
+Indirect calls, data-flow-derived packet writers and hook semantics require review.
+"""
+import argparse
+from contextlib import redirect_stdout
+from collections import defaultdict, deque
+from pathlib import Path
+import re
+
+FUNCTION = re.compile(r"DEFINE_REX_FUNC\(sub_([0-9A-Fa-f]{8})\)\s*\{")
+CALL = re.compile(r"\b(?:__imp__)?sub_([0-9A-Fa-f]{8})\(ctx, base\)")
+HOOK = re.compile(r"REX_HOOK(?:_RAW)?\(\s*sub_([0-9A-Fa-f]{8})")
+
+
+def audit(root):
+    symbols = defaultdict(list)
+    ambiguous = 0
+    for line in (root / 'tools/gpu-aot/d3d-sep13-matches.tsv').read_text().splitlines():
+        if line.startswith('#'):
+            ambiguous += 1
+            continue
+        fields = line.split('\t')
+        if len(fields) == 4:
+            symbols[int(fields[2], 16)].append(fields[0])
+    hooks = set()
+    for path in (root / 'rexlego/src/gpu_native').glob('*.cpp'):
+        hooks.update(int(x, 16) for x in HOOK.findall(path.read_text()))
+    callers = defaultdict(set)
+    locations = {}
+    indirect = set()
+    for path in sorted((root / 'rexlego/generated/default').glob('*recomp*.cpp')):
+        text = path.read_text()
+        starts = list(FUNCTION.finditer(text))
+        for index, match in enumerate(starts):
+            address = int(match[1], 16)
+            body = text[match.end():starts[index + 1].start() if index + 1 < len(starts) else len(text)]
+            locations[address] = path.name
+            for call in CALL.findall(body):
+                callers[int(call, 16)].add(address)
+            if 'REX_CALL_INDIRECT' in body:
+                indirect.add(address)
+    # Walk reverse direct calls, STOP at hooks: their original body isn't
+    # necessarily executed. RAW hooks may explicitly call the original, so
+    # even this barrier is not proof that the path is safe.
+    # Verified symbol map: CBlocker::Check, plus the sample-count PM4 writer
+    # at the last crash. This is NOT an exhaustive list of packet writers.
+    seeds = (0x83FAF180, 0x83FCBF00)
+    paths = {seed: [seed] for seed in seeds}
+    todo = deque(seeds)
+    while todo:
+        target = todo.popleft()
+        for caller in sorted(callers[target]):
+            if caller not in paths:
+                paths[caller] = [caller] + paths[target]
+                if caller not in hooks:
+                    todo.append(caller)
+    print(f'# Functions={len(locations)} hooks={len(hooks)} named_addresses={len(symbols)} ambiguous_rows={ambiguous}')
+    print(f'# Blocker/query-writer reverse-call candidates={len(paths)} (includes seeds and hook boundaries)')
+    print('address\thooked\tindirect\tring_path\tdirect_callers\tsymbols')
+    for address in sorted(set(symbols) | set(paths)):
+        chain = ' -> '.join(f'{x:08X}' for x in paths.get(address, []))
+        print(f'{address:08X}\t{int(address in hooks)}\t{int(address in indirect)}\t{chain}\t{len(callers[address])}\t' + '; '.join(symbols[address]))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[2])
+    parser.add_argument('--output', type=Path, help='Optional generated TSV report')
+    args = parser.parse_args()
+    if args.output:
+        with args.output.open('w', encoding='utf-8', newline='') as report:
+            with redirect_stdout(report):
+                audit(args.root)
+    else:
+        audit(args.root)
diff --git a/tools/gpu-aot/audit_sampler_defaults.py b/tools/gpu-aot/audit_sampler_defaults.py
new file mode 100644
index 0000000..5769a1b
--- /dev/null
+++ b/tools/gpu-aot/audit_sampler_defaults.py
@@ -0,0 +1,110 @@
+"""Audit TU23 sampler initializer data and direct branches without running it.
+
+Uses the mapped PPC image and unchanged generated functions. The image must
+have the expected 0x82000000 mapping. This is static evidence, not a live
+device initialization or visual test.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+import struct
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--image', type=Path, default=Path('xexdump/dump/default.bin'))
+    parser.add_argument('--generated', type=Path, default=Path('rexlego/generated/default'))
+    parser.add_argument('--initializer', type=Path,
+                        default=Path('rexlego/src/gpu_native/hooks_device.cpp'))
+    parser.add_argument('--output', type=Path, required=True)
+    args = parser.parse_args()
+    image = args.image.read_bytes()
+    base = 0x82000000
+
+    def word(at):
+        assert base <= at <= base + len(image) - 4
+        return struct.unpack_from('>I', image, at - base)[0]
+
+    native = args.initializer.read_text()
+    allowed_text = re.search(r'allowed\{([^}]+)\}', native).group(1)
+    allowed = {int(h, 16) for h in re.findall(r'0x([A-Fa-f0-9]+)', allowed_text)}
+    table = [tuple(word(0x847F9FD8 + k * 12 + d * 4) for d in range(3))
+             for k in range(20)]
+    assert {t[1] for t in table} == allowed and len(allowed) == 20
+    bodies = {}
+    for path in args.generated.glob('*.cpp'):
+        source = path.read_text()
+        for setter in allowed:
+            start = source.find(f'DEFINE_REX_FUNC(sub_{setter:08X}) {{')
+            if start < 0:
+                continue
+            assert setter not in bodies, f'Duplicate function {setter:08X}'
+            end = source.find('\nDEFINE_REX_FUNC(', start + 1)
+            bodies[setter] = source[start:end if end >= 0 else len(source)]
+    assert set(bodies) == allowed
+    helper_targets = {0x83E44F7C, 0x83E44FCC}  # __save/__restgprlr_29
+    records = []
+    for k, (getter, setter, default) in enumerate(table):
+        body = bodies[setter]
+        instructions = re.findall(r'^\s*// ([^\n]+)', body, re.M)
+        assert instructions
+        branches = []
+        for index, assembly in enumerate(instructions):
+            at = setter + 4 * index
+            value = word(at)
+            opcode = value >> 26
+            # Direct branches must match the generated branch destination.
+            if opcode == 18:
+                displacement = value & 0x03FFFFFC
+                if displacement & 0x02000000:
+                    displacement -= 0x04000000
+                target = (displacement if value & 2 else at + displacement) & 0xFFFFFFFF
+                match = re.fullmatch(r'b[l]? 0x([0-9a-f]+)', assembly.strip())
+                assert match and int(match[1], 16) == target, (hex(at), assembly)
+                assert setter <= target < setter + 4 * len(instructions) or target in helper_targets
+                if not setter <= target < setter + 4 * len(instructions):
+                    branches.append(f'{target:08X}')
+            elif opcode == 16:
+                displacement = value & 0xFFFC
+                if displacement & 0x8000:
+                    displacement -= 0x10000
+                target = (displacement if value & 2 else at + displacement) & 0xFFFFFFFF
+                assert setter <= target < setter + 4 * len(instructions), (hex(at), assembly)
+            elif opcode == 19:
+                # bclr is a return; bcctr would imply an indirect call/queue path.
+                assert ((value >> 1) & 1023) != 528, (hex(at), assembly)
+            # Branch comments are also checked in the other direction so a
+            # mismatched image cannot hide a generated direct call.
+            if re.fullmatch(r'b[l]? 0x[0-9a-f]+', assembly.strip()):
+                assert opcode == 18, (hex(at), assembly)
+        generated_calls = set(re.findall(r'\b(sub_[0-9A-F]+|__\w+)\(ctx, base\)', body))
+        assert generated_calls <= {'__savegprlr_29', '__restgprlr_29'}, generated_calls
+        records.append({'index': k, 'getter': f'{getter:08X}', 'setter': f'{setter:08X}',
+                        'default': f'{default:08X}', 'instruction_count': len(instructions),
+                        'external_branch_targets': branches,
+                        'image_code_sha256': hashlib.sha256(
+                            image[setter-base:setter-base+4*len(instructions)]).hexdigest(),
+                        'generated_body_sha256': hashlib.sha256(body.encode()).hexdigest()})
+    constants = {f'{at:08X}': struct.unpack_from('>f', image, at-base)[0]
+                 for at in (0x8202776C, 0x8204EA60)}
+    assert constants == {'8202776C': 32.0, '8204EA60': -8.0}
+    report = {
+        'note': 'Static TU23 data/branch audit; not proof of runtime initialization or visuals',
+        'image': str(args.image.resolve()),
+        'image_sha256': hashlib.sha256(image).hexdigest(),
+        'image_base': f'{base:08X}',
+        'sampler_table': '847F9FD8',
+        'table_sha256': hashlib.sha256(image[0x847F9FD8-base:0x847F9FD8-base+240]).hexdigest(),
+        'exact_whitelist_match': True,
+        'no_external_queue_branches': True,
+        'float_constants': constants,
+        'initializers': records,
+    }
+    args.output.write_text(json.dumps(report, indent=2), encoding='utf-8')
+    print('TU23 sampler defaults audited: 20 exact whitelist entries, leaf/save-restore branches only')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/gpu-aot/audit_shader_semantics.py b/tools/gpu-aot/audit_shader_semantics.py
new file mode 100644
index 0000000..c2f7d22
--- /dev/null
+++ b/tools/gpu-aot/audit_shader_semantics.py
@@ -0,0 +1,125 @@
+"""Inventory Xenos fetch semantics in validated containers without changing shaders.
+
+This flags features for comparison with Xenia, not proven rendering defects.
+Decoding follows shader_code.h / shader_texture_usage.h in this workspace.
+"""
+import argparse
+import collections
+import hashlib
+import json
+from pathlib import Path
+import struct
+
+
+def field(value, shift, bits=1):
+    return (value >> shift) & ((1 << bits) - 1)
+
+
+def signed(value, bits):
+    return value - (1 << bits) if value & (1 << (bits - 1)) else value
+
+
+def decode(path):
+    data = path.read_bytes()
+    flags, virtual, physical = struct.unpack_from('>III', data)
+    if flags & 0xFFFFFF00 != 0x102A1100 or virtual < 36 or virtual + 4 > len(data):
+        raise ValueError('invalid container header')
+    physical_base = virtual + (4 if struct.unpack_from('>I', data, virtual)[0] == physical else 0)
+    shader_offset = struct.unpack_from('>I', data, 24)[0]
+    if shader_offset + 8 > virtual or physical_base + physical > len(data):
+        raise ValueError('invalid section bounds')
+    offset, size = struct.unpack_from('>II', data, shader_offset)
+    if not size or size % 12 or offset + size > physical:
+        raise ValueError('invalid instruction bounds')
+    code = data[physical_base + offset:physical_base + offset + size]
+    slots = [struct.unpack_from('>III', code, at) for at in range(0, size, 12)]
+    cf_end = len(slots)
+    visited = set()
+    fetches, control = [], []
+    pair = 0
+    while pair < cf_end:
+        a, b, c = slots[pair]
+        for lo, hi in ((a, b & 0xFFFF), ((b >> 16) | ((c << 16) & 0xFFFFFFFF), c >> 16)):
+            op = hi >> 12
+            control.append(op)
+            if op not in (1, 2, 3, 4, 5, 6, 13, 14):
+                continue
+            address, count, sequence = lo & 0xFFF, field(lo, 12, 3), field(lo, 16, 12)
+            if count > 6 or address <= pair or address + count > len(slots):
+                raise ValueError('invalid EXEC clause')
+            cf_end = min(cf_end, address)
+            for i in range(count):
+                instruction = address + i
+                if not field(sequence, i * 2) or instruction in visited:
+                    continue
+                visited.add(instruction)
+                w0, w1, w2 = slots[instruction]
+                opcode = w0 & 31
+                item = {'instruction': instruction, 'opcode': opcode,
+                        'words': [f'{w:08X}' for w in (w0, w1, w2)]}
+                if opcode == 0:
+                    item.update(kind='vertex', source_register=field(w0, 5, 6),
+                        source_relative=field(w0, 11), constant=field(w0, 20, 5),
+                        constant_select=field(w0, 25, 2), source_component=field(w0, 30, 2),
+                        format=field(w1, 16, 6), signed=field(w1, 12), integer=field(w1, 13),
+                        exponent=signed(field(w1, 24, 6), 6), mini=field(w1, 30),
+                        stride=field(w2, 0, 8), offset=signed(field(w2, 8, 23), 23))
+                else:
+                    item.update(kind='texture', constant=field(w0, 20, 5),
+                        source_register=field(w0, 5, 6), source_swizzle=field(w0, 26, 6),
+                        computed_lod=field(w1, 28), register_lod=field(w1, 29),
+                        register_gradients=field(w2, 0), dimension=field(w2, 14, 2),
+                        mag_filter=field(w1, 12, 2), min_filter=field(w1, 14, 2),
+                        mip_filter=field(w1, 16, 2))
+                fetches.append(item)
+        pair += 1
+    return {'file': str(path.resolve()), 'stage': 'vs' if flags & 1 else 'ps',
+            'container_sha256': hashlib.sha256(data).hexdigest(), 'code_bytes': size,
+            'control_opcodes': sorted(set(control)), 'fetches': fetches}
+
+
+def audit(roots):
+    entries, errors, seen = [], [], set()
+    for root in roots:
+        for path in sorted(root.rglob('*.bin')):
+            try:
+                entry = decode(path)
+            except (ValueError, struct.error) as exc:
+                errors.append({'file': str(path), 'error': str(exc)})
+                continue
+            if entry['container_sha256'] in seen:
+                continue
+            seen.add(entry['container_sha256'])
+            entries.append(entry)
+    findings = collections.defaultdict(list)
+    for entry in entries:
+        for fetch in entry['fetches']:
+            name = Path(entry['file']).name
+            if fetch['kind'] == 'vertex':
+                for key in ('mini', 'exponent', 'source_relative'):
+                    if fetch[key]:
+                        findings[f'vertex_{key}'].append({'shader': name, **fetch})
+                if fetch['source_register'] or fetch['source_component']:
+                    findings['vertex_nondefault_index'].append({'shader': name, **fetch})
+            else:
+                if fetch['opcode'] not in (1, 19):
+                    findings['texture_state_opcode'].append({'shader': name, **fetch})
+                if fetch['opcode'] == 1 and fetch['register_lod']:
+                    findings['texture_register_lod'].append({'shader': name, **fetch})
+                if fetch['opcode'] == 1 and fetch['register_gradients']:
+                    findings['texture_register_gradients'].append({'shader': name, **fetch})
+    return {'note': 'Static feature inventory; findings are not proven visual defects.',
+            'containers': len(entries), 'stages': dict(collections.Counter(e['stage'] for e in entries)),
+            'findings': dict(findings), 'errors': errors, 'shaders': entries}
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('roots', nargs='+', type=Path)
+    parser.add_argument('--output', required=True, type=Path)
+    args = parser.parse_args()
+    report = audit(args.roots)
+    args.output.write_text(json.dumps(report, indent=2), encoding='utf-8')
+    print(json.dumps({'containers': report['containers'], 'stages': report['stages'],
+        'finding_counts': {k: len(v) for k, v in report['findings'].items()},
+        'errors': len(report['errors'])}))
diff --git a/tools/gpu-aot/compare_placement_shader.py b/tools/gpu-aot/compare_placement_shader.py
new file mode 100644
index 0000000..65d5789
--- /dev/null
+++ b/tools/gpu-aot/compare_placement_shader.py
@@ -0,0 +1,53 @@
+"""Rank exact-prefix offline candidates against a bounded raw placement dump.
+
+Diagnostics only: similarity is NOT authorization to bind a shader.
+"""
+import argparse
+from pathlib import Path
+import struct
+
+def compare(dump, root, stage):
+    runtime = dump.read_bytes()
+    candidates = []
+    for path in root.rglob('*'):
+        if not path.is_file():
+            continue
+        data = path.read_bytes()
+        offset = 0
+        while True:
+            offset = data.find(b'\x10\x2a\x11', offset)
+            if offset < 0: break
+            pos = offset
+            offset += 1
+            if pos + 36 > len(data): continue
+            flags, virtual, physical = struct.unpack_from('>III', data, pos)
+            if bool(flags & 1) != (stage == 'vs'): continue
+            if virtual < 36 or max(virtual, physical) > 4 * 1024 * 1024: continue
+            if pos + virtual + 4 > len(data): continue
+            marker = struct.unpack_from('>I', data, pos + virtual)[0] == physical
+            physical_base = pos + virtual + (4 if marker else 0)
+            if physical_base + physical > len(data): continue
+            shader_offset = struct.unpack_from('>I', data, pos + 24)[0]
+            if shader_offset + 8 > virtual: continue
+            code_offset, code_size = struct.unpack_from('>II', data, pos + shader_offset)
+            for kind, start, size in [('physical', physical_base, physical),
+                                      ('code', physical_base + code_offset, code_size)]:
+                if size < 8 or size > len(runtime) or start + size > physical_base + physical: continue
+                blob = data[start:start + size]
+                if blob[:8] != runtime[:8]: continue
+                differences = [i for i in range(size) if blob[i] != runtime[i]]
+                candidates.append((len(differences), -size, str(path), pos, kind, start, differences))
+    for count, negsize, path, pos, kind, start, differences in sorted(candidates)[:12]:
+        print(f'{path}: container={pos} {kind}={start} size={-negsize} differing_bytes={count}')
+        data = Path(path).read_bytes()
+        for i in differences[:24]:
+            print(f'  +{i:04X}: offline={data[start+i]:02X} runtime={runtime[i]:02X}')
+    print(f'{len(candidates)} same-stage exact-prefix candidates (not binding decisions)')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('dump', type=Path)
+    parser.add_argument('root', type=Path)
+    parser.add_argument('stage', choices=['vs', 'ps'])
+    args = parser.parse_args()
+    compare(args.dump, args.root, args.stage)
diff --git a/tools/gpu-aot/d3d-hook-map.md b/tools/gpu-aot/d3d-hook-map.md
index 8af0fb9..ec270d4 100644
--- a/tools/gpu-aot/d3d-hook-map.md
+++ b/tools/gpu-aot/d3d-hook-map.md
@@ -32,6 +32,17 @@ work.
 
 ## TU23 device layout anchors
 
+### Tiling lifecycle (startup crash, 2026-09-29)
+
+Sep'13 symbol matching identifies `83FBCD28` as `D3DDevice_BeginTiling`,
+`83FBD098` as `D3DDevice_EndTiling`, and `83FCAB10` as
+`D3D::StartWorkerQueue`. The 17:03 crash (`lr=83FCABCC`, guest write `0x4`)
+occurs in this worker initialization, not in audio decoding. The native path
+now hooks the outer Begin/End pair to clear/draw/resolve full-size host targets
+without invoking the Xbox worker queue. Begin's bit0 skips the initial clear,
+as confirmed at original branch `83FBCF08`. Tile-specific resolve rectangles,
+depth conversion and GPU-to-CPU readback are not fully implemented.
+
 The Sep'13 device is `0x6080` bytes, not the `0x5000`-byte device used by the
 older re:Blue target. The reset loop at `0x83FC9450` proves these offsets:
 
diff --git a/tools/gpu-aot/extract_sampler_oracle.py b/tools/gpu-aot/extract_sampler_oracle.py
new file mode 100644
index 0000000..9789d8a
--- /dev/null
+++ b/tools/gpu-aot/extract_sampler_oracle.py
@@ -0,0 +1,25 @@
+"""Extract the unchanged TU23 SetTexture body for the differential unit test.
+
+The test supplies a bounded BE guest-memory/CPU harness. The Xbox queue path
+must not run; only the texture/fetch/dirty shadow writes are compared.
+"""
+import argparse
+from pathlib import Path
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--generated', type=Path, default=Path('rexlego/generated/default'))
+    parser.add_argument('--output', type=Path, required=True)
+    args = parser.parse_args()
+    matches = []
+    for path in args.generated.glob('*.cpp'):
+        source = path.read_text()
+        begin = source.find('DEFINE_REX_FUNC(sub_83FB58A8) {')
+        if begin == -1:
+            continue
+        end = source.find('\nDEFINE_REX_FUNC(', begin + 1)
+        matches.append(source[begin:end if end != -1 else len(source)])
+    if len(matches) != 1:
+        raise SystemExit(f'Expected one TU23 function, found {len(matches)}')
+    args.output.write_text(matches[0], encoding='utf-8')
+    print(f'Extracted unchanged SetTexture into {args.output}')
diff --git a/tools/gpu-aot/generate_microcode_index.cpp b/tools/gpu-aot/generate_microcode_index.cpp
new file mode 100644
index 0000000..3f77cbe
--- /dev/null
+++ b/tools/gpu-aot/generate_microcode_index.cpp
@@ -0,0 +1,115 @@
+/** Builds the placement-shader lookup without invoking DXC. */
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <map>
+#include <vector>
+#include <xxhash.h>
+#include "../../rexlego/src/gpu_native/shader_texture_usage.h"
+
+namespace {
+uint32_t Be32(const uint8_t* p) {
+  return uint32_t(p[0]) << 24 | uint32_t(p[1]) << 16 |
+         uint32_t(p[2]) << 8 | uint32_t(p[3]);
+}
+struct Entry {
+  uint64_t prefix_hash, microcode_hash, container_hash;
+  uint32_t microcode_size, stage, texture_mask;
+};
+void ScanFile(const std::filesystem::path& path,
+              std::map<std::pair<uint64_t, uint64_t>, Entry>& entries) {
+  std::ifstream input(path, std::ios::binary | std::ios::ate);
+  if (!input) return;
+  const auto end = input.tellg();
+  if (end <= 0) return;
+  std::vector<uint8_t> data(static_cast<size_t>(end));
+  input.seekg(0);
+  input.read(reinterpret_cast<char*>(data.data()), end);
+  constexpr size_t kHeaderSize = 0x24;
+  for (size_t offset = 0; offset + kHeaderSize <= data.size();) {
+    const uint8_t* container = data.data() + offset;
+    const uint32_t flags = Be32(container);
+    const size_t virtual_size = Be32(container + 4);
+    const size_t physical_size = Be32(container + 8);
+    const size_t shader_offset = Be32(container + 0x18);
+    constexpr size_t kMaximumSectionSize = 4 * 1024 * 1024;
+    const bool sizes_ok = virtual_size <= kMaximumSectionSize &&
+                          physical_size <= kMaximumSectionSize;
+    const bool marker = sizes_ok && virtual_size + 4 <= data.size() - offset &&
+                        Be32(container + virtual_size) == physical_size;
+    const size_t physical_base = size_t(virtual_size) + (marker ? 4 : 0);
+    const size_t total_size = physical_base + physical_size;
+    const bool header_ok = sizes_ok &&
+                           (flags & 0xFFFFFF00u) == 0x102A1100u &&
+                           Be32(container + 0x1C) == 0 &&
+                           Be32(container + 0x20) == 0;
+    if (!header_ok || total_size > data.size() - offset ||
+        shader_offset + 8 > virtual_size) {
+      ++offset;
+      continue;
+    }
+    const uint8_t* shader = container + shader_offset;
+    const size_t physical_offset = Be32(shader);
+    const uint32_t microcode_size = Be32(shader + 4);
+    const size_t microcode_offset = physical_base + physical_offset;
+    if (microcode_size < 8 || microcode_offset > total_size ||
+        microcode_size > total_size - microcode_offset) {
+      offset += total_size;
+      continue;
+    }
+    const uint8_t* microcode = container + microcode_offset;
+    const uint32_t texture_mask = legodimensions::gpu_native::ShaderTextureUsage(
+        {microcode, microcode_size});
+    const uint64_t container_hash = XXH3_64bits(container, total_size);
+    const uint32_t stage = (flags & 1u) == 0 ? 1u : 0u;
+    const auto add_signature = [&](const uint8_t* bytes, uint32_t size) {
+      Entry entry{XXH3_64bits(bytes, 8), XXH3_64bits(bytes, size),
+                  container_hash, size, stage, texture_mask};
+      entries.emplace(std::pair{entry.container_hash, entry.microcode_hash},
+                      entry);
+    };
+    // Depending on the SDK path, placement objects retain either the copied
+    // physical section base or the instruction stream inside that section.
+    add_signature(container + physical_base,
+                  static_cast<uint32_t>(physical_size));
+    add_signature(microcode, microcode_size);
+    offset += total_size;
+  }
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+  if (argc < 3) {
+    std::fprintf(stderr, "usage: lego_gpu_microcode_index <output.cpp> <input-dir>...\n");
+    return 1;
+  }
+  std::map<std::pair<uint64_t, uint64_t>, Entry> unique;
+  for (int i = 2; i < argc; ++i) {
+    const std::filesystem::path root(argv[i]);
+    if (!std::filesystem::exists(root)) continue;
+    for (const auto& item : std::filesystem::recursive_directory_iterator(root))
+      if (item.is_regular_file()) ScanFile(item.path(), unique);
+  }
+  std::vector<Entry> entries;
+  for (const auto& [key, entry] : unique) entries.push_back(entry);
+  std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) {
+    if (a.prefix_hash != b.prefix_hash) return a.prefix_hash < b.prefix_hash;
+    if (a.stage != b.stage) return a.stage < b.stage;
+    return a.microcode_hash < b.microcode_hash;
+  });
+  std::FILE* output = std::fopen(argv[1], "wb");
+  if (!output) return 2;
+  std::fprintf(output, "#include \"gpu_native/shader_archive.h\"\nShaderMicrocodeEntry g_shaderMicrocodeEntries[] = {\n");
+  for (const Entry& e : entries)
+    std::fprintf(output, "  { 0x%016llX, 0x%016llX, 0x%016llX, %u, %u, 0x%08X },\n",
+                 static_cast<unsigned long long>(e.prefix_hash),
+                 static_cast<unsigned long long>(e.microcode_hash),
+                 static_cast<unsigned long long>(e.container_hash),
+                 e.microcode_size, e.stage, e.texture_mask);
+  std::fprintf(output, "};\nconst size_t g_shaderMicrocodeEntryCount = %zu;\n", entries.size());
+  std::fclose(output);
+  std::printf("Indexed %zu shader microcode blobs\n", entries.size());
+  return entries.empty() ? 3 : 0;
+}
diff --git a/tools/gpu-aot/inspect_gpu_snapshot.py b/tools/gpu-aot/inspect_gpu_snapshot.py
new file mode 100644
index 0000000..0b7e06b
--- /dev/null
+++ b/tools/gpu-aot/inspect_gpu_snapshot.py
@@ -0,0 +1,53 @@
+"""Inspect uncompressed DX10 DDS GPU snapshots without third-party packages.
+
+Directory mode prints sampled RGB ranges. --image renders one snapshot as PPM;
+--scale is an explicit diagnostic exposure multiplier, not an in-game change.
+"""
+import argparse
+import math
+from pathlib import Path
+import struct
+
+
+def inspect(path, image, scale):
+    data = path.read_bytes()
+    assert data[:4] == b'DDS ' and data[84:88] == b'DX10'
+    height, width = struct.unpack_from('<II', data, 12)
+    fmt = struct.unpack_from('<I', data, 128)[0]
+    code, divisor, channels = {
+        2: ('4f', 1, (0, 1, 2)),
+        28: ('4B', 255, (0, 1, 2)), 87: ('4B', 255, (2, 1, 0)),
+        10: ('4e', 1, (0, 1, 2)), 11: ('4H', 65535, (0, 1, 2)),
+        13: ('4h', 32767, (0, 1, 2)),
+        34: ('2e', 1, (0, 1, 0)), 35: ('2H', 65535, (0, 1, 0)),
+        41: ('f', 1, (0, 0, 0)),
+    }[fmt]
+    pixel = struct.Struct('<' + code)
+    count = width * height
+    assert len(data) == 148 + count * pixel.size
+    def rgb(index):
+        value = pixel.unpack_from(data, 148 + index * pixel.size)
+        return tuple(value[c] / divisor for c in channels)
+    samples = [rgb(i) for i in range(0, count, max(1, count // 4096))]
+    finite = [v for sample in samples for v in sample if math.isfinite(v)]
+    nonzero = sum(any(math.isfinite(c) and abs(c) > 1e-8 for c in p) for p in samples)
+    print(f'{path.name} {width}x{height} dxgi={fmt} '
+          f'range={min(finite, default=0):.5g}..{max(finite, default=0):.5g} '
+          f'nonzero={nonzero}/{len(samples)}')
+    if image:
+        out = bytearray()
+        for i in range(count):
+            for channel in rgb(i):
+                out.append(round(max(0, min(1, channel * scale)) * 255)
+                           if math.isfinite(channel) else 255)
+        path.with_suffix('.ppm').write_bytes(f'P6\n{width} {height}\n255\n'.encode() + out)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('path', type=Path)
+parser.add_argument('--image', action='store_true')
+parser.add_argument('--scale', type=float, default=1)
+args = parser.parse_args()
+paths = sorted(args.path.glob('*.dds')) if args.path.is_dir() else [args.path]
+for path in paths:
+    inspect(path, args.image, args.scale)
diff --git a/tools/gpu-aot/match_mesh_assets.py b/tools/gpu-aot/match_mesh_assets.py
new file mode 100644
index 0000000..6b1d2ce
--- /dev/null
+++ b/tools/gpu-aot/match_mesh_assets.py
@@ -0,0 +1,64 @@
+"""Match complete traced upload records against decompressed guest assets offline."""
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+def guest_record(observations):
+    attrs = []
+    for line in observations:
+        m = re.fullmatch(r'attr=(\w+) slot=0 offset=(\d+) stride=(\d+) format=\d+ reversed=(\d+) length=\d+ sampled=true words=([0-9A-F]{8}),([0-9A-F]{8})', line)
+        if m:
+            _, offset, stride, reverse, a, b = m.groups()
+            attrs.append((int(offset), int(stride), int(reverse),
+                          int(a, 16).to_bytes(4, 'little') + int(b, 16).to_bytes(4, 'little')))
+    if not attrs or len({a[1] for a in attrs}) != 1:
+        return None
+    stride = attrs[0][1]
+    data = [None] * stride
+    for offset, _, _, raw in attrs:
+        for i, byte in enumerate(raw[:max(0, stride-offset)]):
+            if data[offset+i] is not None and data[offset+i] != byte:
+                raise ValueError('Overlapping trace samples disagree')
+            data[offset+i] = byte
+    if None in data:
+        return None
+    data = bytes(data)
+    # Undo per-field WZYX, then the common guest DWORD endian conversion.
+    for offset, _, reverse, _ in attrs:
+        if reverse:
+            data = data[:offset] + data[offset:offset+4][::-1] + data[offset+4:]
+    return b''.join(data[i:i+4][::-1] for i in range(0, stride, 4))
+
+
+def match(report, assets):
+    rows = []
+    for draw in report['draws']:
+        record = guest_record(draw['observations'])
+        if record is None:
+            continue
+        matches = []
+        for name, data in assets.items():
+            offset = data.find(record)
+            while offset >= 0:
+                matches.append({'asset': name, 'offset': offset})
+                offset = data.find(record, offset+1)
+        rows.append({'id': draw['id'], 'VS': draw['VS'], 'PS': draw['PS'],
+                     'start': draw['start'], 'count': draw['count'], 'base': draw['base'],
+                     'guest_record': record.hex(), 'bytes': len(record), 'matches': matches})
+    return rows
+
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('summary', type=Path)
+    p.add_argument('assets', type=Path)
+    p.add_argument('output', type=Path)
+    a = p.parse_args()
+    assets = {f.name: f.read_bytes() for f in a.assets.glob('*.ghg')}
+    rows = match(json.loads(a.summary.read_text()), assets)
+    result = {'trace': str(a.summary.resolve()), 'assets': {k: len(v) for k,v in assets.items()},
+              'note': 'Full byte-record matches identify stored vertex data, not scene instance or correctness of every vertex/bone.', 'rows': rows}
+    a.output.write_text(json.dumps(result, indent=2)+'\n')
+    print(json.dumps(rows, indent=2))
diff --git a/tools/gpu-aot/merge_microcode_index.py b/tools/gpu-aot/merge_microcode_index.py
new file mode 100644
index 0000000..219fdff
--- /dev/null
+++ b/tools/gpu-aot/merge_microcode_index.py
@@ -0,0 +1,58 @@
+"""Extend an existing placement index while preserving every old signature.
+
+Inputs are the generated ShaderMicrocodeEntry C++ format. Conflicting records
+are refused rather than silently replacing metadata for a compiled shader.
+"""
+import argparse
+import json
+from pathlib import Path
+import re
+
+
+ROW = re.compile(r'\{\s*0x([\dA-Fa-f]+),\s*0x([\dA-Fa-f]+),\s*'
+                 r'0x([\dA-Fa-f]+),\s*(\d+),\s*(\d+),\s*0x([\dA-Fa-f]+)\s*\}')
+
+
+def read(path):
+    source = path.read_text()
+    rows = [tuple(int(v, 10 if i in (3, 4) else 16) for i, v in enumerate(match))
+            for match in ROW.findall(source)]
+    declared = re.search(r'g_shaderMicrocodeEntryCount\s*=\s*(\d+)', source)
+    if not declared or len(rows) != int(declared[1]):
+        raise ValueError(f'Index record count mismatch: {path}')
+    if any(r[3] < 8 or r[4] not in (0, 1) for r in rows):
+        raise ValueError(f'Invalid size or stage: {path}')
+    return rows
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('old', type=Path)
+    parser.add_argument('addition', type=Path)
+    parser.add_argument('output', type=Path)
+    parser.add_argument('--report', type=Path, required=True)
+    args = parser.parse_args()
+    before, addition = read(args.old), read(args.addition)
+    combined = {}
+    for row in before + addition:
+        key = (row[2], row[1])  # container and entire physical/instruction hash
+        if key in combined and combined[key] != row:
+            raise ValueError(f'Conflicting metadata: {key}')
+        combined[key] = row
+    rows = sorted(combined.values(), key=lambda r: (r[0], r[4], r[1], r[2]))
+    assert set(before) <= set(rows) and set(addition) <= set(rows)
+    result = '#include "gpu_native/shader_archive.h"\nShaderMicrocodeEntry g_shaderMicrocodeEntries[] = {\n'
+    result += ''.join(f'  {{ 0x{r[0]:016X}, 0x{r[1]:016X}, 0x{r[2]:016X}, '
+                      f'{r[3]}, {r[4]}, 0x{r[5]:08X} }},\n' for r in rows)
+    result += f'}};\nconst size_t g_shaderMicrocodeEntryCount = {len(rows)};\n'
+    args.output.write_text(result, encoding='utf-8')
+    args.report.write_text(json.dumps({
+        'before': len(before), 'addition': len(addition), 'after': len(rows),
+        'added': len(rows) - len(before), 'preserved_all_old_records': True,
+        'included_all_addition_records': True,
+    }, indent=2), encoding='utf-8')
+    print(f'Placement index: {len(before)} -> {len(rows)}, all prior records preserved')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/gpu-aot/native-depth-alias-findings-20261001.md b/tools/gpu-aot/native-depth-alias-findings-20261001.md
new file mode 100644
index 0000000..9d3bec5
--- /dev/null
+++ b/tools/gpu-aot/native-depth-alias-findings-20261001.md
@@ -0,0 +1,59 @@
+# Native missing-geometry fix: depth aliases
+
+The missing TT logo was a depth failure, not a missing draw or incorrect alpha.
+RenderDoc `oct01-native-logo-rdc1`, draw 514, showed correct clip positions and
+gold/cyan pixel output, but depth 0.935637 failed against an incorrectly empty
+depth buffer (0). Draw 530 contained the production text.
+
+Two missing resource transfers caused this:
+
+1. Depth texture header `ACEB028C` and raw RGBA8 header `ACEB0034` share allocation
+   `FE4D7000` (physical `1E4D8000`). The native depth resolve populated an R32
+   sampling mirror for the first header; the second header uploaded stale CPU
+   bytes instead. Native now packs normalized depth into D24 bits shifted left
+   by 8 when an otherwise matching RGBA8 view samples that allocation.
+2. The game copies those raw bytes through PS `29E6F90BBBCE5539` into color
+   surface `30172000`, EDRAM base 1328. Subsequent forward objects use depth
+   surface `30210000` at the same base. Native now transfers matching color
+   storage to depth with the Xenos 40-sample half-tile permutation.
+
+The original shader already swaps 40 columns; the color-to-depth transfer must
+perform the corresponding storage permutation too. Reference implementation:
+`rexglue-sdk/src/graphics/d3d12/render_target_cache.cpp`, transfer column swap
+around line 2741 and D24 packing around line 5344. No game shader hash is replaced,
+and normal depth tests remain enabled.
+
+The native path is enabled by default. `LEGO_NATIVE_DEPTH_ALIAS=0`, or probe
+`-DisableDepthAlias`, provides an explicit comparison with the broken path.
+The ordinary launch's renderer selection is separately controlled by
+`gpu_native_pm4`: false = native, true = full SDK GPU. These are not live switches.
+
+## Evidence
+
+- Full build succeeds. `test_depth_alias.py` checks all 16,777,216 depth values
+  for monotonicity and half-LSB error, all 921,600 image coordinates against
+  EDRAM addressing, and executes the actual resource tracker with mocked GPU
+  transfers to test write order, failed-copy retry, expiry and rejection cases.
+- Native `oct01-native-depth-alias1`, RenderDoc draw 565: stored depth
+  0.99890226, logo depth 0.9356377, depthTestFailed=false. Logo and production
+  text are visible in the final swap-chain image.
+- That same native run visibly restores translucent Vortech arms, body and
+  legs in the observed minifigure shots, plus portal effects. Its title screen
+  has the central portal, glowing spheres, orbit lines and complete characters.
+- Evidence lives under `rexlego/out/native-gpu/session-20261001/`:
+  `native-depth-alias1/`, `native-depth-alias-replay1/`, `depth-alias-tests/`.
+  Original GPU captures are under `session-20260930/renderdoc-oct01-native-*`.
+
+## Limits
+
+The implemented EDRAM transfer covers equal-base, equal-extent single-sample
+RGBA8/D24 surfaces with complete 80-pixel tiles. This is not a general EDRAM
+overlap/MSAA/pitch conversion implementation. The packed sampling mirror carries
+depth; stencil bytes are not propagated, and restore preserves destination stencil.
+Float24 depth and arbitrary texture reinterpretations are not covered.
+
+No gameplay-wide correctness or FPS claim is made. Giant brick-form closeups
+were not independently captured. F3/F4 overlays still belong to the SDK GPU path;
+native has no immediate drawer implementation. The separate bulk attachment hook
+passes offline tests but was not called in the observed intro, and did not itself
+fix the logo.
diff --git a/tools/gpu-aot/native-performance-20261001.md b/tools/gpu-aot/native-performance-20261001.md
new file mode 100644
index 0000000..89e41da
--- /dev/null
+++ b/tools/gpu-aot/native-performance-20261001.md
@@ -0,0 +1,159 @@
+# Native performance investigation, October 1
+
+User priority: improve FPS first; missing heads/HUD/details in hub remain open.
+Runs are authorized on monitor 1. Close every probe when finished. User input
+may be needed to enter a save and reach the hub. Do not infer native mode from
+F3/F4: those overlays remain unavailable in native.
+
+## Evidence before changes
+
+Original user log preserved at
+`rexlego/out/native-gpu/session-20261001/perf-hub/user-last-game.log`.
+It includes resource/frame-slot GPU waits and placement shader misses with
+prefixes C77B3ABB6F87ACD9 (VS, zero first words) / F7F244F983B4B08F (PS,
+first words look like float constants). Their relationship to missing heads/HUD
+is unproven. These prefixes were already present in September 30 hub evidence.
+
+Baseline executable SHA256:
+`1F873E4F113526882D4EFE07C48C3B49E1E49B9D936C7C3EEDFC43A189A0037C`.
+Config SHA256:
+`E97CAFFAC325E5CBDF1A8D25FD757A0BA6911EAE844D77D9E9D7D6C2CA71BF5B`.
+
+Probe `oct01-fps-baseline1`: native, depth alias enabled, timing CSV, bounded
+missing-shader capture, no RenderDoc. Visually confirmed title. At frame 3217,
+last 20.05 s: 10.87 FPS, median 90.29 ms, p95 101.86 ms, texture CPU 33.49 ms,
+constants 36.73 ms (includes texture CPU), vertices 10.84 ms, buffer hash 9.51 ms,
+resource waits 5.61 ms. About 1080 draws/frame. Closed PID 36200.
+
+Probe `oct01-fps-windows1`: same exe with BufferWindows enabled; other candidate
+options off. Title visually confirmed. At frame 3392, last 20.08 s: 9.76 FPS,
+median 97.60 ms, p95 129.47 ms, buffer hash 0.89 ms, vertices 9.87 ms,
+texture CPU 37.51 ms. Animation phases differ, so this is not a deterministic
+benchmark. No demonstrated overall FPS gain. Window uploads converted about
+5 MB/frame and repeatedly allocated GPU buffers. Closed PID 36048.
+
+System samples during title: 14672 MB available RAM, 3 pages input/s, global GPU
+utilization 21%, global VRAM 2006/8188 MiB. Browser also running. These are title
+samples, not hub measurements; no evidence of memory capacity exhaustion.
+
+## Changes under validation
+
+1. `BaseHeap::IsRangeCommittedReadable` checks only requested pages under the
+heap mutex, including commitment and read protection. Previous texture
+validation called QueryRegionInfo, which scans to the end of the allocation
+on each bind. Texture content hashing remains unchanged by this change.
+2. Buffer window cache replaces unordered-map begin eviction/64 entries with
+LRU, a per-parent byte budget (256 KiB to 16 MiB based on parent length) and
+4096 entry cap. A single oversized required range may exceed the byte budget.
+Still opt-in; immutable upload lifetime and every-use hashing preserved.
+3. `LEGO_NATIVE_AUDIT_TEXTURE_WATCH` / probe `-AuditTextureWatch` verifies each
+would-be pixel-texture cache hit against complete source hashes. Logs false
+clean stamps as errors. Vertex textures always force hashing. Audit results
+are required before considering promotion of the existing pixel-only watch.
+
+Offline checks: actual bounded range method passed holes, reserved/read-only
+pages, unaligned bounds, 4 GiB edge and 10000 differential cases. Actual buffer
+resolve/adoption code passed 10000 fetched-byte equivalence cases, relocation,
+same-frame writes, old upload lifetime, 1000 rebasing cases and a 256-range
+working set reused for three frames with no new GPU allocation (fake driver).
+
+Build note: game links installed SDK, not SDK source directly. Initial game-only
+build failed because the installed header lacked the new method. SDK Release
+install succeeded; rexruntime.dll deployed (previous DLL preserved in perf-hub).
+Game rebuild pending at time of this entry. Do not treat old exe as updated.
+
+Follow-up: full SDK install/game build succeeded, followed by an incremental
+check. Executable SHA256 E5666C247F0D4BB9F771547C0D51B6D96406716BD1C6DF56B3299E30F58205E7.
+SDK build now identifies as 0.10.0.40-dev.gd846a43; config remains native. The
+SDK install caused CMake regeneration and extensive recompilation; no generated
+game functions were changed by codegen. Previous runtime DLL preserved.
+
+Shader placement lookup now hashes each distinct candidate byte length once
+per lookup, preserving the candidate iteration/matching and rehashing on every
+new call. Actual-function test: 3001 shared-prefix entries, stage/missing/hit
+checks, mutation at same pointer, at most four hashes instead of 1501.
+
+Real SDK integration test `test_memory_watch_sdk.py` also passed: native watcher
+plus installed runtime, real protected-page CPU writes/rearming, Protect,
+explicit physical-copy invalidation, Decommit and bounded commitment query.
+Standalone fixture needed the same LLD linker/runtime flags as the application.
+
+Probe `oct01-fps-audit1` launched on monitor 1 with range cache + full texture
+watch verification + bounded missing shader capture. This is a correctness
+audit, not a clean performance sample. Completed 3300000 verified cache hits
+with zero stale-content mismatches through intro/title; PID 28740 closed.
+
+## Resolution
+
+Window/video-mode/resolution config already requests 1920x1080/1080p. Main game
+render targets are still 1280x720. Generated game initializer sub_82B67190
+(legodimensions_recomp.133.cpp) explicitly passes 1280/720 repeatedly to
+sub_82B64790 and related target creation. Native device also starts with a
+1280x720 viewport. A config edit alone is not true 1080p. Scaling must preserve
+postprocessing, depth aliases and the guest's 40-column depth-bank permutation;
+no true 1080p implementation has been validated yet.
+
+## End of October 1: measured hub and saved configuration
+
+All game probes are closed. The user is going to sleep and requested wrapping
+up after the hub measurement. No more launches tonight. Final executable built
+successfully; its final config-backed option wiring was checked offline, not
+with another game launch. The rendering algorithms had already run in the
+earlier audit/fast probes.
+
+`oct01-fps-fast1` (E5666C... executable, BufferWindows and StaticTextureWatch
+explicitly enabled): visually confirmed title, last 25.01 s at frame 8400:
+20.23 FPS, median 49.39 ms, p95 53.00 ms; texture CPU 0.79 ms, vertices 6.15 ms,
+buffer hash 0.80 ms. Animation phases differ from baseline, so this is an
+indicative title comparison, not a deterministic benchmark. Giant red minifigure
+passed in front of the title camera; saved image `fast1-title-anomaly.jpg` should
+not alone be taken as proof of a new geometry bug. PID 28948 closed.
+
+`oct01-fps-textures1` PID 32380: user loaded the save and reported ready.
+Foreground confirmed hub, headless Sonic and missing HUD reproduced. Foreground
+frames 5881..6350: 470 frames / 22.85343 s = **20.56584 FPS**, median 48.0882 ms,
+p95 54.405 ms, max 79.183 ms. Texture CPU 0.640 ms, vertices 10.324 ms, buffer
+hash 1.463 ms, about 505 draws/frame. No explicit Synchronize calls in that
+window; other frame-ring waits still exist (begin ~10 ms in sampled logs).
+This is the first measured hub interval; there is no matching pre-change hub
+baseline, so do not claim a measured hub speedup ratio.
+
+Hub resource sample: working set 2283 MiB, private 2152 MiB, 15178 MB free RAM,
+zero pages input/read per second. Hottest render thread ~75% of one core;
+whole process ~1.18 cores. Global GPU 36%, VRAM 1885/8188 MiB (other apps share
+these GPU counters). No evidence of memory capacity pressure. Repeated scans,
+conversions, allocations, CPU work and frame-ring waits remain the leads.
+
+Important probe caveat: textures1 ran the earlier E5666C... executable after the
+launcher gained explicit `LEGO_NATIVE_BUFFER_WINDOWS=0`. That older executable's
+presence-only option handling may treat `0` as enabled. Its flag record says
+false but is NOT proof of a buffer-cache-off A/B. Use the measured FPS as scene
+evidence, not as isolation of that switch. Final build parses `0` explicitly,
+logs its effective range-cache state, and future probes save the config plus
+actual environment overrides. This is not a renderer-route ambiguity: both
+probes explicitly used native D3D12.
+
+Final saved config (legodimensions.toml): native D3D12, gpu_native_pm4=false,
+gpu_native_texture_watch=true, gpu_native_buffer_windows=true, monitor=1,
+output/window 1920x1080 but internal targets 1280x720, depth alias enabled,
+DoF=false, frame_rate=60/framerate_limit=240 (not measured native FPS), existing
+pixel-shader skip CFEAC7ADB912F8A9 retained. AsyncCpuResources/Stencil/Viewport
+candidates OFF. F3/F4 still unavailable. Saved xenos plugin/backend fields are
+fallback settings; native bootstrap disables the SDK GPU plugin.
+
+Final executable SHA256:
+7A8ECB1F61D466FFC0A7D5765B99885353588EBDD7D9DB36F218D3DF5710E786.
+Config SHA256: AD4B5E767FC3C61CEA6DCF6BB78D2D7503FD78428E097D43141E90E44EEF50E5.
+Runtime SHA256: 42DCF8CBE7083AD69DE129B29F19F685C4A57D326CC06E9AFC09A7E5C57501B8.
+Proof: `session-20261001/perf-hub/final-build-config.json`.
+Offline final checks passed model callbacks/disabled overrides, real SDK page
+fault/rearm/Protect/Decommit/physical writes both env-selected and cvar-selected,
+TOML parse, and full native build. CSV now retains all draw stages for the next
+profile. No SDK rebuild was needed for final config wiring.
+
+Next: confirm saved config on a fresh launch, capture full missing shader
+containers in the hub, profile vertex upload allocation/conversion plus
+frame-ring waits. Missing heads/HUD/details and true 1080p remain open.
+No new video recordings. Intentional evidence is on E under
+`rexlego/out/native-gpu/session-20261001/perf-hub` and named oct01-fps files
+under `session-20260930`; transient app/Sky caches may still use C.
diff --git a/tools/gpu-aot/patches/xenosrecomp-lego.patch b/tools/gpu-aot/patches/xenosrecomp-lego.patch
index 40fbe1d..de404ff 100644
--- a/tools/gpu-aot/patches/xenosrecomp-lego.patch
+++ b/tools/gpu-aot/patches/xenosrecomp-lego.patch
@@ -1,10 +1,5 @@
-warning: in the working copy of 'XenosRecomp/dxc_compiler.cpp', LF will be replaced by CRLF the next time Git touches it
-warning: in the working copy of 'XenosRecomp/main.cpp', LF will be replaced by CRLF the next time Git touches it
-warning: in the working copy of 'XenosRecomp/shader_common.h', LF will be replaced by CRLF the next time Git touches it
-warning: in the working copy of 'XenosRecomp/shader_recompiler.cpp', LF will be replaced by CRLF the next time Git touches it
-warning: in the working copy of 'XenosRecomp/shader_recompiler.h', LF will be replaced by CRLF the next time Git touches it
 diff --git a/XenosRecomp/dxc_compiler.cpp b/XenosRecomp/dxc_compiler.cpp
-index 4a851c2..ceffb3f 100644
+index 4a851c2..412aa39 100644
 --- a/XenosRecomp/dxc_compiler.cpp
 +++ b/XenosRecomp/dxc_compiler.cpp
 @@ -1,5 +1,7 @@
@@ -15,7 +10,17 @@ index 4a851c2..ceffb3f 100644
  DxcCompiler::DxcCompiler()
  {
      HRESULT hr = DxcCreateInstance(CLSID_DxcCompiler, IID_PPV_ARGS(&dxcCompiler));
-@@ -82,7 +84,19 @@ IDxcBlob* DxcCompiler::compile(const std::string& shaderSource, bool compilePixe
+@@ -62,6 +64,9 @@ IDxcBlob* DxcCompiler::compile(const std::string& shaderSource, bool compilePixe
+     args[argCount++] = L"-DUNLEASHED_RECOMP";
+ #endif
+ 
++    if (std::getenv("XENOS_RECOMP_LEGO_NATIVE_SCALE"))
++        args[argCount++] = L"-DLEGO_NATIVE_GPU";
++
+     IDxcResult* result = nullptr;
+     HRESULT hr = dxcCompiler->Compile(&source, args, argCount, nullptr, IID_PPV_ARGS(&result));
+ 
+@@ -82,7 +87,19 @@ IDxcBlob* DxcCompiler::compile(const std::string& shaderSource, bool compilePixe
                  hr = result->GetOutput(DXC_OUT_ERRORS, IID_PPV_ARGS(&errors), nullptr);
                  assert(SUCCEEDED(hr) && errors != nullptr);
  
@@ -37,10 +42,24 @@ index 4a851c2..ceffb3f 100644
                  errors->Release();
              }
 diff --git a/XenosRecomp/main.cpp b/XenosRecomp/main.cpp
-index 754198c..5ee2ca9 100644
+index 754198c..dcab846 100644
 --- a/XenosRecomp/main.cpp
 +++ b/XenosRecomp/main.cpp
-@@ -31,6 +31,8 @@ struct RecompiledShader
+@@ -2,6 +2,7 @@
+ #include "shader_recompiler.h"
+ #include "dxc_compiler.h"
+ 
++#include <algorithm>
+ #include <mutex>
+ #include <vector>
+ 
+@@ -26,11 +27,13 @@ static void writeAllBytes(const char* filePath, const void* data, size_t dataSiz
+ 
+ struct RecompiledShader
+ {
+-    uint8_t* data = nullptr;
++    std::vector<uint8_t> data;
+     IDxcBlob* dxil = nullptr;
      std::vector<uint8_t> spirv;
      uint32_t specConstantsMask = 0;
      std::string sourceName;
@@ -49,7 +68,15 @@ index 754198c..5ee2ca9 100644
  };
  
  // Per-shader recompile failures, collected from the parallel loop and reported before exit.
-@@ -83,6 +85,8 @@ int main(int argc, char** argv)
+@@ -45,6 +48,7 @@ static std::vector<ShaderFailure> g_failures;
+ 
+ int main(int argc, char** argv)
+ {
++    setvbuf(stdout, nullptr, _IONBF, 0);
+ #ifndef XENOS_RECOMP_INPUT
+     if (argc < 4)
+     {
+@@ -83,13 +87,14 @@ int main(int argc, char** argv)
      else if (const char* env = std::getenv("XENOS_RECOMP_HLSL_DUMP"))
          hlslDumpDir = env;
  
@@ -58,17 +85,41 @@ index 754198c..5ee2ca9 100644
      size_t includeSize = 0;
      auto includeData = readAllBytes(includeInput, includeSize);
      std::string_view include(reinterpret_cast<const char*>(includeData.get()), includeSize);
-@@ -107,7 +111,8 @@ int main(int argc, char** argv)
+ 
+     if (std::filesystem::is_directory(input))
+     {
+-        std::vector<std::unique_ptr<uint8_t[]>> files;
+         std::map<XXH64_hash_t, RecompiledShader> shaders;
+ 
+         for (auto& file : std::filesystem::recursive_directory_iterator(input))
+@@ -107,7 +112,18 @@ int main(int argc, char** argv)
              for (size_t i = 0; fileSize > sizeof(ShaderContainer) && i < fileSize - sizeof(ShaderContainer) - 1;)
              {
                  auto shaderContainer = reinterpret_cast<const ShaderContainer*>(fileData.get() + i);
 -                size_t dataSize = shaderContainer->virtualSize + shaderContainer->physicalSize;
 +                const size_t virtualSize = shaderContainer->virtualSize;
-+                const size_t dataSize = virtualSize + sizeof(uint32_t) + shaderContainer->physicalSize;
++                const size_t physicalSize = shaderContainer->physicalSize;
++                const size_t unpaddedSize = virtualSize + physicalSize;
++                size_t physicalPadding = 0;
++                if (virtualSize + sizeof(uint32_t) <= fileSize - i)
++                {
++                    const auto marker = reinterpret_cast<const be<uint32_t>*>(
++                        fileData.get() + i + virtualSize);
++                    if (uint32_t(*marker) == physicalSize)
++                        physicalPadding = sizeof(uint32_t);
++                }
++                const size_t dataSize = unpaddedSize + physicalPadding;
  
                  if ((shaderContainer->flags & 0xFFFFFF00) == 0x102A1100 &&
                      dataSize <= (fileSize - i) &&
-@@ -123,6 +128,8 @@ int main(int argc, char** argv)
+@@ -118,11 +134,14 @@ int main(int argc, char** argv)
+                     auto shader = shaders.try_emplace(hash);
+                     if (shader.second)
+                     {
+-                        shader.first->second.data = fileData.get() + i;
++                        shader.first->second.data.assign(fileData.get() + i,
++                            fileData.get() + i + dataSize);
+                         std::string stem = file.path().stem().string();
                          if (containerIndex > 0)
                              stem += fmt::format(".{}", containerIndex);
                          shader.first->second.sourceName = std::move(stem);
@@ -77,7 +128,20 @@ index 754198c..5ee2ca9 100644
                          foundAny = true;
                      }
  
-@@ -142,12 +149,36 @@ int main(int argc, char** argv)
+@@ -131,23 +150,63 @@ int main(int argc, char** argv)
+                 }
+                 else
+                 {
+-                    i += sizeof(uint32_t);
++                    ++i; // TT shader records are byte-packed, not dword-aligned.
+                 }
+             }
+ 
+-            if (foundAny)
+-                files.emplace_back(std::move(fileData));
++            (void)foundAny;
+         }
+ 
          if (!hlslDumpDir.empty())
              std::filesystem::create_directories(hlslDumpDir);
  
@@ -97,17 +161,36 @@ index 754198c..5ee2ca9 100644
          std::atomic<uint32_t> progress = 0;
  
 -        std::for_each(std::execution::par_unseq, shaders.begin(), shaders.end(), [&](auto& hashShaderPair)
++        using ShaderPair = decltype(shaders)::value_type;
++        std::vector<ShaderPair*> compileOrder;
++        compileOrder.reserve(shaders.size());
++        for (auto& shader : shaders)
++            compileOrder.push_back(&shader);
++        // Runtime-generated containers use the markerless physical layout.
++        // Compile them before the large archive corpus so their result cannot
++        // depend on long-lived compiler state or peak batch memory.
++        std::stable_sort(compileOrder.begin(), compileOrder.end(),
++            [](const ShaderPair* lhs, const ShaderPair* rhs)
++            {
++                const bool lhsRuntime = lhs->second.sourcePath.find("runtime") != std::string::npos;
++                const bool rhsRuntime = rhs->second.sourcePath.find("runtime") != std::string::npos;
++                return lhsRuntime && !rhsRuntime;
++            });
++
 +        // DXC's COM implementation is unstable under this tool's parallel
 +        // invocation on Windows. Deterministic AOT is more important here.
-+        std::for_each(std::execution::seq, shaders.begin(), shaders.end(), [&](auto& hashShaderPair)
++        std::for_each(std::execution::seq, compileOrder.begin(), compileOrder.end(), [&](ShaderPair* hashShaderPair)
              {
-                 auto& shader = hashShaderPair.second;
-                 const XXH64_hash_t hash = hashShaderPair.first;
+-                auto& shader = hashShaderPair.second;
+-                const XXH64_hash_t hash = hashShaderPair.first;
++                auto& shader = hashShaderPair->second;
++                const XXH64_hash_t hash = hashShaderPair->first;
 +                if (std::getenv("XENOS_RECOMP_VERBOSE_RANGE") != nullptr)
 +                {
-+                    const auto container = reinterpret_cast<const ShaderContainer*>(shader.data);
-+                    const auto metadata = reinterpret_cast<const Shader*>(shader.data + container->shaderOffset);
-+                    fmt::println(stderr, "Container virtual={} physical={} shaderOffset={} codeOffset={} codeSize={}",
++                    const auto container = reinterpret_cast<const ShaderContainer*>(shader.data.data());
++                    const auto metadata = reinterpret_cast<const Shader*>(shader.data.data() + container->shaderOffset);
++                    fmt::println(stderr, "Container hash=0x{:016X} virtual={} physical={} shaderOffset={} codeOffset={} codeSize={}",
++                        hash,
 +                        uint32_t(container->virtualSize), uint32_t(container->physicalSize),
 +                        uint32_t(container->shaderOffset), uint32_t(metadata->physicalOffset), uint32_t(metadata->size));
 +                    fmt::println(stderr, "Source path={} offset={}", shader.sourcePath, shader.sourceOffset);
@@ -115,20 +198,47 @@ index 754198c..5ee2ca9 100644
  
                  auto recordFailure = [hash](const char* reason)
                  {
-@@ -159,6 +190,12 @@ int main(int argc, char** argv)
-                 recompiler = {};
-                 recompiler.recompile(shader.data, include);
+@@ -155,9 +214,27 @@ int main(int argc, char** argv)
+                     g_failures.push_back({hash, reason});
+                 };
  
+-                thread_local ShaderRecompiler recompiler;
+-                recompiler = {};
+-                recompiler.recompile(shader.data, include);
++                // ShaderRecompiler owns DXC/COM state and a large amount of
++                // per-program scratch storage. Reconstruct it for each shader:
++                // aggregate assignment left stale internal state after large
++                // LEGO batches and made runtime shaders order-dependent.
++                ShaderRecompiler recompiler;
++                try {
++                    recompiler.recompile(shader.data.data(), include);
++                } catch (const std::exception& error) {
++                    fprintf(stderr, "Recompile 0x%016llX: %s\n", static_cast<unsigned long long>(hash), error.what());
++                    fprintf(stderr, "Last HLSL: %s\n", recompiler.out.substr(recompiler.out.size() > 1024 ? recompiler.out.size() - 1024 : 0).c_str());
++                    recordFailure("recompiler-exception");
++                    return;
++                }
++                if (std::getenv("XENOS_RECOMP_VERBOSE_RANGE"))
++                    fprintf(stderr, "HLSL ready: %zu bytes\n", recompiler.out.size());
++
 +                if (!recompiler.valid)
 +                {
 +                    recordFailure("invalid-microcode-range");
 +                    return;
 +                }
-+
+ 
                  shader.specConstantsMask = recompiler.specConstantsMask;
  
-                 if (!hlslDumpDir.empty())
-@@ -193,35 +230,69 @@ int main(int argc, char** argv)
+@@ -181,6 +258,8 @@ int main(int argc, char** argv)
+ 
+ #ifdef XENOS_RECOMP_DXIL
+                 shader.dxil = dxcCompiler.compile(recompiler.out, recompiler.isPixelShader, recompiler.specConstantsMask != 0, false);
++                if (std::getenv("XENOS_RECOMP_VERBOSE_RANGE"))
++                    fprintf(stderr, "DXIL compile returned\n");
+                 if (shader.dxil == nullptr)
+                 {
+                     recordFailure("dxc-dxil-compile-failed");
+@@ -193,35 +272,69 @@ int main(int argc, char** argv)
                  }
  #endif
  
@@ -214,7 +324,7 @@ index 754198c..5ee2ca9 100644
          fmt::println("Creating shader cache...");
  
          StringBuffer f;
-@@ -286,6 +357,9 @@ int main(int argc, char** argv)
+@@ -286,6 +399,9 @@ int main(int argc, char** argv)
          f.println("const size_t g_shaderCacheEntryCount = {};", shaders.size());
  
          writeAllBytes(output, f.out.data(), f.out.size());
@@ -224,11 +334,33 @@ index 754198c..5ee2ca9 100644
      }
      else
      {
+diff --git a/XenosRecomp/shader.h b/XenosRecomp/shader.h
+index 67968f6..ab25e58 100644
+--- a/XenosRecomp/shader.h
++++ b/XenosRecomp/shader.h
+@@ -104,4 +104,15 @@ struct ShaderContainer
+     be<uint32_t> shaderOffset;
+     be<uint32_t> field1C;
+     be<uint32_t> field20;
+-};
+\ No newline at end of file
++
++    uint32_t physicalBase() const
++    {
++        const uint32_t virtualBytes = virtualSize;
++        const auto marker = reinterpret_cast<const be<uint32_t>*>(
++            reinterpret_cast<const uint8_t*>(this) + virtualBytes);
++        // Archived LEGO containers repeat physicalSize at the virtual/physical
++        // boundary. Runtime-compiled containers omit that four-byte marker.
++        return virtualBytes + (uint32_t(*marker) == uint32_t(physicalSize)
++            ? sizeof(uint32_t) : 0u);
++    }
++};
 diff --git a/XenosRecomp/shader_common.h b/XenosRecomp/shader_common.h
-index 63c37c9..64e2d6c 100644
+index 63c37c9..ae3b0fe 100644
 --- a/XenosRecomp/shader_common.h
 +++ b/XenosRecomp/shader_common.h
-@@ -44,17 +44,17 @@ struct PushConstants
+@@ -44,17 +44,20 @@ struct PushConstants
  
  #ifdef REBLUE_RECOMP
  // 256-bit boolean register file (BD bool addresses reach ~158), then per-usage 16-bit-pair swap masks.
@@ -254,12 +386,21 @@ index 63c37c9..64e2d6c 100644
 +#define g_SwappedPositions         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 576)
 +#define g_SintTexcoords            vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 580)
 +#define g_ShadowPcfScale           vk::RawBufferLoad<float>(g_PushConstants.SharedConstants + 584)
++#ifdef LEGO_NATIVE_GPU
++#define g_ColorOutputScale        vk::RawBufferLoad<float4>(g_PushConstants.SharedConstants + 608)
++#endif
  #else
  #define g_Booleans                 vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 256)
  #define g_SwappedTexcoords         vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 260)
-@@ -70,17 +70,17 @@ struct PushConstants
+@@ -69,18 +72,24 @@ struct PushConstants
+ #else
  
  #ifdef REBLUE_RECOMP
++#ifdef LEGO_NATIVE_GPU
++#define LEGO_SHARED_CONSTANTS() float4 g_ColorOutputScale : packoffset(c38);
++#else
++#define LEGO_SHARED_CONSTANTS()
++#endif
  #define DEFINE_SHARED_CONSTANTS() \
 -    uint4 g_BooleansArr[2] : packoffset(c16); \
 -    uint g_SwappedTexcoords : packoffset(c18.x); \
@@ -282,11 +423,12 @@ index 63c37c9..64e2d6c 100644
 +    uint g_SwappedBlendWeights : packoffset(c35.w); \
 +    uint g_SwappedPositions : packoffset(c36.x); \
 +    uint g_SintTexcoords : packoffset(c36.y); \
-+    float g_ShadowPcfScale : packoffset(c36.z);
++    float g_ShadowPcfScale : packoffset(c36.z); \
++    LEGO_SHARED_CONSTANTS()
  
  #define g_Booleans(i) (g_BooleansArr[(i) / 4][(i) % 4])
  #else
-@@ -118,6 +118,12 @@ float4 tfetch2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float
+@@ -118,6 +127,12 @@ float4 tfetch2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float
      return texture.Sample(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord + offset / getTexture2DDimensions(texture));
  }
  
@@ -299,7 +441,7 @@ index 63c37c9..64e2d6c 100644
  float2 getWeights2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float2 texCoord, float2 offset)
  {
      Texture2D<float4> texture = g_Texture2DDescriptorHeap[resourceDescriptorIndex];
-@@ -221,6 +227,11 @@ float4 tfetch3D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float
+@@ -221,6 +236,11 @@ float4 tfetch3D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float
      return g_Texture3DDescriptorHeap[resourceDescriptorIndex].Sample(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord);
  }
  
@@ -311,7 +453,7 @@ index 63c37c9..64e2d6c 100644
  struct CubeMapData
  {
  	#ifdef REBLUE_RECOMP
-@@ -239,6 +250,11 @@ float4 tfetchCube(uint resourceDescriptorIndex, uint samplerDescriptorIndex, flo
+@@ -239,6 +259,11 @@ float4 tfetchCube(uint resourceDescriptorIndex, uint samplerDescriptorIndex, flo
      return g_TextureCubeDescriptorHeap[resourceDescriptorIndex].Sample(g_SamplerDescriptorHeap[samplerDescriptorIndex], cubeMapData.cubeMapDirections[texCoord.z]);
  }
  
@@ -324,7 +466,7 @@ index 63c37c9..64e2d6c 100644
  // DEC3N normal decode; IA binds as R32_UINT so lane .x carries the raw bits (asuint recovers them).
  float4 tfetchR11G11B10(float4 value)
 diff --git a/XenosRecomp/shader_recompiler.cpp b/XenosRecomp/shader_recompiler.cpp
-index feaa90b..5e34dc0 100644
+index feaa90b..32e74f7 100644
 --- a/XenosRecomp/shader_recompiler.cpp
 +++ b/XenosRecomp/shader_recompiler.cpp
 @@ -372,7 +372,7 @@ void ShaderRecompiler::recompile(const TextureFetchInstruction& instr, bool bicu
@@ -389,7 +531,25 @@ index feaa90b..5e34dc0 100644
                  }
              }
  
-@@ -1368,11 +1358,11 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -699,6 +689,7 @@ void ShaderRecompiler::recompile(const AluInstruction& instr)
+                 break;           
+             case ExportRegister::PSDepth:
+                 exportRegister = "oDepth";
++                exportsDepth = true;
+                 break;
+             }
+         }
+@@ -727,7 +718,8 @@ void ShaderRecompiler::recompile(const AluInstruction& instr)
+             default:
+             {
+                 auto findResult = interpolators.find(instr.vectorDest);
+-                assert(findResult != interpolators.end());
++                if (findResult == interpolators.end())
++                    throw std::runtime_error(fmt::format("unsupported vertex export register {} (no interpolator; memexport requires writeback)", uint32_t(instr.vectorDest)));
+                 exportRegister = findResult->second;
+                 break;
+             }
+@@ -1368,11 +1360,11 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
              for (size_t j = 0; j < std::size(TEXTURE_DIMENSIONS); j++)
              {
                  println("#define {}_Texture{}DescriptorIndex vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + {})",
@@ -403,7 +563,7 @@ index feaa90b..5e34dc0 100644
  
              samplers.emplace(constantInfo->registerIndex, constantName);
              break;
-@@ -1382,18 +1372,18 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1382,18 +1374,18 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
      }
  
  #ifdef REBLUE_RECOMP
@@ -426,7 +586,7 @@ index feaa90b..5e34dc0 100644
      }
  #endif
  
-@@ -1401,37 +1391,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1401,37 +1393,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
  
      println("cbuffer {}ShaderConstants : register(b{}, space4)", isPixelShader ? "Pixel" : "Vertex", isPixelShader ? 1 : 0);
      out += "{\n";
@@ -465,7 +625,7 @@ index feaa90b..5e34dc0 100644
  
      out += "};\n\n";
  
-@@ -1450,33 +1410,39 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1450,33 +1412,39 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
              for (size_t j = 0; j < std::size(TEXTURE_DIMENSIONS); j++)
              {
                  println("\tuint {}_Texture{}DescriptorIndex : packoffset(c{}.{});",
@@ -510,10 +670,21 @@ index feaa90b..5e34dc0 100644
      out += "#endif\n";
  
      for (uint32_t i = 0; i < constantTableContainer->constantTable.constants; i++)
-@@ -1535,16 +1501,14 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1514,6 +1482,9 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+ 
+     out += "void main(\n";
+ 
++    size_t depthSignatureOffset = 0;
++    size_t depthInitializerOffset = 0;
++
+     if (isPixelShader)
+     {
+         out += "\tin float4 iPos : SV_Position,\n";
+@@ -1534,17 +1505,15 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+         out += "\tin uint iFace : SV_IsFrontFace\n";
          out += "#endif\n";
  
-         auto pixelShader = reinterpret_cast<const PixelShader*>(shader);
+-        auto pixelShader = reinterpret_cast<const PixelShader*>(shader);
 -        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_COLOR0)
 -            out += ",\n\tout float4 oC0 : SV_Target0";
 -        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_COLOR1)
@@ -525,17 +696,18 @@ index feaa90b..5e34dc0 100644
 -        if (pixelShader->outputs & PIXEL_SHADER_OUTPUT_DEPTH)
 -            out += ",\n\tout float oDepth : SV_Depth";
 +        // Some LEGO containers under-report MRT outputs in reflection while
-+        // the microcode still exports them. A superset signature is harmless
-+        // in D3D12 and prevents valid shaders from failing AOT compilation.
++        // the microcode still exports them. Keep the color-output superset,
++        // but add SV_Depth only after decoding an actual depth export. Merely
++        // declaring SV_Depth overrides raster depth, even with no guest export.
 +        out += ",\n\tout float4 oC0 : SV_Target0";
 +        out += ",\n\tout float4 oC1 : SV_Target1";
 +        out += ",\n\tout float4 oC2 : SV_Target2";
 +        out += ",\n\tout float4 oC3 : SV_Target3";
-+        out += ",\n\tout float oDepth : SV_Depth";
++        depthSignatureOffset = out.size();
      }
      else
      {
-@@ -1606,6 +1570,21 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1606,6 +1575,21 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
      out += ")\n";
      out += "{\n";
  
@@ -551,23 +723,23 @@ index feaa90b..5e34dc0 100644
 +        out += "\toC1 = 0.0;\n";
 +        out += "\toC2 = 0.0;\n";
 +        out += "\toC3 = 0.0;\n";
-+        out += "\toDepth = 0.0;\n";
++        depthInitializerOffset = out.size();
 +    }
 +
  #ifdef UNLEASHED_RECOMP
      if (hasMtxProjection)
      {
-@@ -1628,7 +1607,8 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1628,7 +1612,8 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
          while (*definitions != 0)
          {
              auto definition = reinterpret_cast<const Float4Definition*>(definitions);
 -            auto value = reinterpret_cast<const be<uint32_t>*>(shaderData + shaderContainer->virtualSize + definition->physicalOffset);
-+            const uint32_t physicalBase = uint32_t(shaderContainer->virtualSize) + sizeof(uint32_t);
++            const uint32_t physicalBase = shaderContainer->physicalBase();
 +            auto value = reinterpret_cast<const be<uint32_t>*>(shaderData + physicalBase + definition->physicalOffset);
              for (uint16_t i = 0; i < (definition->count + 3) / 4; i++)
              {
  #ifdef REBLUE_RECOMP
-@@ -1659,11 +1639,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1659,11 +1644,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
                      }
                      return fmt::format("asfloat(0x{:X}u)", u);
                  };
@@ -581,7 +753,7 @@ index feaa90b..5e34dc0 100644
                  println("\tfloat4 c{} = asfloat(uint4(0x{:X}, 0x{:X}, 0x{:X}, 0x{:X}));",
                      definition->registerIndex + i - (isPixelShader ? 256 : 0), value[0].get(), value[1].get(), value[2].get(), value[3].get());
  #endif
-@@ -1692,7 +1674,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1692,7 +1679,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
  
                  value = definition->values[i].get();
  
@@ -590,7 +762,7 @@ index feaa90b..5e34dc0 100644
                      (definition->registerIndex - 8992) / 4 + i, x, y, z, w);
              }
              definitions += 2;
-@@ -1702,7 +1684,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1702,7 +1689,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
          out += "\n";
      }
  
@@ -599,7 +771,7 @@ index feaa90b..5e34dc0 100644
  
      uint32_t interpolatorCount = (shader->interpolatorInfo >> 5) & 0x1F;
  
-@@ -1744,7 +1726,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1744,7 +1731,7 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
          out += "\n";
      }
  
@@ -608,17 +780,17 @@ index feaa90b..5e34dc0 100644
      {
          if (!printedRegisters[i])
          {
-@@ -1782,7 +1764,8 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1782,7 +1769,8 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
  #endif
      }
  
 -    const be<uint32_t>* code = reinterpret_cast<const be<uint32_t>*>(shaderData + shaderContainer->virtualSize + shader->physicalOffset);
-+    const uint32_t physicalBase = uint32_t(shaderContainer->virtualSize) + sizeof(uint32_t);
++    const uint32_t physicalBase = shaderContainer->physicalBase();
 +    const be<uint32_t>* code = reinterpret_cast<const be<uint32_t>*>(shaderData + physicalBase + shader->physicalOffset);
  
      union
      {
-@@ -1798,7 +1781,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -1798,7 +1786,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
  
      auto controlFlowCode = code;
      uint32_t instrAddress = 0;
@@ -633,7 +805,7 @@ index feaa90b..5e34dc0 100644
      bool simpleControlFlow = true;
  
      // Per-instruction control flow summary for the structurizer below.
-@@ -2209,6 +2198,16 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+@@ -2209,6 +2203,16 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
              }
              }
  
@@ -650,16 +822,45 @@ index feaa90b..5e34dc0 100644
              auto instructionCode = code + address * 3;
              
              for (uint32_t i = 0; i < count; i++)
+@@ -2315,6 +2319,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+                     indent();
+                     out += '}';
+                 #endif
++                    // Native EDRAM format/exponent is bound dynamically per RT.
++                    // Alpha testing above uses the original shader alpha.
++                    out += "\n#ifdef LEGO_NATIVE_GPU\n";
++                    out += "\toC0 *= g_ColorOutputScale.x;\n";
++                    out += "\toC1 *= g_ColorOutputScale.y;\n";
++                    out += "\toC2 *= g_ColorOutputScale.z;\n";
++                    out += "\toC3 *= g_ColorOutputScale.w;\n#endif\n";
+                 }
+                 else
+                 {
+@@ -2390,4 +2401,13 @@ void ShaderRecompiler::recompile(const uint8_t* shaderData, const std::string_vi
+ #endif
+ 
+     out += "}";
++
++    if (isPixelShader && exportsDepth)
++    {
++        // Insert from back to front so offsets remain valid. Keep the existing
++        // deterministic initialization for real depth-export shaders only;
++        // ordinary shaders must retain fixed-function raster depth instead.
++        out.insert(depthInitializerOffset, "\toDepth = 0.0;\n");
++        out.insert(depthSignatureOffset, ",\n\tout float oDepth : SV_Depth");
++    }
+ }
 diff --git a/XenosRecomp/shader_recompiler.h b/XenosRecomp/shader_recompiler.h
-index 791def7..c7336ad 100644
+index 791def7..b307770 100644
 --- a/XenosRecomp/shader_recompiler.h
 +++ b/XenosRecomp/shader_recompiler.h
-@@ -24,11 +24,13 @@ struct StringBuffer
+@@ -24,11 +24,14 @@ struct StringBuffer
  struct ShaderRecompiler : StringBuffer
  {
      uint32_t indentation = 0;
 +    bool valid = true;
      bool isPixelShader = false;
++    bool exportsDepth = false;
      const uint8_t* constantTableData = nullptr;
      std::unordered_map<uint32_t, VertexElement> vertexElements;
      std::unordered_map<uint32_t, std::string> interpolators;
@@ -668,4 +869,12 @@ index 791def7..c7336ad 100644
      std::unordered_map<uint32_t, const char*> boolConstants;
      std::unordered_map<uint32_t, const char*> samplers;
      std::unordered_map<uint32_t, uint32_t> ifEndLabels;
-
+@@ -50,6 +53,8 @@ struct ShaderRecompiler : StringBuffer
+ 
+     void indent()
+     {
++        if (indentation > 256)
++            throw std::runtime_error("unbalanced shader control-flow indentation");
+         for (uint32_t i = 0; i < indentation; i++)
+             out += '\t';
+     }
diff --git a/tools/gpu-aot/prepare_alpha_candidate.py b/tools/gpu-aot/prepare_alpha_candidate.py
new file mode 100644
index 0000000..5974d6c
--- /dev/null
+++ b/tools/gpu-aot/prepare_alpha_candidate.py
@@ -0,0 +1,84 @@
+"""Build inputs for an isolated 624-byte, eight-compare native alpha compiler.
+
+Does not install shader archives or touch the explicit-LOD 752-byte candidate.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('scratch', type=Path)
+a = p.parse_args()
+root = Path(__file__).resolve().parents[2]
+checkout = root / 'research/reblue/thirdparty/XenosRecomp'
+module = checkout / 'XenosRecomp'
+source = a.scratch.resolve() / 'source'
+source.mkdir(parents=True, exist_ok=True)
+manifest = {}
+for name in ('constant_table.h', 'dxc_compiler.cpp', 'dxc_compiler.h', 'main.cpp',
+             'pch.h', 'shader.h', 'shader_code.h', 'shader_recompiler.cpp',
+             'shader_recompiler.h', 'shader_common.h'):
+    path = module / name
+    manifest[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
+    shutil.copyfile(path, source / name)
+
+def replace_once(s, old, new):
+    assert s.count(old) == 1, old
+    return s.replace(old, new)
+
+s = (source / 'shader_recompiler.cpp').read_text()
+s = replace_once(s, 'exportRegister = "oC0";',
+    'exportRegister = "oC0";\n                exportsColor0 = true;')
+s = replace_once(s, 'out += "[branch] if (g_SpecConstants() & SPEC_CONSTANT_ALPHA_TEST)";',
+    'out += "\\n#if !defined(LEGO_NATIVE_GPU) || LEGO_SHADER_WRITES_COLOR0\\n";\n'
+    '                    out += "[branch] if (g_SpecConstants() & SPEC_CONSTANT_ALPHA_TEST)";')
+s = replace_once(s, 'out += "\\n#ifdef LEGO_NATIVE_GPU\\n";',
+    'out += "\\n#endif\\n#ifdef LEGO_NATIVE_GPU\\n";')
+# The export flag is known only after the complete control flow has been
+# decoded. Gate all return sites, including those preceding later exports.
+pos = s.rfind('\n}')
+assert pos > 0
+s = s[:pos] + ('\n    out.insert(0, exportsColor0 ? "#define LEGO_SHADER_WRITES_COLOR0 1\\n" : '
+               '"#define LEGO_SHADER_WRITES_COLOR0 0\\n");\n') + s[pos:]
+s = replace_once(s, 'out += "\\tclip(oC0.w - g_AlphaThreshold);\\n";',
+    'out += "\\n#ifdef LEGO_NATIVE_GPU\\n";\n'
+    '                    out += "\\tif (!LegoAlphaPass(oC0.w, g_AlphaThreshold, g_AlphaFunction)) discard;\\n";\n'
+    '                    out += "#else\\n\\tclip(oC0.w - g_AlphaThreshold);\\n#endif\\n";')
+(source / 'shader_recompiler.cpp').write_text(s)
+s = (source / 'shader_recompiler.h').read_text()
+s = replace_once(s, '    bool exportsDepth = false;',
+    '    bool exportsDepth = false;\n    bool exportsColor0 = false;')
+(source / 'shader_recompiler.h').write_text(s)
+s = (source / 'shader_common.h').read_text()
+s = replace_once(s,
+    '#define g_ColorOutputScale        vk::RawBufferLoad<float4>(g_PushConstants.SharedConstants + 608)',
+    '#define g_ColorOutputScale        vk::RawBufferLoad<float4>(g_PushConstants.SharedConstants + 608)\n'
+    '#define g_AlphaFunction           vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 600)')
+s = replace_once(s,
+    '#define LEGO_SHARED_CONSTANTS() float4 g_ColorOutputScale : packoffset(c38);',
+    '#define LEGO_SHARED_CONSTANTS() uint g_AlphaFunction : packoffset(c37.z); \\\n'
+    '    float4 g_ColorOutputScale : packoffset(c38);')
+s = replace_once(s, 'Texture2D<float4> g_Texture2DDescriptorHeap[]',
+    '#ifdef LEGO_NATIVE_GPU\n' + (root / 'tools/gpu-aot/alpha_compare.hlsli').read_text() +
+    '\n#endif\n\nTexture2D<float4> g_Texture2DDescriptorHeap[]')
+(source / 'shader_common.h').write_text(s)
+dep = checkout / 'thirdparty'
+old_build = checkout / 'out/build/lego-release'
+cmake = '''cmake_minimum_required(VERSION 3.25)
+project(LegoAlphaCandidate LANGUAGES CXX)
+add_executable(alpha_compiler main.cpp dxc_compiler.cpp shader_recompiler.cpp "@SMOL@/smolv.cpp")
+target_compile_features(alpha_compiler PRIVATE cxx_std_17)
+target_compile_definitions(alpha_compiler PRIVATE REBLUE_RECOMP XENOS_RECOMP_DXIL FMT_SHARED _CRT_SECURE_NO_WARNINGS NOMINMAX)
+target_compile_options(alpha_compiler PRIVATE -fms-extensions -D_DLL -D_MT -Xclang --dependent-lib=msvcrt)
+target_include_directories(alpha_compiler PRIVATE "@SMOL@" "@DEP@/xxHash" "@DEP@/zstd/lib" "@DEP@/fmt/include" "@DEP@/dxc-bin/inc")
+target_precompile_headers(alpha_compiler PRIVATE pch.h)
+target_link_libraries(alpha_compiler PRIVATE "@DEP@/dxc-bin/lib/x64/dxcompiler.lib" "@BUILD@/thirdparty/xxHash/cmake_unofficial/xxhash.lib" "@BUILD@/thirdparty/zstd/build/cmake/lib/zstd_static.lib" "@BUILD@/thirdparty/fmt/fmt.lib")
+add_custom_command(TARGET alpha_compiler POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "@DEP@/dxc-bin/bin/x64/dxcompiler.dll" "@DEP@/dxc-bin/bin/x64/dxil.dll" "@BUILD@/thirdparty/fmt/bin/fmt.dll" $<TARGET_FILE_DIR:alpha_compiler>)
+'''
+cmake = cmake.replace('@SMOL@', (dep / 'smol-v/source').as_posix())
+cmake = cmake.replace('@DEP@', dep.as_posix()).replace('@BUILD@', old_build.as_posix())
+(source / 'CMakeLists.txt').write_text(cmake)
+(a.scratch / 'original-source-hashes.json').write_text(json.dumps(manifest, indent=2))
+print(source)
diff --git a/tools/gpu-aot/prepare_lod_candidate.py b/tools/gpu-aot/prepare_lod_candidate.py
new file mode 100644
index 0000000..f030531
--- /dev/null
+++ b/tools/gpu-aot/prepare_lod_candidate.py
@@ -0,0 +1,156 @@
+"""Prepare an isolated explicit-LOD compiler and ABI candidate, not a live fix.
+
+Copies only the XenosRecomp module sources; dependencies use the existing local
+build. All outputs stay under scratch. Does not change the compiler checkout,
+native source, runtime shader bank, config, or game executable.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+
+
+def replace_once(source, old, new):
+    if source.count(old) != 1:
+        raise ValueError(f'Expected one source anchor: {old[:100]!r}')
+    return source.replace(old, new)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('scratch', type=Path)
+    args = parser.parse_args()
+    root = Path(__file__).resolve().parents[2]
+    checkout = root / 'research/reblue/thirdparty/XenosRecomp'
+    module = checkout / 'XenosRecomp'
+    source_dir = args.scratch.resolve() / 'source'
+    source_dir.mkdir(parents=True, exist_ok=True)
+    manifest = {}
+    for name in ('constant_table.h', 'dxc_compiler.cpp', 'dxc_compiler.h',
+                 'main.cpp', 'pch.h', 'shader.h', 'shader_code.h',
+                 'shader_recompiler.cpp', 'shader_recompiler.h', 'shader_common.h'):
+        path = module / name
+        manifest[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
+        shutil.copyfile(path, source_dir / name)
+    compiler = (source_dir / 'shader_recompiler.cpp').read_text()
+    compiler = replace_once(compiler,
+        'if (instr.opcode != FetchOpcode::TextureFetch && instr.opcode != FetchOpcode::GetTextureWeights)\n        return;',
+        'if (instr.opcode != FetchOpcode::TextureFetch && instr.opcode != FetchOpcode::GetTextureWeights &&\n'
+        '        instr.opcode != FetchOpcode::SetTextureLod)\n        return;')
+    # The same register LOD state is shared by all texture units. It is updated
+    # under instruction predication, not under the predicate of the later fetch.
+    compiler = replace_once(compiler, '    std::string constName;\n', '''    if (instr.opcode == FetchOpcode::SetTextureLod)
+    {
+        if (instr.srcRegisterAm)
+            throw std::runtime_error("Explicit LOD candidate: relative setTexLOD unsupported");
+        indent();
+        out += "textureLod = ";
+        printSrcRegister(1);
+        out += ";\\n";
+        if (instr.isPredicated)
+        {
+            --indentation;
+            indent();
+            out += "}\\n";
+        }
+        return;
+    }
+    const bool computedLod = instr.useCompLod && (isPixelShader || instr.useRegGradients);
+    const bool explicitLod = instr.opcode == FetchOpcode::TextureFetch && !computedLod;
+
+    std::string constName;
+''')
+    compiler = replace_once(compiler,
+        '        out += isPixelShader ? "tfetch" : "tfetchVertex";',
+        '        out += explicitLod ? "tfetchLevel" : (isPixelShader ? "tfetch" : "tfetchVertex");')
+    compiler = replace_once(compiler, '    out += ").";\n', '''    if (explicitLod)
+    {
+        // Instruction bias is signed fixed point with four fractional bits.
+        print(", {} + g_FetchLodBias({}) + {}", instr.useRegLod ? "textureLod" : "0.0",
+            uint32_t(instr.constIndex), float(instr.lodBias) / 16.0f);
+    }
+    out += ").";
+''')
+    compiler = replace_once(compiler,
+        '    // Xenos exposes 64 temporary vector registers and 16 integer constants.\n',
+        '    out += "\\tfloat textureLod = 0.0;\\n";\n\n'
+        '    // Xenos exposes 64 temporary vector registers and 16 integer constants.\n')
+    (source_dir / 'shader_recompiler.cpp').write_text(compiler)
+
+    common = (source_dir / 'shader_common.h').read_text()
+    common = replace_once(common,
+        '#define g_ColorOutputScale        vk::RawBufferLoad<float4>(g_PushConstants.SharedConstants + 608)',
+        '#define g_ColorOutputScale        vk::RawBufferLoad<float4>(g_PushConstants.SharedConstants + 608)\n'
+        '#define g_FetchLodBias(i)          vk::RawBufferLoad<float>(g_PushConstants.SharedConstants + 624 + (i)*4)')
+    common = replace_once(common,
+        '#define LEGO_SHARED_CONSTANTS() float4 g_ColorOutputScale : packoffset(c38);',
+        '#define LEGO_SHARED_CONSTANTS() float4 g_ColorOutputScale : packoffset(c38); \\\n'
+        '    float4 g_FetchLodBiasArr[8] : packoffset(c39);\n'
+        '#define g_FetchLodBias(i) (g_FetchLodBiasArr[(i) / 4][(i) % 4])')
+    common = replace_once(common, 'float4 tfetchVertex2D(', '''float4 tfetchLevel2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float2 texCoord, float2 offset, float lod)
+{
+    Texture2D<float4> texture = g_Texture2DDescriptorHeap[resourceDescriptorIndex];
+    return texture.SampleLevel(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord + offset / getTexture2DDimensions(texture), lod);
+}
+
+float4 tfetchVertex2D(''')
+    common = replace_once(common, 'float4 tfetchVertex3D(', '''float4 tfetchLevel3D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float3 texCoord, float lod)
+{
+    return g_Texture3DDescriptorHeap[resourceDescriptorIndex].SampleLevel(g_SamplerDescriptorHeap[samplerDescriptorIndex], texCoord, lod);
+}
+
+float4 tfetchVertex3D(''')
+    common = replace_once(common, 'float4 tfetchVertexCube(', '''float4 tfetchLevelCube(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float3 texCoord, inout CubeMapData cubeMapData, float lod)
+{
+    return g_TextureCubeDescriptorHeap[resourceDescriptorIndex].SampleLevel(g_SamplerDescriptorHeap[samplerDescriptorIndex], cubeMapData.cubeMapDirections[texCoord.z], lod);
+}
+
+float4 tfetchVertexCube(''')
+    (source_dir / 'shader_common.h').write_text(common)
+
+    # This ABI change must accompany the candidate shaders at eventual runtime.
+    # Keep it as an isolated source copy until the sampler-only A/B is observed.
+    draw = (root / 'rexlego/src/gpu_native/draw.cpp').read_text()
+    draw = replace_once(draw, '  float color_output_scale[4]{1, 1, 1, 1};\n',
+                        '  float color_output_scale[4]{1, 1, 1, 1};\n  float fetch_lod_bias[32]{};\n')
+    draw = replace_once(draw, 'static_assert(sizeof(SharedConstants) == 624);',
+                        'static_assert(sizeof(SharedConstants) == 752);\n'
+                        'static_assert(offsetof(SharedConstants, fetch_lod_bias) == 624);')
+    draw = replace_once(draw, '    shared.samplers[i] = HostDevice::RegisterSampler(fetch);',
+                        '    shared.samplers[i] = HostDevice::RegisterSampler(fetch);\n'
+                        '    const int32_t lod_bias = int32_t((fetch[4] >> 12) & 1023);\n'
+                        '    shared.fetch_lod_bias[i] = float(lod_bias >= 512 ? lod_bias - 1024 : lod_bias) / 32.0f;')
+    (source_dir / 'draw-candidate.cpp').write_text(draw)
+
+    dep = checkout / 'thirdparty'
+    old_build = checkout / 'out/build/lego-release'
+    cmake = '''cmake_minimum_required(VERSION 3.25)
+project(LegoExplicitLodCandidate LANGUAGES CXX)
+add_executable(lod_compiler main.cpp dxc_compiler.cpp shader_recompiler.cpp "@SMOL@/smolv.cpp")
+target_compile_features(lod_compiler PRIVATE cxx_std_17)
+target_compile_definitions(lod_compiler PRIVATE REBLUE_RECOMP XENOS_RECOMP_DXIL FMT_SHARED _CRT_SECURE_NO_WARNINGS NOMINMAX)
+target_compile_options(lod_compiler PRIVATE -Wno-switch -Wno-unused-variable -Wno-null-arithmetic -fms-extensions -D_DLL -D_MT -Xclang --dependent-lib=msvcrt)
+target_include_directories(lod_compiler PRIVATE "@SMOL@" "@DEP@/xxHash" "@DEP@/zstd/lib" "@DEP@/fmt/include" "@DEP@/dxc-bin/inc")
+target_precompile_headers(lod_compiler PRIVATE pch.h)
+target_link_libraries(lod_compiler PRIVATE "@DEP@/dxc-bin/lib/x64/dxcompiler.lib" "@BUILD@/thirdparty/xxHash/cmake_unofficial/xxhash.lib" "@BUILD@/thirdparty/zstd/build/cmake/lib/zstd_static.lib" "@BUILD@/thirdparty/fmt/fmt.lib")
+add_custom_command(TARGET lod_compiler POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "@DEP@/dxc-bin/bin/x64/dxcompiler.dll" "@DEP@/dxc-bin/bin/x64/dxil.dll" "@BUILD@/thirdparty/fmt/bin/fmt.dll" $<TARGET_FILE_DIR:lod_compiler>)
+'''
+    cmake = cmake.replace('@SMOL@', (dep / 'smol-v/source').as_posix())
+    cmake = cmake.replace('@DEP@', dep.as_posix()).replace('@BUILD@', old_build.as_posix())
+    (source_dir / 'CMakeLists.txt').write_text(cmake)
+    (source_dir / 'README.txt').write_text('''ISOLATED CANDIDATE — NOT APPLIED TO NATIVE.
+Compile with XENOS_RECOMP_LEGO_NATIVE_SCALE=1. New HLSL requires an extra
+32 float fetch biases at SharedConstants offset624, total752 bytes.
+draw-candidate.cpp shows the matching upload change. Do not replace the live
+shader cache without this ABI update. Current sampler-only executable and
+archives remain unchanged. Supports explicit LOD on currently supported
+2D/3D/cube helpers; relative setTexLOD, explicit gradients, instruction filter
+overrides, texcoord-denorm, and 1D helper coverage remain separate limitations.
+''')
+    (args.scratch / 'original-source-hashes.json').write_text(json.dumps(manifest, indent=2))
+    print(f'Isolated LOD candidate prepared: {source_dir}')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/gpu-aot/prepare_viewport_candidate.py b/tools/gpu-aot/prepare_viewport_candidate.py
new file mode 100644
index 0000000..501811b
--- /dev/null
+++ b/tools/gpu-aot/prepare_viewport_candidate.py
@@ -0,0 +1,53 @@
+"""Prepare a separate ABI624 compiler for TU23's 0x43F/0x400 viewport modes.
+
+Requires the already tested alpha candidate; does not install shader banks.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('alpha_source', type=Path)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+source = a.output.resolve() / 'source'
+source.mkdir(parents=True, exist_ok=True)
+hashes = {}
+for f in a.alpha_source.iterdir():
+    if f.is_file():
+        hashes[f.name] = hashlib.sha256(f.read_bytes()).hexdigest()
+        shutil.copyfile(f, source/f.name)
+
+common = source/'shader_common.h'
+s = common.read_text()
+old = '#define LEGO_SHARED_CONSTANTS() uint g_AlphaFunction : packoffset(c37.z);'
+assert s.count(old) == 1
+s = s.replace(old, '#define LEGO_SHARED_CONSTANTS() float2 g_ViewportScale : packoffset(c37); \\\n    uint g_ViewportMode : packoffset(c37.w); \\\n    uint g_AlphaFunction : packoffset(c37.z);')
+old = '#define g_AlphaFunction           vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 600)'
+assert s.count(old) == 1
+s = s.replace(old, old+'\n#define g_ViewportScale           vk::RawBufferLoad<float2>(g_PushConstants.SharedConstants + 592)\n#define g_ViewportMode            vk::RawBufferLoad<uint>(g_PushConstants.SharedConstants + 604)')
+old = 'Texture2D<float4> g_Texture2DDescriptorHeap[]'
+assert s.count(old) == 1
+s = s.replace(old, '''// TU23 83FB8FD0 writes 0x43F (viewport on) or 0x400 (pixel space).
+// Convert homogeneous pixel coordinates to host NDC before the host viewport.
+#ifdef LEGO_NATIVE_GPU
+float4 LegoViewportPosition(float4 position, float2 scale, uint mode)
+{
+    if (mode == 1)
+        position.xy = position.xy * scale + float2(-1, 1) * position.w;
+    return position;
+}
+#endif
+
+'''+old)
+common.write_text(s)
+compiler = source/'shader_recompiler.cpp'
+s = compiler.read_text()
+old = 'out += "\\toPos.xy += g_HalfPixelOffset * oPos.w;\\n";'
+assert s.count(old) == 2
+s = s.replace(old, 'out += "\\n#ifdef LEGO_NATIVE_GPU\\n\\toPos = LegoViewportPosition(oPos, g_ViewportScale, g_ViewportMode);\\n#endif\\n";\n                        '+old)
+compiler.write_text(s)
+(a.output/'input-source-hashes.json').write_text(json.dumps(hashes, indent=2)+'\n')
+print(source)
diff --git a/tools/gpu-aot/questions-pending-20260930.md b/tools/gpu-aot/questions-pending-20260930.md
new file mode 100644
index 0000000..9b0f358
--- /dev/null
+++ b/tools/gpu-aot/questions-pending-20260930.md
@@ -0,0 +1,264 @@
+# Вопросы после возвращения пользователя
+
+Пользователь вернулся 30 сентября. Ссылка и приоритетный вопрос переданы ему
+через форму для пересылки агенту. Прямого сообщения другому агенту не было;
+ответ получен в tiling-answer-20260930.md, Addendum 14. Ниже сохранены
+исходные вопросы; продолжение для следующего разбора — в конце файла.
+
+1. TU23 BlockUntilNotBusy83FBF900 сейчас в native вызывает полный drain всех
+   host frame fences. Подготовлен opt-in LEGO_NATIVE_ASYNC_CPU_RESOURCES:
+   известные native VB/IB используют неизменяемые host UPLOAD snapshots;
+   известные AOT shaders читаются из встроенного архива; borrowed CPU textures
+   (!surface, !resolved_on_host) копируются из guest в retained UPLOAD storage,
+   затем в host texture строго в порядке одной graphics queue. Для них GPU
+   никогда не читает саму guest allocation. Неизвестные ресурсы, surfaces,
+   GPU-resolved textures, BlockOnFence и InsertCallback сохраняют полный wait.
+   Проверь вызывающих83FBF900: нет ли у этих CPU ресурсов дополнительного
+   CPU-worker readiness контракта, который был случайно обеспечен GPU drain?
+   Нужны адреса вызывающих и смысл guest fence полей, особенно82BD6088,
+   82CB9120/82CB9070 phase0. Первый runtime probe почти не пропускал wait:
+   в конце 48 drains/кадр и 29.89мс, skips=0; стабильные30 не достигнуты.
+
+2. После исправления borrowed GetResourceType пользователь выдержал5минут
+   на Press START без прежней порчи (Addendum13 цепочка подтверждена логом),
+   но в катсцене у Vortech по-прежнему исчезают руки. Банк118, все VS slots16/17
+   формат38 1024x128 теперь full-hash при каждом draw, buffer contents также
+   full-hash, metadata заголовков обновляется. Какая ветвь анимации/декларация
+   отвечает за руки, и может ли часть геометрии использовать другие skinning
+   textures/streams? Нужен список писателей/commit boundaries для сопоставления
+   с runtime capture, не новая гипотеза 'все textures stale'.
+
+3. В главном меню отсутствует содержимое портала и похожие эффекты; при Enter
+   цвета кратко бледно-коричневые, затем нормализуются. Тонемап3A47E5DDE66B42C6
+   после резкой HDR/half картинки сильно блюрит final. Нужны конкретные
+   shader/pass/constant inputs для portal и blur/DoF, и роль PS c12/slot3;
+   какие ожидаемые значения/LOD/alpha controls сравнить с эталонной Xenia?
+   Существующий isolated explicit-LOD752 candidate не установлен, runtime ABI624.
+
+4. Приоритет FPS: свежая разбивка fps-sync-reasons1 показывает в тяжёлом
+   30-секундном участке 41.89 drains/кадр, 28.87мс суммарно. Из них
+   InsertCallback83FC0F38: 39.28/кадр и 28.74мс; остальные причины вместе
+   около0.13мс. Native выполняет callback синхронно после полного drain.
+   В generated callers82BCE2C8/82BCE304 передают callback82BCDE68, а
+   82BCE4F8/82BCE534 —82BCDF30, flags=1. Это статически найденные кандидаты,
+   принадлежность всех runtime waits именно им ещё не подтверждена.
+   Проверь обязанности этих callbacks и83FBD2D8/83EB711C: записывают ли они
+   только bookkeeping/refcounts/releases CPU-snapshot ресурсов или читают
+   GPU-produced данные? Какие guest wait/owner зависимости требуют точного
+   завершения? Нужен контракт безопасного отложенного исполнения по host
+   fence либо раннего CPU-only исполнения для конкретных адресов, включая
+   последствия для pool relocation/анимации. Просто убирать все callbacks
+   или объявлять их completed без доказательства не планируем.
+
+Контекст: tools/gpu-aot/tiling-answer-20260930.md Addenda11–13;
+session-20260930/placement-type1-game.log, placement-type1-title-start.png,
+placement-type1-title-3min.png и placement-type1-title-5min.png
+(последний уже показывает главное меню после пользовательского Enter).
+FPS доказательства: session-20260930/fps-sync-reasons1-game.log,
+fps-sync-reasons1-frames.csv(.summary.json), fps-sync-reasons1-cutscene.png.
+
+## Следующие уточнения после Addendum 14
+
+Подготовлены во время отсутствия пользователя. Пункты1–2 переданы ему через
+форму для пересылки после возврата с обратной связью о новой сборке; ответ
+получен в Addendum15. Пункт3 ещё не передан. Пользователь подтвердил более
+стабильную катсцену/меню, но руки и vortex всё ещё отсутствуют.
+
+**Коррекция после ответа:** +10568 — cull/fill, НЕ alpha func/ref.
+Правильные alpha setters:83FB82C0 (+10556 low3),83FB7E00 (+10556 bit3),
+83FB92B8 (raw FLOAT +10620). Пункт1 ниже оставлен как история отправленного
+вопроса, не как достоверная карта alpha. Tonemap смешивает sharp slot0 с
+downsampled slot4; slot3 — добавочный glow. См. Addendum15.
+
+1. Alpha test: compiler уже генерирует `clip(oC0.w-g_AlphaThreshold)` перед
+   EDRAM scale, spec bit1; host оставляет threshold=0 и никогда не включает
+   этот bit (только normal bit0). Поэтому ветвь существует, но не применяется.
+   В TU23 нашёл setter83FB7E00: enable в device+10556 bit3;
+   setter83FB7DA0/getter83FB7DC0: function в device+10568 bits0..2;
+   setter83FB7DD0/getter83FB7DF0: reference в +10568 bits3..10 (8 бит).
+   Подтверди связь с D3DRS_ALPHATESTENABLE/ALPHAFUNC/ALPHAREF, перевод compare
+   (сырое значение D3DCMP или Xenos enum), нормализацию reference и порядок
+   относительно shader output/EDRAM exp. Почему ref упакован в +10568, а не
+   отдельный float RB_ALPHA_REF? Нужен путь, который превращает этот shadow
+   в настоящие hardware registers. Unleashed использует value/256 и только
+   >= clip, но переносить его приближение без проверки TU23 не хочу.
+   Нужно также выяснить defaults этих трёх состояний: native CreateDevice
+   инициализирует sampler defaults, но не вызывает render-state init.
+   Это найденный пробел, не доказательство причины невидимого портала/HUD.
+
+2. Для следующего разбора blur: baseline уже пишет `pixel shader filter
+   matches 1 AOT containers` (fps-sync-reasons1-game.log 19:33:27.041).
+   Значит, одно отключение CFEAC7ADB912F8A9 уже применяется, но размытие
+   осталось. Текущий runtime ABI624, isolated explicit-LOD candidate752
+   не установлен. Нужен разбор tonemap3A47E5DDE66B42C6: как именно c12 и
+   slot3 влияют на выбор/смешивание sharp и half/blur; какие значения в
+   микрокоде дают чистый sharp output? Не просим объявлять DoF причиной
+   пропавших рук. При необходимости предоставим disassembly контейнера.
+
+3. Callback83EB4480 invokes vtable+92. Новая очередь сохраняет completion
+   по fence всего содержащего submission и FIFO, без drain в InsertCallback.
+   Уточни обязанности конкретных virtual23 в helper83EB6D68: только release
+   CPU snapshot/renderer bookkeeping или game ждёт side effects до следующего
+   draw/кадра? Нужны конкретные vtable/method addresses и lifetime объекта,
+   чтобы проверить допустимость исполнения на guest hook thread после
+   containing submission (вместо точного GPU marker).
+
+## Обновление после офлайн-проверки alpha/defaults (ещё не отправлено)
+
+Для следующего ответа агенту: исправление к Addendum15 — таблица TU23
+0x847F9B18, descriptor59 содержит getter83FB92E0/setter83FB92B8/default3F800000.
+Это float1.0, не0.0. Native теперь применяет15 проверенных leaf setters для
+alpha/blend/масок цвета; результат проверен на оригинальных generated functions.
+Все8 alpha compares реализованы до EDRAM scale; depth-only PS исключены.
+9290 основных и118 runtime контейнеров пересобраны, ABI624 сохранён.
+
+Гипотеза embedded fetch patches для рук не подтвердилась на captured corpus:
+у всех118 контейнеров нет copy/masked records. Отдельно не проверено, меняет ли
+игра эту таблицу позднее в уже созданном shader object. Native SetShader пока
+эти таблицы не применяет. Нужен конкретный писатель, если такие динамические
+изменения существуют, а не рекомендация снова расширить банк.
+
+Остаётся ранее подготовленный вопрос3 о UI/video callback83EB4480 и
+virtual23; добавочный контекст — TT Games logo отсутствует перед катсценой.
+Нужно отделить путь видео/загрузки от geometry draws и назвать конкретные
+методы/ресурсы. QuickStartup сокращает legal/loading, но TT-logo причинно
+не подтверждён. Пока игру по просьбе пользователя не запускаем.
+
+## Oct1 follow-up after Addenda16/17 — runtime evidence
+
+User authorized today's tests, monitor3 only. All probes now closed. We read
+both addenda and applied the DS rebind enable-bit tail missing from our native
+SetDepthStencilSurface hook. 300 cases match the original TU23 PPC operations
+(including requested bits surviving NULL then non-NULL binds). Stencil defaults
+were already seeded using actual TU23 descriptor values, masks00FFFF00; this
+was not a new fix. Candidate remains opt-in. No visual repair claimed.
+
+Bounded target-mesh tracing now records before native rejection and after draw
+submission, raw controls, c48..54/c73/c92..95/c247, PS c4/c45, stream metadata,
+first actual index+base and converted host attribute bytes. No render overrides.
+Artifacts: rexlego/out/native-gpu/session-20261001/*-summary.json; complete logs
+in session-20260930/oct01-mesh1/2/3-game.log and oct01-rebind1-game.log.
+
+mesh1: 43 distinct samples, ALL submitted. 0406577EF00099BD/7CA67FB21C799FE4
+and35DB03916F21B74F/A71CC74B4EE251E5 each12 samples. Other no-fog pairs and
+C3958E2D1B795ED9/0C1840BF35E84F2F never observed (trace is before rejection,
+not the old capped stage log). Hash matches still do not identify an asset.
+
+mesh2 sample9 (0406/7CA6): start42204,count1440,base1243, first_vertex1243,
+VBstride24,length36864; POSITION half4, NORMAL UBYTE4N WZYX at8, COLOR at12,
+BLENDINDICES UBYTE4 WZYX at16 host00000000, BLENDWEIGHT normalized WZYX at20
+host000000FF (first weight1). Other samples use indices00000100 and weights
+000001FE (indices0/1 and weights254/255,1/255). Bone c92..95 populated, c73.w1.
+35DB sample13: start39402,count1350,base1220,first_vertex1220,VBstride36,
+length69632; indices at28 host00000100, weights at32 host000003FC. Other ranges
+start33954/base0,count2724;36678/base610,count2724;43644/base1535,count1440.
+No packed-R11-normal flag in these samples. Native culling is NONE; raw
+PA_SU_SC_MODE_CNTL=5 or6. Don't assume these samples are arms rather than body.
+
+Logo pair9100/EEE5 uses five quads in the same VB/IB: index starts0,6,12,18,24,
+bases0,4,8,12,16. POSITION float3, NORMAL UBYTE4N WZYX hostFF000000 -> w1,
+UV half2. c73.w starts0 then fades up: rebind1 captures0.50081104 and0.99324435
+for starts18/24. Alpha control00000007 (ALWAYS, enableOFF), ref1; PS c4 all1,
+PS c45=(2,4,0.25,0). Initial zero opacity was normal fade behavior, NOT proof
+of a bug. Shader multiplies normal.w*c73.w into COLOR0.x then PS output alpha.
+
+Useful remaining static questions (no request to run game):
+1. Can the exact VB/IB mesh ranges/counts above be matched to armleft/armright,
+   body or a different character using extracted GHG? Need material/submesh
+   identity and declared bone indices/count, not shader-hash identity alone.
+2. Which of the five opening-logo quads is TT? Is C395/0C18 actually selected
+   in this intro under fog=true, or merely an unused alternative in the bank?
+   Which GSC object/material controls TT's draw, and which texture entry is it?
+3. For the actual affected 0406/35DB draws, audit expected shader alpha and
+   semantics against microcode (normal.w, material opacity, skin index*3,
+   relative c92 load); don't infer shader correctness only because some body
+   portions render. User clarified torso and legs are also missing/damaged.
+4. Clip/depth state: does this title change PA_CL_CLIP_CNTL / PA_CL_VTE_CNTL
+   for these meshes/effects? Our host always enables depth clip and uses cached
+   viewport MinZ/MaxZ. Exact setters/fields only if relevant evidence exists.
+
+## Ответ получен: Addendum 18 (tiling-answer-20260930.md)
+
+**Указание пользователя: игру НЕ запускать, пока не исправлено всё из Addenda 16–18 и оставшихся пунктов и не пройдены офлайн-тесты. Потом один тестовый запуск — только по просьбе пользователя.**
+
+## Oct1 offline follow-up after Addendum18
+
+No new game launches. Exact uploaded records were inverted through both native
+conversions (per-field WZYX, then DWORD endian swap) and searched in extracted
+GHG files. ALL eight skinned samples in mesh2 belong uniquely to vortech_body:
+0406 IDs9/10/11/12 -> file offsets398811/368979/392595/380787 (24 bytes);
+35DB IDs13/14/15/16 ->450013/461353/428053/406093 (36 bytes).
+No full-record matches in either arm file. Evidence:
+rexlego/out/native-gpu/session-20261001/mesh2-asset-matches.json.
+This proves stored vertex identity, not full geometry or scene-instance validity.
+The two arm files total120722 bytes; the 'together smaller than90KB' sentence
+in Addendum18 is arithmetically incorrect. Byte matching supersedes size inference.
+
+Instance c48..51 in these body samples is finite/nonzero and shared across
+the eight draws; c92..95 varies per submesh and is populated. c247=0 alone
+does not prove invalid skinning. Full palette and camera were not captured.
+
+Opening textures: the first THREE DDS entries are dummy DXT1/OPAQ lightmaps.
+ALL FIVE real logo textures are DXT5, not just the last two. Real entries:
+0xD71 1024x1024,0x100DF1 2048x1024,0x300E71 1024x256,
+0x340EF1 1024x128,0x360F71 1024x1024. Raw DDS BC3 alpha decoder finds
+37583 fully opaque +5432 intermediate alpha pixels in TT text and49965 fully
+opaque +229595 intermediate in TT logo: neither source image is empty.
+rebind1 quad18 textureAAD29740 is1024x128 and quad24 AB71BF40 is1024x1024;
+both adopted as Xenosformat20/BC3,tiled1,swizzle688. Dimensions support object
+order but are not an exact backing-address/payload identity proof.
+
+The source shader banks of body and both arms do NOT contain the skipped
+DoF pixel hashCFEAC7ADB912F8A9. No evidence to revert the confirmed blur fix.
+
+Please continue STATIC analysis, without launching:
+1. Trace scene object creation/visibility for vortech_arms (and body submeshes)
+   into82B74FB8. Which CPU prerequisites can prevent arm Draw completely?
+   Are they gated by occlusion-query results, shader bind return codes, or an
+   animation/LOD condition? Need concrete callers/fields, not a guessed toggle.
+2. Parse the body GHG bone/submesh tables if feasible. Which material/bone group
+   occupies the eight offsets above; which remaining groups form torso/legs?
+3. For TT quads18/24, identify expected depth/instance/camera and UV c23/24.
+   Could scene material state place these behind the animated galaxy under
+   the observed ZfuncGREATEREQUAL? Native alpha test is OFF on these samples.
+4. Correct stage note: actual TU23SetVertexShader83FB6A30 usesdevice+13072;
+   SetPixelShader83FB6828 uses+13068. Confirm table ownership on these objects
+   before tracing post-creation writes (+40/+872 may refer to other structs).
+
+## Ответ получен: Addendum 19
+
+Главное: гигантский Vortech в 0001_PrologueC — рой krawlies (12000+1200+1200 кирпичей) на скелетах body/arms; меши рук — эмиттеры, их отсутствие в Draw ожидаемо. **Игру по-прежнему НЕ запускать до полного набора исправлений.**
+
+
+Oct1 offline implementation update: verified TU23 viewport states43F/400 now
+have an isolated opt-in candidate (LEGO_NATIVE_VIEWPORT). Shared ABI624 unchanged:
+592/596 NDC pixel scale,604 mode,600 alpha compare,608 color output scale.
+400 uses full-RT host viewport0..1 and VS conversion; CLIP_DISABLEbit16 controls
+host depth clipping and separate PSO identity. Opt-in init seedsVTE43F/CLIPbit19.
+No full arbitrary PA_CL_VTE_CNTL implementation claimed. This is not identified
+as a cause of missing Vortech/world meshes. Matching9290+118banks installed,
+18444+192specializations prelinked, hostlinked; alloffline tests pass, no game
+launched. New diagnostics include fullcamera/UV/bonepalette and boundedBC3logo
+upload capture; still OFF bydefault. Main visual/FPS issues remain open.
+
+## Oct1 GPU correction after reading Addendum19 (no repeated questions)
+
+The earlier GPU question about TT ZFUNC GREATEREQUAL was incorrect. Actual
+oct01-rebind1 TT samples (IDs8/9,14/15,16/17,18/19) all log RB_DEPTHCONTROL
+00700032. DecodeDepthState extracts (control>>4)&7 =3, and native's comparison
+table maps3 to LESS_EQUAL. Depth writes are OFF (bit2 clear). Clear z=1 is
+consistent with this, so the Addendum19 reversed-Z/clear0 hypothesis does not
+apply to those samples. The mistaken GEQUAL premise came from our question,
+not from new evidence. Also IDs18/24 are not stable between probe logs:
+rebind1 ID18 is TT logo quad start24/base16; ID19 is TT text start18/base12;
+rebind1 ID24 is a skinned body draw. Identify draws by source log +geometry,
+never by a naked ID from another session.
+
+New static anchor: the full class string is MechKrawlieRenderCharInstAddOn
+at821F3E58 (not the substring821F3E5C). ppcref finds830E3954/830E395C in
+830E3920 (lazy class-name hash initialization). Its factory830FDE30 creates
+a24-byte addon with vtable821F86FC. Vtable render callbacks at+20/+28 point
+to830EBA08/830EBA70 ->830E2708. The latter draws addon+16 through82D4F260;
+a direct path also reaches82CBCBE8 through82B2BAD0/82AFCE90/82BCFEF8/82CBDAE8.
+Whether this is the animated emitter drawing or the visible brick swarm is
+still not established. No game launched.
diff --git a/tools/gpu-aot/run_native_probe.ps1 b/tools/gpu-aot/run_native_probe.ps1
new file mode 100644
index 0000000..f21e394
--- /dev/null
+++ b/tools/gpu-aot/run_native_probe.ps1
@@ -0,0 +1,156 @@
+param(
+    [Parameter(Mandatory)][ValidatePattern('^[a-z0-9-]+$')][string]$Label,
+    [ValidateSet('normal','timing','snapshot')][string]$Mode = 'normal',
+    [switch]$AllTextures,
+    [switch]$CaptureMissing,
+    [switch]$LongProbe,
+    [switch]$NoMemoryWatch,
+    [switch]$StaticTextureWatch,
+    [switch]$AuditTextureWatch,
+    [switch]$NoDrawArena,
+    [switch]$AsyncCpuResources,
+    [switch]$Stencil,
+    [switch]$MeshTrace,
+    [switch]$LogoUploads,
+    [switch]$Viewport,
+    [switch]$BufferWindows,
+    [switch]$DepthAlias,
+    [switch]$DisableDepthAlias,
+    [switch]$Pm4Reference,
+    [switch]$ConfiguredRenderer,
+    [string]$LogoCaptureRenderDocDll,
+    [ValidateRange(1,8)][int]$Monitor = 1,
+    [switch]$Close
+)
+$ErrorActionPreference = 'Stop'
+if ($ConfiguredRenderer -and $Pm4Reference) { throw 'Select configured renderer or explicit PM4 reference, not both' }
+if ($DepthAlias -and $DisableDepthAlias) { throw 'DepthAlias and DisableDepthAlias are mutually exclusive' }
+if (($Pm4Reference -or $ConfiguredRenderer) -and ($Stencil -or $MeshTrace -or $LogoUploads -or $Viewport -or $BufferWindows -or $DepthAlias -or $DisableDepthAlias -or $LongProbe -or $LogoCaptureRenderDocDll)) {
+    throw 'PM4 reference mode uses original XDK/SDK objects; native-only diagnostic/candidate switches must be disabled'
+}
+$projectRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
+$runRoot = Join-Path $projectRoot 'rexlego/out/native-gpu/session-20260930'
+$exePath = Join-Path $projectRoot 'rexlego/out/build/win-amd64-release/legodimensions.exe'
+$recordPath = Join-Path $runRoot "$Label-process.json"
+if ($Close) {
+    $record = Get-Content -LiteralPath $recordPath | ConvertFrom-Json
+    $game = Get-Process -Id $record.id -ErrorAction SilentlyContinue
+    if (!$game) { Write-Output 'Probe already closed'; return }
+    if ($game.Path -ne $record.path -or
+        $game.StartTime.ToUniversalTime().Ticks -ne ([datetime]$record.startedAt).ToUniversalTime().Ticks) {
+        throw 'Process identity mismatch; refusing to close another process'
+    }
+    if (!$game.CloseMainWindow()) { throw 'Probe has no closeable main window' }
+    Write-Output "Close requested for probe PID $($record.id)"
+    return
+}
+if (Get-Process legodimensions -ErrorAction SilentlyContinue) { throw 'Game already running' }
+if (Test-Path -LiteralPath $recordPath) { throw 'Use a fresh label to preserve earlier evidence' }
+$keys = @('SDL_WINDOW_ACTIVATE_WHEN_SHOWN','LEGO_NATIVE_TIMING','LEGO_NATIVE_ALL_TEXTURES',
+    'LEGO_GPU_SNAPSHOT_DIR','LEGO_GPU_SNAPSHOT_START_AT_TILING',
+    'LEGO_GPU_SNAPSHOT_TIME_WINDOWS','LEGO_DUMP_MISSING_SHADERS',
+    'LEGO_NATIVE_GPU_DEBUG','LEGO_XENOS_TILING_TRACE','LEGO_DUMP_TEXTURE_UPLOADS','LEGO_NATIVE_TRACE_DIR',
+    'LEGO_NATIVE_NO_MEMORY_WATCH','LEGO_NATIVE_STATIC_TEXTURE_WATCH','LEGO_NATIVE_AUDIT_TEXTURE_WATCH','LEGO_NATIVE_NO_DRAW_ARENA',
+    'LEGO_NATIVE_FRAME_METRICS','LEGO_NATIVE_ASYNC_CPU_RESOURCES','LEGO_NATIVE_STENCIL','LEGO_NATIVE_MESH_TRACE',
+    'LEGO_NATIVE_VIEWPORT','LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY','LEGO_NATIVE_BUFFER_WINDOWS','LEGO_NATIVE_PM4_REFERENCE',
+    'LEGO_NATIVE_RENDERDOC_DLL','LEGO_NATIVE_RENDERDOC_CAPTURE','LEGO_NATIVE_DEPTH_ALIAS')
+$saved = @{}
+foreach ($key in $keys) {
+    $saved[$key] = [Environment]::GetEnvironmentVariable($key,'Process')
+    Remove-Item -LiteralPath "Env:\$key" -ErrorAction SilentlyContinue
+}
+try {
+    $env:SDL_WINDOW_ACTIVATE_WHEN_SHOWN = '0'
+    if ($Mode -eq 'timing') { $env:LEGO_NATIVE_TIMING = '1' }
+    if ($NoMemoryWatch) { $env:LEGO_NATIVE_NO_MEMORY_WATCH = '1' }
+    if (!$ConfiguredRenderer) {
+        $env:LEGO_NATIVE_STATIC_TEXTURE_WATCH = $(if ($StaticTextureWatch -or $AuditTextureWatch) {'1'} else {'0'})
+        $env:LEGO_NATIVE_BUFFER_WINDOWS = $(if ($BufferWindows) {'1'} else {'0'})
+    }
+    if ($StaticTextureWatch) { $env:LEGO_NATIVE_STATIC_TEXTURE_WATCH = '1' }
+    if ($AuditTextureWatch) {
+        $env:LEGO_NATIVE_STATIC_TEXTURE_WATCH = '1'
+        $env:LEGO_NATIVE_AUDIT_TEXTURE_WATCH = '1'
+    }
+    if ($NoDrawArena) { $env:LEGO_NATIVE_NO_DRAW_ARENA = '1' }
+    if ($AsyncCpuResources) { $env:LEGO_NATIVE_ASYNC_CPU_RESOURCES = '1' }
+    if ($Stencil) { $env:LEGO_NATIVE_STENCIL = '1' }
+    if ($MeshTrace) { $env:LEGO_NATIVE_MESH_TRACE = '1' }
+    if ($Viewport) { $env:LEGO_NATIVE_VIEWPORT = '1' }
+    if ($BufferWindows) { $env:LEGO_NATIVE_BUFFER_WINDOWS = '1' }
+    if ($DepthAlias) { $env:LEGO_NATIVE_DEPTH_ALIAS = '1' }
+    if ($DisableDepthAlias) { $env:LEGO_NATIVE_DEPTH_ALIAS = '0' }
+    if (!$ConfiguredRenderer) { $env:LEGO_NATIVE_PM4_REFERENCE = $(if ($Pm4Reference) {'1'} else {'0'}) }
+    if ($LogoCaptureRenderDocDll) {
+        $env:LEGO_NATIVE_RENDERDOC_DLL = (Resolve-Path -LiteralPath $LogoCaptureRenderDocDll).Path
+        $captureDir = Join-Path $runRoot "renderdoc-$Label"
+        New-Item -ItemType Directory -Path $captureDir | Out-Null
+        $env:LEGO_NATIVE_RENDERDOC_CAPTURE = Join-Path $captureDir 'tt-logo'
+    }
+    if ($LogoUploads) {
+        $env:LEGO_DUMP_TEXTURE_UPLOADS = Join-Path $runRoot "logo-uploads-$Label"
+        $env:LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY = '1'
+    }
+    if ($Mode -eq 'timing') { $env:LEGO_NATIVE_FRAME_METRICS = Join-Path $runRoot "$Label-frames.csv" }
+    if ($LongProbe) {
+        $env:LEGO_NATIVE_TIMING = '1'
+        $env:LEGO_NATIVE_TRACE_DIR = Join-Path $runRoot "trace-$Label"
+        New-Item -ItemType Directory -Path $env:LEGO_NATIVE_TRACE_DIR -Force | Out-Null
+        $env:LEGO_GPU_SNAPSHOT_DIR = Join-Path $runRoot "gpu-stages-$Label"
+        $env:LEGO_GPU_SNAPSHOT_START_AT_TILING = '1'
+    }
+    if ($AllTextures) { $env:LEGO_NATIVE_ALL_TEXTURES = '1' }
+    if ($CaptureMissing) {
+        $env:LEGO_DUMP_MISSING_SHADERS = Join-Path $runRoot "missing-$Label"
+    }
+    if ($Mode -eq 'snapshot') {
+        $env:LEGO_GPU_SNAPSHOT_DIR = Join-Path $runRoot "gpu-stages-$Label"
+        $env:LEGO_GPU_SNAPSHOT_START_AT_TILING = '1'
+        $env:LEGO_GPU_SNAPSHOT_TIME_WINDOWS = '1'
+    }
+    $logPath = Join-Path $runRoot "$Label-game.log"
+    $configSnapshot = Join-Path $runRoot "$Label-config.toml"
+    Copy-Item -LiteralPath (Join-Path (Split-Path $exePath) 'legodimensions.toml') -Destination $configSnapshot
+    $nativeEnvironment = [ordered]@{}
+    foreach ($key in $keys) {
+        $value = [Environment]::GetEnvironmentVariable($key, 'Process')
+        if ($null -ne $value) { $nativeEnvironment[$key] = $value }
+    }
+    $game = Start-Process -FilePath $exePath -WorkingDirectory (Split-Path $exePath) `
+        -ArgumentList ("--monitor=$Monitor"),("--log_file=`"$logPath`"") -WindowStyle Normal `
+        -RedirectStandardOutput (Join-Path $runRoot "$Label-stdout.log") `
+        -RedirectStandardError (Join-Path $runRoot "$Label-stderr.log") -PassThru
+    # Process.Path can briefly be empty immediately after Start-Process.
+    # The executable was launched by this exact resolved path; later checks
+    # still require the live process's actual path and start time to match.
+    $record = [ordered]@{id=$game.Id;path=$exePath;startedAt=$game.StartTime.ToString('o');
+        mode=$Mode;allTextures=[bool]$AllTextures;captureMissing=[bool]$CaptureMissing;monitor=$Monitor;
+        longProbe=[bool]$LongProbe;noMemoryWatch=[bool]$NoMemoryWatch;staticTextureWatch=[bool]$StaticTextureWatch;
+        auditTextureWatch=[bool]$AuditTextureWatch;
+        noDrawArena=[bool]$NoDrawArena;asyncCpuResources=[bool]$AsyncCpuResources;
+        stencil=[bool]$Stencil;meshTrace=[bool]$MeshTrace;
+        logoUploads=[bool]$LogoUploads;viewport=[bool]$Viewport;bufferWindows=[bool]$BufferWindows;
+        nativeDepthAliasEnabled=![bool]$DisableDepthAlias;
+        pm4Reference=[bool]$Pm4Reference;
+        configuredRenderer=[bool]$ConfiguredRenderer;
+        configSnapshot=$configSnapshot;environmentOverrides=$nativeEnvironment;
+        logoCaptureRenderDocDll=$LogoCaptureRenderDocDll;
+        exeSha256=(Get-FileHash -LiteralPath $exePath).Hash;
+        configSha256=(Get-FileHash -LiteralPath (Join-Path (Split-Path $exePath) 'legodimensions.toml')).Hash}
+    $record | ConvertTo-Json | Set-Content -LiteralPath $recordPath
+    if ($LongProbe) {
+        $probeShell=(Get-Process -Id $PID).Path
+        $watchScript=Join-Path $PSScriptRoot 'watch_native_probe.ps1'
+        $watcher=Start-Process -FilePath $probeShell -ArgumentList '-NoProfile','-File',("`"$watchScript`""),'-Label',$Label `
+            -WindowStyle Hidden -RedirectStandardOutput (Join-Path $runRoot "$Label-watch-stdout.log") `
+            -RedirectStandardError (Join-Path $runRoot "$Label-watch-stderr.log") -PassThru
+        [ordered]@{id=$watcher.Id;path=$watcher.Path;startedAt=$watcher.StartTime.ToString('o');gamePid=$game.Id} |
+            ConvertTo-Json | Set-Content -LiteralPath (Join-Path $runRoot "$Label-watch-process.json")
+    }
+    $record
+} finally {
+    foreach ($key in $keys) {
+        if ($null -eq $saved[$key]) { Remove-Item -LiteralPath "Env:\$key" -ErrorAction SilentlyContinue }
+        else { Set-Item -LiteralPath "Env:\$key" -Value $saved[$key] }
+    }
+}
diff --git a/tools/gpu-aot/sample_native_process.ps1 b/tools/gpu-aot/sample_native_process.ps1
new file mode 100644
index 0000000..1af7d43
--- /dev/null
+++ b/tools/gpu-aot/sample_native_process.ps1
@@ -0,0 +1,43 @@
+param(
+    [Parameter(Mandatory)][ValidatePattern('^[a-z0-9-]+$')][string]$Label,
+    [ValidateRange(1,30)][int]$Seconds = 5
+)
+$ErrorActionPreference = 'Stop'
+$root = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
+$runRoot = Join-Path $root 'rexlego/out/native-gpu/session-20260930'
+$record = Get-Content -LiteralPath (Join-Path $runRoot "$Label-process.json") | ConvertFrom-Json
+$game = Get-Process -Id $record.id
+if ($game.Path -ne $record.path -or $game.StartTime.ToUniversalTime().Ticks -ne
+    ([datetime]$record.startedAt).ToUniversalTime().Ticks) { throw 'Probe identity mismatch' }
+$threadsBefore = @{}
+foreach ($thread in $game.Threads) {
+    try { $threadsBefore[$thread.Id] = $thread.TotalProcessorTime.TotalSeconds } catch {}
+}
+$started = Get-Date
+$cpuBefore = $game.TotalProcessorTime.TotalSeconds
+Start-Sleep -Seconds $Seconds
+$game.Refresh()
+$elapsed = ((Get-Date) - $started).TotalSeconds
+$threads = foreach ($thread in $game.Threads) {
+    try {
+        if ($threadsBefore.ContainsKey($thread.Id)) {
+            [pscustomobject]@{id=$thread.Id;oneCorePercent=
+                [math]::Round(100*($thread.TotalProcessorTime.TotalSeconds-$threadsBefore[$thread.Id])/$elapsed,2)}
+        }
+    } catch {}
+}
+$memory = Get-CimInstance Win32_PerfFormattedData_PerfOS_Memory |
+    Select-Object AvailableMBytes,PagesInputPersec,PageReadsPersec
+$gpu = & nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader 2>$null
+$result = [ordered]@{
+    label=$Label;startedAt=$started.ToString('o');sampleSeconds=$elapsed;pid=$record.id;
+    workingSetMiB=[math]::Round($game.WorkingSet64/1MB,1);
+    privateMiB=[math]::Round($game.PrivateMemorySize64/1MB,1);
+    processCpuInCores=($game.TotalProcessorTime.TotalSeconds-$cpuBefore)/$elapsed;
+    hottestThreads=@($threads | Sort-Object oneCorePercent -Descending | Select-Object -First 6);
+    systemMemory=$memory;globalGpuNvidiaSmi=$gpu;
+    note='GPU utilization and paging are system-wide; scene must be identified separately.'
+}
+$path=Join-Path $runRoot "$Label-resources-$($started.ToString('HHmmss')).json"
+$result | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $path
+$result | ConvertTo-Json -Depth 5
diff --git a/tools/gpu-aot/simpsons-adoption.md b/tools/gpu-aot/simpsons-adoption.md
new file mode 100644
index 0000000..24d83b9
--- /dev/null
+++ b/tools/gpu-aot/simpsons-adoption.md
@@ -0,0 +1,98 @@
+# Simpsons architecture experiment for LEGO Dimensions TU23
+
+Reference: TheSimpsonsGameRecomp `8c2f2c62b1dcf1fc352766e6f41827c58604c6f7`,
+[`native_renderer_plan.md`](https://github.com/YesterMester/TheSimpsonsGameRecomp/blob/8c2f2c62b1dcf1fc352766e6f41827c58604c6f7/simpsons/re/native_renderer_plan.md).
+
+That project keeps the original XDK D3D command emission, uses PM4 as the state
+source, and progressively replaces parts of the SDK Vulkan renderer. Its shader
+translations and native resolves are checked against captured reference frames.
+Our current renderer instead replaces XDK functions with direct D3D12 draws.
+
+## Implemented first experiment
+
+`LEGO_NATIVE_PM4_REFERENCE=1` routes **all 55 native GPU replacements** to their
+generated original `__imp__sub_*` implementations. The SDK GPU plugin therefore
+receives the original command stream. This covers creation, placement resources,
+shader binding, state, draws, resolves, queries, kicks and synchronization.
+Both typed and raw hook wrappers select the route before any native body runs.
+Selection is fixed at first use to prevent mixing resource ownership models.
+Normal launches read the restart-required `gpu_native_pm4` setting. An exact
+environment value `1` forces PM4 and `0` forces the detached native renderer;
+other environment values use the setting. Its compiled default is false, while
+the current development configuration now persists true after visual validation.
+The application startup uses the same selector: reference mode preserves the
+SDK GPU plugin and skips creating the detached native device. Shutdown skips
+native device teardown in reference mode as well.
+
+The probe runner exposes `-Pm4Reference` to force PM4 and `-ConfiguredRenderer`
+to test the persisted selection with no environment override. With neither flag
+the runner explicitly selects detached native mode for existing diagnostic probes.
+Native-only diagnostic and candidate
+flags cannot be combined with this mode. The runner clears and restores the
+environment and records the chosen route and executable/config hashes.
+
+This is a reference experiment, **not a port of the Simpsons native Vulkan
+resolves, AOT SPIR-V storage, or headless trace replayer**. It uses our installed
+SDK GPU plugin and its configured backend (currently D3D12). The original SDK
+translation/cache path supplies shaders; our DXIL archive and native blur
+override are bypassed along with the native hooks. Existing configuration shader
+filters still apply. Performance and blur must therefore be assessed separately
+from whether missing geometry, logos and effects appear.
+
+## Verification and next decision
+
+`test_renderer_route.py` compiles the actual routing header against a mock PPC
+context/marshaler, generates a wrapper for every real native GPU hook, and checks
+both paths across twelve environment/configuration cases. It verifies original-body
+exclusivity, persisted selection and overrides, argument/register preservation,
+and immutable route selection.
+Full application linking also verifies every original target is available.
+These are offline dispatch checks, not proof that original GPU initialization
+and the full game work in this executable.
+
+The first authorized run (`oct01-pm4-reference1`) exposed a missing startup
+switch: the old app setup cleared the SDK plugin even though the hooks selected
+PM4. It stopped before the cutscene with no GPU ring consumer. The process was
+closed. Startup selection is now corrected and the offline test executes the
+actual `OnPreSetup` / `OnPreLaunchModule` bodies against mock host services to
+verify plugin preservation and exclusive native initialization.
+
+`oct01-pm4-reference2` successfully booted on the main monitor. Observed in the
+cutscene: Vortech's legs and translucent geometry, blue particles/light, and
+the Dimensions logo vortex. At the title screen, the central portal, floating
+glowing spheres, orbit lines and three complete characters appeared. Evidence
+is under `rexlego/out/native-gpu/session-20261001/pm4-reference2/`. This materially
+improves the reported defects; it does not prove every game area is correct.
+The process was closed after the title-screen capture. The first pipeline warmup
+took 78.5 seconds; a subsequent configured run reused the driver cache and took
+0.97 seconds for the same 9707 pipeline descriptions. This is startup timing,
+not gameplay FPS.
+
+`oct01-pm4-configured1` also used this route with no environment override and
+captured Vortech's translucent torso and brick-built hand/arm. The persisted
+setting therefore affects an ordinary launch, rather than only a probe command.
+`oct01-pm4-tt3` captured the complete TT Games logo and production text in
+`session-20261001/pm4-tt3/logo-12.png`. Earlier spot observations missed its short
+display interval; they were not evidence that it was still missing.
+
+The latest user instruction authorizes necessary tests on the PRIMARY monitor
+without requesting approval for every run. Close each exact process after the
+test. Correct rendering takes priority over FPS. The reference path has now
+visibly restored the reported cutscene geometry, logo and title effects, but
+unvisited levels and performance remain unverified. An improved PM4 reference
+identifies a difference somewhere in the native hook/translation pipeline; it
+does not isolate that difference by itself.
+
+Menu navigation remains unverified: brief automated Return/Escape/J presses
+did not leave the title prompt. This is an input-test limitation, not established
+evidence of a rendering regression. The existing project notes already mention
+that injected keyboard events may fail to reach SDL. No controller/input code
+was changed for this renderer experiment. Do not disable synchronous memexport
+readback to improve this result: previous fast/batched/on-demand trials caused
+known geometry and UI regressions (see PROJECT-MEMORY.md sections 5.4-5.5).
+
+Before adopting native resolve optimizations from Simpsons, establish an SDK
+reference image, map TU23 resolve formats/aliasing/CPU reads, and reproduce the
+same captured commands offline. Title-specific addresses and hardcoded reference
+resolve regions from Simpsons must not be used for LEGO. No native resolve
+optimization is enabled by this experiment.
diff --git a/tools/gpu-aot/summarize_brick_resources.py b/tools/gpu-aot/summarize_brick_resources.py
new file mode 100644
index 0000000..161de64
--- /dev/null
+++ b/tools/gpu-aot/summarize_brick_resources.py
@@ -0,0 +1,36 @@
+"""Group bounded mesh samples by resource provenance, without asset attribution."""
+import argparse
+import collections
+import json
+import re
+from pathlib import Path
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('mesh', type=Path)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+groups = collections.defaultdict(list)
+for draw in json.loads(a.mesh.read_text())['draws']:
+    if not any('brick_pair=true' in o for o in draw['observations']):
+        continue
+    observations = draw['observations']
+    resource = next((o for o in observations if o.startswith('stream=0 data=')), '')
+    index = next((o for o in observations if o.startswith('index_data=')), '')
+    fields = dict(re.findall(r'(\w+)=([^ ]+)', resource + ' ' + index))
+    key = (draw['IB'], fields.get('data'), fields.get('index_data'), draw['start'], draw['count'])
+    constants = {}
+    for o in observations:
+        m = re.fullmatch(r'c(\d+)=\(([^)]+)\)', o)
+        if m:
+            constants[int(m[1])] = [float(v) for v in m[2].split(',')]
+    frame = next((int(re.search(r'frame=(\d+)', o)[1]) for o in observations if o.startswith('frame=')), None)
+    groups[key].append({'id':draw['id'], 'time':draw['timestamp'], 'frame':frame,
+        'resource':resource, 'index':index, 'constants':constants})
+report = {'note':'Bounded samples only. Shader/resource identity does not prove Vortech ownership. Stable geometry with varying instance matrices can be normal individual brick drawing.',
+    'groups':[{'IB':key[0], 'vertex_data':key[1], 'index_data':key[2],
+        'start':key[3], 'count':key[4], 'samples':rows,
+        'distinct_uploads':len({r['resource'] for r in rows}),
+        'distinct_instance_matrices':len({str([r['constants'].get(c) for c in range(48,52)]) for r in rows})}
+        for key,rows in groups.items()]}
+a.output.write_text(json.dumps(report,indent=2)+'\n')
+print(json.dumps([{k:g[k] for k in ('IB','vertex_data','index_data','start','count','distinct_uploads','distinct_instance_matrices')} for g in report['groups']],indent=2))
diff --git a/tools/gpu-aot/summarize_frame_metrics.py b/tools/gpu-aot/summarize_frame_metrics.py
new file mode 100644
index 0000000..752bb8b
--- /dev/null
+++ b/tools/gpu-aot/summarize_frame_metrics.py
@@ -0,0 +1,48 @@
+"""Summarize every-present timings; snapshots are unnecessary for this probe."""
+import argparse
+import csv
+import io
+import json
+from pathlib import Path
+
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('file',type=Path)
+p.add_argument('--tail-seconds',type=float,default=30)
+p.add_argument('--after-frame',type=int,default=0)
+p.add_argument('--through-frame',type=int)
+a=p.parse_args()
+text=a.file.read_text();text=text[:text.rfind('\n')+1] # Exclude a partial live row.
+rows=[]
+for row in csv.DictReader(io.StringIO(text)):
+ try: rows.append({k:float(v) for k,v in row.items()})
+ except (TypeError,ValueError):continue
+rows=[r for r in rows if r['frame']>a.after_frame and
+      (a.through_frame is None or r['frame']<=a.through_frame)]
+tail=[];elapsed=0
+for row in reversed(rows):
+ tail.append(row);elapsed+=row['interval_ms']/1000
+ if elapsed>=a.tail_seconds:break
+def stats(data):
+ if not data:return {}
+ times=sorted(r['interval_ms'] for r in data)
+ mean=lambda key:sum(r[key] for r in data)/len(data)
+ result={'frames':len(data),'seconds':sum(times)/1000,'mean_fps':1000/mean('interval_ms'),
+ 'frame_ms_p50':times[(len(times)-1)//2],'frame_ms_p95':times[int((len(times)-1)*.95)],
+ 'frame_ms_max':max(times),'over_33_5ms_percent':100*sum(t>33.5 for t in times)/len(times),
+ 'texture_ms_mean':mean('texture_ms'),'constants_ms_mean':mean('constants_ms'),
+ 'vertices_ms_mean':mean('vertices_ms'),'buffer_hash_ms_mean':mean('buffer_hash_ms'),
+ 'present_cpu_ms_mean':mean('present_cpu_ms'),'draws_mean':mean('draw_calls')}
+ for key in ['cpu_resource_wait_skips','bindings_ms','begin_ms','pipeline_ms','issue_ms',
+             'tail_ms','buffer_converted_bytes'] + [k for k in data[0] if k.startswith(('sync_', 'callbacks_'))]:
+  if key in data[0]:result[key+'_mean']=mean(key)
+ return result
+record_path=a.file.with_name(a.file.name.replace('-frames.csv','-process.json'))
+record=json.loads(record_path.read_text(encoding='utf-8-sig')) if record_path.exists() else None
+heavy_capture=bool(record and (record.get('mode')=='snapshot' or any(record.get(k) for k in
+ ['longProbe','logoUploads','logoCaptureRenderDocDll','captureMissing','meshTrace','auditTextureWatch'])))
+report={'source':str(a.file),'whole_run':stats(rows),'tail':stats(tail),
+        'latest_frame':rows[-1]['frame'] if rows else None,
+        'after_frame':a.after_frame,'through_frame':a.through_frame,
+        'scene_identity_requires_visual_confirmation':True,
+        'clean_performance_probe':not heavy_capture if record else None}
+a.file.with_suffix('.summary.json').write_text(json.dumps(report,indent=2))
+print(json.dumps(report,indent=2))
diff --git a/tools/gpu-aot/summarize_long_probe.py b/tools/gpu-aot/summarize_long_probe.py
new file mode 100644
index 0000000..5813864
--- /dev/null
+++ b/tools/gpu-aot/summarize_long_probe.py
@@ -0,0 +1,36 @@
+"""Summarize long-run events without treating transitions as confirmed faults."""
+import argparse
+from collections import Counter,deque
+import csv
+import json
+from pathlib import Path
+import re
+
+parser=argparse.ArgumentParser(description=__doc__)
+parser.add_argument('directory',type=Path)
+args=parser.parse_args()
+counts=Counter(); candidates=deque(maxlen=30); intervals=[]; duration=0; pixels=[]
+with (args.directory/'events.tsv').open() as file:
+    for row in csv.DictReader(file,delimiter='\t'):
+        if not row.get('details'): continue
+        counts[row['event']]+=1
+        duration=max(duration,float(row['elapsed_ms']))
+        if row['anomaly_candidate']=='1': candidates.append(row)
+        if row['event']=='frame':
+            match=re.search(r'interval_ms=\s*([0-9.eE+-]+)',row['details'])
+            if match: intervals.append(float(match[1]))
+        if row['event']=='present_pixels': pixels.append(row)
+ordered=sorted(intervals)
+def percentile(p): return ordered[min(len(ordered)-1,int((len(ordered)-1)*p))] if ordered else None
+report={'duration_seconds':duration/1000,'event_counts':dict(counts),
+        'frame_count':len(intervals),'frame_ms_p50':percentile(.5),'frame_ms_p95':percentile(.95),
+        'frame_ms_max':max(intervals,default=0),'last_anomaly_candidates':list(candidates),
+        'present_pixel_samples':pixels,'diagnostic_capture_overhead_included':True,
+        'menu_idle_10min_confirmed':False,'visual_faults_require_screenshot_review':True}
+monitor=args.directory/'monitor.jsonl'
+if monitor.exists():
+    for line in monitor.read_text().splitlines():
+        event=json.loads(line)
+        if event['event']=='menu_idle_600s_elapsed': report['menu_idle_10min_confirmed']=True
+(args.directory/'summary.json').write_text(json.dumps(report,indent=2),encoding='utf-8')
+print(json.dumps({k:v for k,v in report.items() if k not in ('last_anomaly_candidates','present_pixel_samples')},indent=2))
diff --git a/tools/gpu-aot/summarize_mesh_trace.py b/tools/gpu-aot/summarize_mesh_trace.py
new file mode 100644
index 0000000..41ed2dc
--- /dev/null
+++ b/tools/gpu-aot/summarize_mesh_trace.py
@@ -0,0 +1,33 @@
+"""Summarize opt-in mesh trace evidence; hashes alone do not identify models."""
+import argparse
+import collections
+import json
+import re
+from pathlib import Path
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('log', type=Path)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+draws = {}
+for line in a.log.read_text(errors='replace').splitlines():
+    m = re.search(r'Native mesh: id=(\d+) (.*)', line)
+    if not m:
+        continue
+    row = draws.setdefault(m[1], {'id': int(m[1]), 'observations': []})
+    payload = m[2]
+    if payload.startswith('VS='):
+        row.update(re.findall(r'(\w+)=([^ ]+)', payload))
+        row['timestamp'] = line.split(']')[0].lstrip('[')
+    elif payload.startswith('submitted='):
+        row['submitted'] = payload.endswith('true')
+    else:
+        row['observations'].append(payload)
+pairs = collections.Counter((v.get('VS'), v.get('PS')) for v in draws.values())
+report = {'log': str(a.log.resolve()), 'note': 'Distinct geometry samples, not total draws. First-use opacity can be zero during fades. Same shader hash does not prove asset identity. Trace overhead excludes FPS comparison.',
+          'sample_count': len(draws), 'submitted': sum(v.get('submitted', False) for v in draws.values()),
+          'pairs': [{'VS': vs, 'PS': ps, 'samples': n} for (vs, ps), n in pairs.items()],
+          'draws': list(draws.values())}
+a.output.parent.mkdir(parents=True, exist_ok=True)
+a.output.write_text(json.dumps(report, indent=2) + '\n')
+print(json.dumps({k: report[k] for k in ('sample_count', 'submitted', 'pairs')}, indent=2))
diff --git a/tools/gpu-aot/test_alpha_test.py b/tools/gpu-aot/test_alpha_test.py
new file mode 100644
index 0000000..5a2d39a
--- /dev/null
+++ b/tools/gpu-aot/test_alpha_test.py
@@ -0,0 +1,177 @@
+"""TU23 alpha setters/defaults, all compares, and actual microcode -> DXIL.
+
+No game or window is started. C++ tests share the exact emitted HLSL helper.
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import struct
+import subprocess
+from test_depth_export import fixture
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('compiler', type=Path)
+p.add_argument('common', type=Path)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+root = Path(__file__).resolve().parents[2]
+a.output.mkdir(parents=True, exist_ok=True)
+
+def body(source, sig):
+    start = source.index(sig)
+    end = source.index('{', start) + 1
+    depth = 1
+    while depth:
+        depth += (source[end] == '{') - (source[end] == '}')
+        end += 1
+    return source[start:end]
+
+image = (root / 'xexdump/dump/default.bin').read_bytes()
+indices = (15,16,18,19,20,21,22,23,24,26,53,54,55,56,59)
+table = [struct.unpack_from('>III', image, 0x847F9B18 - 0x82000000 + 12*k) for k in indices]
+assert [t[1] for t in table] == [0x83FB7E38,0x83FB81C8,0x83FB7F58,0x83FB7FE8,
+    0x83FB7EC8,0x83FB80E8,0x83FB8158,0x83FB8078,0x83FB7E00,0x83FB82C0,
+    0x83FB8A68,0x83FB8AA8,0x83FB8AE8,0x83FB8B28,0x83FB92B8]
+assert [t[2] for t in table] == [0,0,1,0,0,1,0,0,0,7,15,15,15,15,0x3F800000]
+functions = []
+for path in (root / 'rexlego/generated/default').glob('*.cpp'):
+    s = path.read_text()
+    for _, setter, _ in table:
+        sig = f'DEFINE_REX_FUNC(sub_{setter:08X})'
+        if sig in s:
+            f = body(s, sig)
+            # All direct branches stay inside these CPU-only setters.
+            instructions = re.findall(r'^\s*// ([^\n]+)', f, re.M)
+            for asm in instructions:
+                branch = re.match(r'b[l]? 0x([A-Fa-f0-9]+)', asm)
+                if branch:
+                    assert setter <= int(branch[1],16) < setter+4*len(instructions)
+                assert not re.match(r'bctr|bctrl', asm)
+            assert not re.search(r'13928\(r\d+\)|13932\(r\d+\)', f)
+            functions.append(f)
+assert len(functions) == 15
+initializer = body((root / 'rexlego/src/gpu_native/hooks_device.cpp').read_text(),
+                   'bool InitializeRenderDefaults(')
+shared = body((root / 'rexlego/src/gpu_native/draw.cpp').read_text(), 'struct SharedConstants') + ';'
+h = a.output / 'alpha-test.cpp'
+h.write_text(r'''
+#include <array>
+#include <bit>
+#include <cassert>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <vector>
+#include "gpu_native/alpha_test.h"
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using uint=unsigned;
+#include "alpha_compare.hlsli"
+union PPCRegister {u64 u64;int64_t s64;u32 u32;int32_t s32;u8 u8;float f32;};
+struct FReg {double f64=0;};struct Fpscr {void disableFlushMode(){}};
+struct XER {u8 ca=0;};
+struct CR {bool eq=false,gt=false,lt=false;template<class T>void compare(T a,T b,XER){eq=a==b;gt=a>b;lt=a<b;}};
+struct PPCContext {
+'''+''.join(f'PPCRegister r{i}{{}};\n' for i in range(32))+r'''
+ FReg f0;Fpscr fpscr;XER xer;CR cr0,cr6;
+};
+std::array<u8,65536> ram{};
+u32 Load(u32 p){return (u32(ram[p])<<24)|(u32(ram[p+1])<<16)|(u32(ram[p+2])<<8)|ram[p+3];}
+void Store(u32 p,u32 v){for(u32 i=0;i<4;++i)ram[p+i]=u8(v>>(24-8*i));}
+u64 Load64(u32 p){return (u64(Load(p))<<32)|Load(p+4);}
+void Store64(u32 p,u64 v){Store(p,u32(v>>32));Store(p+4,u32(v));}
+#define REX_LOAD_U32(p) Load(p)
+#define REX_STORE_U32(p,v) Store(p,v)
+#define REX_LOAD_U64(p) Load64(p)
+#define REX_STORE_U64(p,v) Store64(p,v)
+#define REX_FUNC_PROLOGUE() ((void)0)
+#define DEFINE_REX_FUNC(name) void name(PPCContext& ctx,u8* base)
+#define REXLOG_ERROR(...) ((void)0)
+#define REXLOG_INFO(...) ((void)0)
+'''+ '\n'.join(functions) + r'''
+struct be_u32 {u32 raw;operator u32()const{return __builtin_bswap32(raw);}};
+constexpr u32 kRenderStateTable=4096;
+std::vector<u32> calls;
+struct Dispatcher {u32 missing=0;u32 GetFunction(u32 p){return p==missing?0:p;}} dispatcher;
+struct Kernel {auto function_dispatcher(){return &dispatcher;}} kernel;
+#define REX_KERNEL_STATE() (&kernel)
+namespace rex::ppc {
+template<class T>void GuestToHostFunction(u32 f,u32 device,u32 value){
+ PPCContext c{};c.r1.u32=60000;c.r3.u32=device;c.r4.u32=value;calls.push_back(f);
+'''+ '\n'.join((' if' if i==0 else ' else if')+f'(f==0x{t[1]:08X})sub_{t[1]:08X}(c,ram.data());'
+               for i,t in enumerate(table))+r'''
+ else assert(false);
+}
+}
+'''+initializer+'\n'+shared+r'''
+int main(){
+ constexpr u32 device=12288;
+'''+''.join(f'Store(kRenderStateTable+12*{k}+4,{t[1]}u);Store(kRenderStateTable+12*{k}+8,{t[2]}u);\n'
+           for k, t in zip(indices, table))+r'''
+ assert(InitializeRenderDefaults(device,ram.data()));assert(calls.size()==15);
+ assert(Load(device+10556)==7&&Load(device+10620)==0x3F800000);
+ assert(Load(device+10568)==0); // Cull/fill untouched.
+ for(u32 at: {10552u,10584u,10588u,10592u})assert(Load(device+at)==0x00010001);
+ for(u32 at: {12292u,12296u,12300u,12304u})assert(Load(device+at)==15);
+ dispatcher.missing=0x83FB92B8;calls.clear();assert(!InitializeRenderDefaults(device,ram.data()));assert(calls.empty());
+ dispatcher.missing=0;Store(kRenderStateTable+24*12+4,0xDEADBEEF);
+ assert(!InitializeRenderDefaults(device,ram.data()));assert(calls.empty());
+ using namespace legodimensions::gpu_native;
+ const float values[]={-INFINITY,-32.f,-1.f,-0.f,0.f,0.2f,0.5f,1.f,32.f,INFINITY,NAN};
+ for(float alpha:values)for(float reference:values)for(u32 f=0;f<8;++f){
+  // Independent Xenos less/equal/greater pass mask, with its NOTEQUAL/ALWAYS rules.
+  bool expected=f==7||(f==5?alpha!=reference:
+    (((f&1)!=0&&alpha<reference)||((f&2)!=0&&alpha==reference)||((f&4)!=0&&alpha>reference)));
+  assert(LegoAlphaPass(alpha,reference,f)==expected);
+  auto decoded=DecodeNativeAlphaState(0xF0000000|8|f,reference);
+  assert(decoded.enabled&&decoded.function==f);
+  assert(std::bit_cast<u32>(decoded.reference)==std::bit_cast<u32>(reference));
+  assert(!DecodeNativeAlphaState(f,reference).enabled);
+ }
+ // Execute original setters for every compare and non-integer reference.
+ for(u32 f=0;f<8;++f){
+  rex::ppc::GuestToHostFunction<void>(0x83FB82C0,device,f);
+  rex::ppc::GuestToHostFunction<void>(0x83FB7E00,device,1);
+  rex::ppc::GuestToHostFunction<void>(0x83FB92B8,device,std::bit_cast<u32>(0.37f));
+  auto d=DecodeNativeAlphaState(Load(device+10556),std::bit_cast<float>(Load(device+10620)));
+  assert(d.enabled&&d.function==f&&d.reference==0.37f);
+ }
+ static_assert(sizeof(SharedConstants)==624);
+ static_assert(offsetof(SharedConstants,alpha_function)==600);
+ static_assert(offsetof(SharedConstants,alpha_threshold)==556);
+ static_assert(offsetof(SharedConstants,color_output_scale)==608);
+}
+''')
+exe = a.output / 'alpha-test.exe'
+subprocess.run(['clang++', '-std=c++20', '-DNOMINMAX', '-I'+str(root/'rexlego/src'),
+                '-I'+str(root/'tools/gpu-aot'), str(h), '-o', str(exe)], check=True)
+subprocess.run([str(exe.resolve())], check=True)
+
+inputs = a.output / 'input'
+inputs.mkdir(exist_ok=True)
+cases = {'color': {}, 'color_predicated': {'predicated': True}, 'depth': {'depth': True},
+         'depth_scalar': {'depth': True, 'scalar': True}, 'depth_predicated': {'depth': True, 'predicated': True}}
+for name, opts in cases.items():
+    (inputs / f'{name}.bin').write_bytes(fixture(**opts))
+env = os.environ.copy()
+env['XENOS_RECOMP_DXIL_ONLY'] = '1'
+env['XENOS_RECOMP_LEGO_NATIVE_SCALE'] = '1'
+env.pop('XENOS_RECOMP_ONLY_HASH', None)
+subprocess.run([str(a.compiler.resolve()), str(inputs.resolve()),
+                str((a.output/'fixture-cache.cpp').resolve()), str(a.common.resolve()),
+                str((a.output/'hlsl').resolve())], env=env, check=True)
+for name, opts in cases.items():
+    s = (a.output/'hlsl'/f'{name}.hlsl').read_text()
+    assert f'#define LEGO_SHADER_WRITES_COLOR0 {int(not opts.get("depth",False))}' in s
+    assert 'g_AlphaFunction : packoffset(c37.z)' in s
+    assert s.index('if (!LegoAlphaPass(') < s.index('oC0 *= g_ColorOutputScale.x;')
+    assert ('oDepth : SV_Depth' in s) == opts.get('depth',False)
+(a.output/'verification.json').write_text(json.dumps({
+    'passed': True, 'actual_tu23_setters': [f'{t[1]:08X}' for t in table],
+    'table_defaults': [f'{t[2]:08X}' for t in table], 'shared_bytes': 624,
+    'checks': ['original leaf setters and native initializer', 'table validation before mutation',
+               '968 comparisons incl NaN/inf', 'raw float reference', 'no cull/fill writes',
+               '5 actual microcode/HLSL/DXIL fixtures', 'depth-only gate', 'compare before EDRAM scale'],
+    'game_visual_validation': False}, indent=2))
+print('PASS: TU23 defaults/setters, eight compares, and five HLSL/DXIL fixtures; no game launched.')
diff --git a/tools/gpu-aot/test_audit_d3d_paths.py b/tools/gpu-aot/test_audit_d3d_paths.py
new file mode 100644
index 0000000..5d4ef91
--- /dev/null
+++ b/tools/gpu-aot/test_audit_d3d_paths.py
@@ -0,0 +1,43 @@
+import contextlib
+import io
+import importlib.util
+from pathlib import Path
+import tempfile
+import unittest
+
+spec = importlib.util.spec_from_file_location(
+    'audit_d3d_paths', Path(__file__).with_name('audit_d3d_paths.py'))
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+audit = module.audit
+
+
+class AuditTests(unittest.TestCase):
+    def test_direct_calls_hook_barrier_and_indirect_warning(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            for path in ('tools/gpu-aot', 'rexlego/src/gpu_native', 'rexlego/generated/default'):
+                (root / path).mkdir(parents=True)
+            (root / 'tools/gpu-aot/d3d-sep13-matches.tsv').write_text(
+                'Seed\t0x00000000\t0x83FAF180\td3d9:ring.obj\n'
+                '# ambiguous ignored\t0x0\t0x1,0x2\tx\n')
+            (root / 'rexlego/src/gpu_native/hooks.cpp').write_text('REX_HOOK_RAW(sub_83000002) {}')
+            (root / 'rexlego/generated/default/test_recomp.cpp').write_text('''
+DEFINE_REX_FUNC(sub_83000001) { sub_83FAF180(ctx, base); REX_CALL_INDIRECT_FUNC(ctx.ctr.u32); }
+DEFINE_REX_FUNC(sub_83000002) { sub_83000001(ctx, base); }
+DEFINE_REX_FUNC(sub_83000003) { sub_83000002(ctx, base); }
+DEFINE_REX_FUNC(sub_83000004) { sub_83FCBF00(ctx, base); }
+''')
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                audit(root)
+            result = output.getvalue()
+            self.assertIn('83000001\t0\t1\t83000001 -> 83FAF180', result)
+            self.assertIn('83000002\t1\t0\t83000002 -> 83000001 -> 83FAF180', result)
+            self.assertNotIn('83000003\t', result)
+            self.assertIn('83000004\t0\t0\t83000004 -> 83FCBF00', result)
+            self.assertIn('ambiguous_rows=1', result)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tools/gpu-aot/test_blend_state.cpp b/tools/gpu-aot/test_blend_state.cpp
new file mode 100644
index 0000000..394d179
--- /dev/null
+++ b/tools/gpu-aot/test_blend_state.cpp
@@ -0,0 +1,17 @@
+#include "../../rexlego/src/gpu_native/blend_state.h"
+using namespace legodimensions::gpu_native;
+static_assert(!DecodeColorBlend(0x00010001).enabled);
+static_assert(DecodeColorBlend(0x00010001).supported);
+// Actual TT logo/fade blend: source alpha / inverse source alpha; independent alpha.
+constexpr auto fade = DecodeColorBlend(0x07010706);
+static_assert(fade.enabled && fade.supported);
+static_assert(fade.source == 6 && fade.destination == 7 && fade.operation == 0);
+static_assert(fade.alpha_source == 1 && fade.alpha_destination == 7 && fade.alpha_operation == 0);
+static_assert(DecodeColorBlend(0x01010101).supported); // additive
+static_assert(DecodeColorBlend(0x00010081).operation == 4); // reverse subtract
+static_assert(!DecodeColorBlend(0x000100E1).supported); // invalid op
+static_assert(!DecodeColorBlend(0x00010002).supported); // invalid factor
+static_assert(!DecodeColorBlend(0x0001000E).supported); // constant alpha is not constant RGB
+static_assert(DecodeColorBlend(0x00010D0C).supported); // constant RGB
+static_assert(!DecodeColorBlend(0xE001E001).enabled); // reserved bits don't affect identity
+int main() {}
diff --git a/tools/gpu-aot/test_buffer_header.cpp b/tools/gpu-aot/test_buffer_header.cpp
new file mode 100644
index 0000000..470f7a5
--- /dev/null
+++ b/tools/gpu-aot/test_buffer_header.cpp
@@ -0,0 +1,14 @@
+#include "../../rexlego/src/gpu_native/buffer_header.h"
+using namespace legodimensions::gpu_native;
+constexpr auto vertex = DecodePlacementBuffer(1, 0xF933E003, 0x100AF002, false);
+static_assert(vertex.address == 0xF933E000 && vertex.size == 0xAF000);
+constexpr auto index16 = DecodePlacementBuffer(0x20000002, 0xF8823000, 0x1000, true);
+static_assert(index16.address == 0xF8823000 && index16.size == 0x1000 && index16.index_format == 1);
+constexpr auto index32 = DecodePlacementBuffer(0xA0000002, 0xF8823000, 0x1000, true);
+static_assert(index32.index_format == 2);
+static_assert(!DecodePlacementBuffer(1, 0xF933E003, 0x100AF002, true).address);
+static_assert(!DecodePlacementBuffer(6, 0xF933E003, 0x100AF002, false).address);
+static_assert(!DecodePlacementBuffer(1, 3, 0x1000, false).address);
+static_assert(!DecodePlacementBuffer(1, 0xF0000000, 2, false).address);
+static_assert(!DecodePlacementBuffer(1, 0xFFFFFFF0, 0x1000, false).address);
+int main() {}
diff --git a/tools/gpu-aot/test_buffer_variants.py b/tools/gpu-aot/test_buffer_variants.py
new file mode 100644
index 0000000..5ca4a8c
--- /dev/null
+++ b/tools/gpu-aot/test_buffer_variants.py
@@ -0,0 +1,214 @@
+"""Check fetched-byte equivalence and actual buffer conversion cache behavior."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True)
+source=(root/'rexlego/src/gpu_native/buffers.cpp').read_text()
+def body(signature):
+ start=source.index(signature);pos=source.index('{',start);end=pos+1;depth=1
+ while depth:depth+=(source[end]=='{')-(source[end]=='}');end+=1
+ return source[start:end]
+resource=source[source.index('struct BufferResource {'):source.index('std::mutex g_buffers_mutex;')]
+h=a.output/'buffer-variants-test.cpp'
+h.write_text(r'''
+#include <array>
+#include <cassert>
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <random>
+#include <span>
+#include <unordered_map>
+#include <vector>
+#define XXH_INLINE_ALL
+#include <xxhash.h>
+#include "gpu_native/vertex_byte_order.h"
+#include "gpu_native/memory_watch.h"
+#include "gpu_native/buffer_header.h"
+#include "gpu_native/buffer_window.h"
+using namespace legodimensions::gpu_native;
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using be_u32=uint32_t;
+#define REXLOG_WARN(...) ((void)0)
+#define REXLOG_ERROR(...) ((void)0)
+#define REXLOG_INFO(...) ((void)0)
+enum class BufferKind:u32{kVertex=6,kIndex=7};
+namespace plume {
+enum class RenderHeapType {UPLOAD};
+struct RenderBufferDesc {
+ u32 size;static auto IndexBuffer(u32 n,RenderHeapType){return RenderBufferDesc{n};}
+ static auto VertexBuffer(u32 n,RenderHeapType){return RenderBufferDesc{n};}
+};
+struct RenderBuffer {
+ std::vector<u8> data; explicit RenderBuffer(u32 size):data(size){}
+ void* map(){return data.data();}void unmap(){}void setName(const char*){}
+};
+struct Device {
+ u32 allocations=0;
+ std::unique_ptr<RenderBuffer> createBuffer(RenderBufferDesc d){++allocations;return std::make_unique<RenderBuffer>(d.size);}
+};
+}
+struct HostDevice {
+ static bool IsReady(){return true;}
+ static inline plume::Device device;
+ static inline std::vector<std::shared_ptr<void>> held;
+ static auto Device(){return &device;}
+ static void RetireResource(std::shared_ptr<void> r){held.push_back(std::move(r));}
+};
+struct Memory {
+ std::array<u8,65536> bytes{};
+ template<class T>T TranslateVirtual(u32 at){return reinterpret_cast<T>(bytes.data()+at);}
+};
+Memory memory;
+#define REX_KERNEL_MEMORY() (&memory)
+bool NativeTextureTimingEnabled(){return true;}
+struct BufferUploadTiming {u64 calls=0,hashed_bytes=0,converted_bytes=0;double hash_ms=0;};
+BufferUploadTiming g_buffer_timing;
+'''+resource+r'''
+std::shared_ptr<BufferResource> resource;
+auto AdoptBuffer(u32,BufferKind)->std::shared_ptr<BufferResource>{return resource;}
+struct BufferResourceView {
+ plume::RenderBuffer* buffer=nullptr;u32 length=0,guest_format=0,mirror_address=0;
+ u64 content_hash=0,byte_order_hash=0;
+};
+'''+body('void ByteSwapElements(')+'\n'+body('plume::RenderBuffer* ResolveBufferContents(')+'\n'+body('plume::RenderBuffer* ResolveBufferResource(')+'\n'+body('BufferResourceView ResolveBufferResourceWindow(')+r'''
+// Exercise placement-header adoption too: the conversion tests above use a
+// fixed resource and cannot detect selecting the wrong half of a pool owner.
+struct D3DBuffer {struct {u32 common;u32 rest[5];} resource;u32 fetch_lo,fetch_hi;};
+std::mutex g_buffers_mutex;
+std::unordered_map<u32,std::shared_ptr<BufferResource>> g_buffers;
+D3DBuffer* GuestBuffer(u32 at){return memory.TranslateVirtual<D3DBuffer*>(at);}
+std::shared_ptr<BufferResource> FindBuffer(u32 at){
+ std::lock_guard lock(g_buffers_mutex);auto it=g_buffers.find(at);
+ return it==g_buffers.end()?nullptr:it->second;
+}
+'''+body('std::shared_ptr<BufferResource> AdoptBuffer(').replace('AdoptBuffer(', 'ActualAdoptBuffer(')+ '\n'+body('plume::RenderBuffer* ResolveBufferResource(').replace('ResolveBufferResource(', 'ResolveAdoptedBuffer(').replace('AdoptBuffer(', 'ActualAdoptBuffer(')+r'''
+int main() {
+ std::mt19937 random(17);
+ // Canonicalization must preserve every byte the ORIGINAL stream can fetch.
+ for(u32 n=0;n<10000;++n) {
+  const u32 stride=4*(1+random()%24),offset=random()%512;
+  const std::array<u32,2> fields{0, stride-4};
+  std::vector<u8> old(1024),canonical(1024);
+  for(auto& byte:old)byte=random();canonical=old;
+  const VertexByteOrder order{stride,offset,fields};
+  assert(ApplyVertexByteOrder(old.data(),old.size(),order));
+  assert(ApplyVertexByteOrder(canonical.data(),canonical.size(),CanonicalVertexByteOrder(order)));
+  assert(std::equal(old.begin()+offset,old.end(),canonical.begin()+offset));
+ }
+ resource=std::make_shared<BufferResource>();resource->length=1024;resource->mirror_address=32;
+ for(u32 i=0;i<1024;++i)memory.bytes[32+i]=u8(i*7+3);
+ const std::array<u32,1> field{4};VertexByteOrder order{16,32,field};
+ auto* first=ResolveBufferResource(1,BufferKind::kVertex,order);assert(first);
+ const auto original_bytes=first->data;
+ assert(HostDevice::device.allocations==1);
+ order.stream_offset=160;assert(ResolveBufferResource(1,BufferKind::kVertex,order)==first);
+ assert(HostDevice::device.allocations==1);
+ auto* plain=ResolveBufferResource(1,BufferKind::kVertex,{});assert(plain&&plain!=first);
+ assert(ResolveBufferResource(1,BufferKind::kVertex,order)==first);
+ assert(HostDevice::device.allocations==2);
+ memory.bytes[32+164]^=0xFF; // CPU update between draws in the same frame.
+ auto* changed=ResolveBufferResource(1,BufferKind::kVertex,order);assert(changed&&changed!=first);
+ assert(changed->data!=original_bytes&&first->data==original_bytes);
+ assert(resource->variants.size()==1&&HostDevice::held.size()==2);
+ for(u32 phase=0;phase<16;++phase){order.stream_offset=phase;assert(ResolveBufferResource(1,BufferKind::kVertex,order));}
+ assert(resource->variants.size()==8); // Evicted versions survive their fence.
+ assert(first->data==original_bytes);
+ order.stream_offset=2048;assert(!ResolveBufferResource(1,BufferKind::kVertex,order));
+ resource->kind=BufferKind::kIndex;resource->variants.clear();resource->buffer.reset();resource->format=1;
+ auto* index=ResolveBufferResource(1,BufferKind::kIndex,{});assert(index);
+ assert(index->data[0]==memory.bytes[33]&&index->data[1]==memory.bytes[32]);
+ assert(g_buffer_timing.hashed_bytes>=1024*20); // No per-frame/dirty-only skip.
+ // Two TU23 headers, as at owner+24 and owner+56. Relocate only the selected
+ // header, alternate its backing, and rewrite contents between recorded draws.
+ constexpr u32 owner=128;
+ for(u32 parity=0;parity<2;++parity){
+  auto* header=GuestBuffer(owner+24+32*parity);
+  header->resource.common=1;header->fetch_lo=4096+2048*parity;header->fetch_hi=1024;
+ }
+ std::vector<std::pair<plume::RenderBuffer*,std::vector<u8>>> recorded;
+ for(u32 frame=0;frame<32;++frame){
+  const u32 parity=frame%2,header_address=owner+24+32*parity;
+  auto* header=GuestBuffer(header_address);
+  // Phase-2 relocation updates the same header, not merely its owner address.
+  header->fetch_lo=4096+2048*parity+((frame/2)%2)*1024;
+  for(u32 i=0;i<1024;++i)memory.bytes[header->fetch_lo+i]=u8(frame*13+i);
+  auto adopted=ActualAdoptBuffer(header_address,BufferKind::kVertex);
+  assert(adopted&&adopted->mirror_address==header->fetch_lo);
+  auto* uploaded=ResolveAdoptedBuffer(header_address,BufferKind::kVertex,{});
+  assert(uploaded&&uploaded->data[0]==memory.bytes[header->fetch_lo+3]);
+  assert(ResolveAdoptedBuffer(header_address,BufferKind::kVertex,{})==uploaded);
+  recorded.emplace_back(uploaded,uploaded->data);
+  memory.bytes[header->fetch_lo]^=0x80;
+  auto* rewrite=ResolveAdoptedBuffer(header_address,BufferKind::kVertex,{});
+  assert(rewrite!=uploaded&&rewrite->data!=uploaded->data);
+  for(const auto& [old,expected]:recorded)assert(old->data==expected);
+ }
+ assert(g_buffers.size()==2);
+ GuestBuffer(owner+24)->resource.common=2;
+ assert(!ActualAdoptBuffer(owner+24,BufferKind::kVertex));
+ // A rebased draw must fetch exactly the same bytes as the original whole
+ // upload, including packed field reversal and a nonzero stream offset.
+ resource=std::make_shared<BufferResource>();resource->length=32768;resource->mirror_address=16000;
+ for(u32 i=0;i<resource->length;++i)memory.bytes[16000+i]=u8(random());
+ for(u32 n=0;n<1000;++n){
+  const u32 stride=4*(2+random()%10),stream_offset=4*(random()%8);
+  const u32 low=1+random()%100,high=low+random()%8;
+  const int32_t base=int32_t(random()%21)-10;
+  const auto window=DrawVertexWindow(low,high,base,stride,stream_offset,resource->length);
+  if(int64_t(low)+base<0){assert(!window.length);continue;}
+  assert(window.length);
+  const std::array<u32,1> packed{stride-4};
+  const auto* full=ResolveBufferResource(1,BufferKind::kVertex,{stride,stream_offset,packed});
+  auto slice=ResolveBufferResourceWindow(1,BufferKind::kVertex,window.offset,window.length,{stride,0,packed});
+  assert(slice.buffer&&slice.mirror_address==16000+window.offset&&slice.length==window.length);
+  assert(std::equal(slice.buffer->data.begin(),slice.buffer->data.end(),full->data.begin()+window.offset));
+  for(u32 index=low;index<=high;++index){
+   const u32 old_offset=stream_offset+(index+base)*stride;
+   const u32 new_offset=(index-low)*stride;
+   assert(std::equal(slice.buffer->data.begin()+new_offset,slice.buffer->data.begin()+new_offset+stride,
+                    full->data.begin()+old_offset));
+  }
+ }
+ assert(resource->windows.size()<=4096);
+ assert(resource->window_cache_bytes<=256*1024);
+ resource->windows.clear();resource->window_cache_bytes=0;
+ for(u32 i=0;i<256;++i)
+   assert(ResolveBufferResourceWindow(1,BufferKind::kVertex,i*32,32,{}).buffer);
+ const auto warmed_allocations=HostDevice::device.allocations;
+ for(u32 frame=0;frame<3;++frame)
+   for(u32 i=0;i<256;++i)
+     assert(ResolveBufferResourceWindow(1,BufferKind::kVertex,i*32,32,{}).buffer);
+ assert(HostDevice::device.allocations==warmed_allocations);
+ assert(!DrawVertexWindow(10,20,-11,16,0,32768).length);
+ assert(!DrawVertexWindow(20,10,0,16,0,32768).length);
+ assert(!DrawVertexWindow(UINT32_MAX,UINT32_MAX,INT32_MAX,UINT32_MAX,0,32768).length);
+ auto slice=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+ const auto old_slice=slice.buffer->data;
+ memory.bytes[16000+128+7]^=0x80;
+ auto edited=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+ assert(edited.buffer!=slice.buffer&&edited.buffer->data!=old_slice&&slice.buffer->data==old_slice);
+ const auto hashed_before=g_buffer_timing.hashed_bytes;
+ for(u32 i=0;i<100;++i)assert(ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{}).buffer==edited.buffer);
+ assert(g_buffer_timing.hashed_bytes-hashed_before==256*100);
+ assert(!ResolveBufferResourceWindow(1,BufferKind::kVertex,129,256,{}).buffer);
+ assert(!ResolveBufferResourceWindow(1,BufferKind::kVertex,32760,16,{}).buffer);
+}
+''')
+exe=a.output/'buffer-variants-test.exe'
+subprocess.run(['clang++','-std=c++20','-DNOMINMAX','-I'+str(root/'rexlego/src'),
+ '-I'+str(root/'rexglue-sdk/thirdparty/xxHash'),str(h),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+(a.output/'verification.json').write_text(json.dumps({'actual_buffer_resolve_body':True,'fake_driver':True,
+ 'canonical_equivalence_cases':10000,'checks':['same-phase reuse','alternating declarations','same-frame CPU update',
+ 'immutable old versions','8-version bound','invalid offset','index endian','every-use content hashing',
+ 'actual placement adoption','32 alternating owner-header frames','same-header relocation',
+ 'same-frame rewritten placement data','all recorded placement versions preserved',
+ '1000 window rebasing/conversion cases','byte/entry cache bound','256-range working set reused without allocations','immutable rewritten windows',
+ 'every-use hashing restricted to requested window','invalid window/overflow rejection'],
+ 'actual_buffer_adoption_body':True,'passed':True},indent=2))
+print('Passed 10000 fetched-byte equivalence cases and actual conversion-cache body, including same-frame writes and old-version lifetime.')
diff --git a/tools/gpu-aot/test_bulk_attachments.py b/tools/gpu-aot/test_bulk_attachments.py
new file mode 100644
index 0000000..d4fba15
--- /dev/null
+++ b/tools/gpu-aot/test_bulk_attachments.py
@@ -0,0 +1,73 @@
+"""Exercise the actual bulk attachment hook, including a clobbered guest r3."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+a.output.mkdir(parents=True, exist_ok=True)
+root = Path(__file__).resolve().parents[2]
+text = (root/'rexlego/src/gpu_native/hooks_state.cpp').read_text()
+start = text.index('REX_HOOK_RAW(sub_83FBB110)')
+pos = text.index('{', start) + 1
+end, depth = pos, 1
+while depth:
+    depth += (text[end] == '{') - (text[end] == '}')
+    end += 1
+hook = text[pos:end-1]
+code = r'''
+#include <array>
+#include <cassert>
+#include <cstdint>
+#include <mutex>
+using u8=uint8_t; using u32=uint32_t;
+struct be_u32 {u32 raw; operator u32()const{return __builtin_bswap32(raw);}
+ be_u32& operator=(u32 v){raw=__builtin_bswap32(v);return *this;}};
+struct PPCRegister {u32 u32;};
+struct PPCContext {PPCRegister r3,r4;u32 result;};
+namespace legodimensions::gpu_native {
+constexpr u32 kNativeRenderTargets=4;
+struct D3DDevice { std::array<u8,12816> padding;be_u32 render_targets[4],depth_stencil; };
+std::array<u32,5> bound;
+void BindRenderTarget(u32 i,u32 v){bound[i]=v;}
+void BindDepthStencil(u32 v){bound[4]=v;}
+struct HostDevice {static auto LockRecording(){static std::recursive_mutex m;return std::unique_lock(m);}};
+}
+std::array<u8,65536> memory{};
+std::array<u32,5> desired;
+u32 original_calls=0;
+void __imp__sub_83FBB110(PPCContext& ctx,u8* base){
+ assert(base==memory.data());assert(ctx.r3.u32==512);assert(ctx.r4.u32==256);
+ ++original_calls;
+ auto* d=reinterpret_cast<legodimensions::gpu_native::D3DDevice*>(base+ctx.r3.u32);
+ for(u32 i=0;i<4;++i)d->render_targets[i]=desired[i];
+ d->depth_stencil=desired[4];
+ ctx.r3.u32=0xDEADBEEF;ctx.result=0x1234ABCD;
+}
+#define REXLOG_INFO(...) ((void)0)
+void Hook(PPCContext& ctx,u8* base){
+''' + hook + r'''
+}
+int main(){
+ for(u32 mask=0;mask<32;++mask){
+   for(u32 i=0;i<5;++i){desired[i]=(mask&(1u<<i))?0x30200000+4096*i:0;
+     legodimensions::gpu_native::bound[i]=0xBAD00000+i;}
+   PPCContext ctx{{512},{256},0};Hook(ctx,memory.data());
+   assert(original_calls==mask+1);
+   assert(legodimensions::gpu_native::bound==desired);
+   assert(ctx.r3.u32==0xDEADBEEF && ctx.result==0x1234ABCD);
+ }
+}
+'''
+cpp=a.output/'bulk.cpp'; exe=a.output/'bulk.exe'
+cpp.write_text(code)
+subprocess.run(['clang++','-std=c++20',str(cpp),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+report={'passed':True,'null_attachment_combinations':32,'actual_hook_body':True,
+        'original_xdk_call_mocked':True,'original_called_once':True,
+        'saved_device_survives_guest_r3_clobber':True,
+        'all_native_attachments_follow_guest_shadow':True}
+(a.output/'bulk-verification.json').write_text(json.dumps(report,indent=2)+'\n')
+print(json.dumps(report,indent=2))
diff --git a/tools/gpu-aot/test_completion_callbacks.py b/tools/gpu-aot/test_completion_callbacks.py
new file mode 100644
index 0000000..f6a7fe0
--- /dev/null
+++ b/tools/gpu-aot/test_completion_callbacks.py
@@ -0,0 +1,112 @@
+"""Run actual host enqueue/poll/fence attribution against deterministic fences."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True)
+source=(root/'rexlego/src/gpu_native/device.cpp').read_text()
+def body(sig):
+ start=source.index(sig);end=source.index('{',start)+1;depth=1
+ while depth:depth+=(source[end]=='{')-(source[end]=='}');end+=1
+ return source[start:end]
+functions='\n'.join(body(sig) for sig in ['void RefreshCompletedSubmissionsLocked(',
+ 'void MarkSubmissionLocked(', 'bool HostDevice::EnqueueCompletionCallback(',
+ 'void HostDevice::PollCompletionCallbacks('])
+h=a.output/'completion-callbacks.cpp'
+h.write_text(r'''
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <thread>
+#include <vector>
+#include "gpu_native/completion_queue.h"
+using namespace legodimensions::gpu_native;
+using u32=std::uint32_t;using u64=std::uint64_t;
+enum class Backend {kD3D12,kVulkan};
+namespace plume {
+struct NativeFence {u64 completed=0;u64 GetCompletedValue(){return completed;}};
+struct D3D12CommandFence {NativeFence native;NativeFence* d3d=&native;u64 fenceValue=1;};
+}
+struct State {
+ static constexpr u32 kFramesInFlight=3;
+ Backend backend=Backend::kD3D12;
+ std::array<bool,3> frame_submitted{};
+ std::array<std::unique_ptr<plume::D3D12CommandFence>,3> frame_fences;
+ std::array<u64,3> slot_submission{};
+ u64 last_submission=0,completed_submission=0;
+ CompletionQueue completion_callbacks;
+ u64 callbacks_enqueued=0,callbacks_executed=0;
+ bool command_list_open=false;
+ State(){for(auto& f:frame_fences)f=std::make_unique<plume::D3D12CommandFence>();}
+};
+std::mutex g_mutex;std::recursive_mutex recording_mutex;std::unique_ptr<State> g_state=std::make_unique<State>();
+struct HostDevice {
+ static auto LockRecording(){return std::unique_lock(recording_mutex);}
+ static bool EnqueueCompletionCallback(std::function<void()>);
+ static void PollCompletionCallbacks();
+};
+'''+functions+r'''
+int main(){
+ std::vector<int> ran;
+ auto task=[&](int n){return [&,n]{assert(g_mutex.try_lock());g_mutex.unlock();
+  bool unlocked=false;std::thread check([&]{unlocked=recording_mutex.try_lock();if(unlocked)recording_mutex.unlock();});check.join();assert(unlocked);
+  ran.push_back(n);};};
+ auto submit=[&](u32 slot){auto& s=*g_state;RefreshCompletedSubmissionsLocked(s);
+  s.command_list_open=false;++s.frame_fences[slot]->fenceValue;MarkSubmissionLocked(s,slot);};
+ auto complete=[&](u32 slot){auto& f=*g_state->frame_fences[slot];f.native.completed=f.fenceValue-1;};
+ // No prior GPU work: callback is ready without submission/wait.
+ assert(HostDevice::EnqueueCompletionCallback(task(0)));HostDevice::PollCompletionCallbacks();
+ assert((ran==std::vector<int>{0}));
+ g_state->command_list_open=true;
+ assert(HostDevice::EnqueueCompletionCallback(task(1)));
+ assert(HostDevice::EnqueueCompletionCallback(task(2)));
+ HostDevice::PollCompletionCallbacks();assert(ran.size()==1&&g_state->last_submission==0);
+ submit(0);HostDevice::PollCompletionCallbacks();assert(ran.size()==1);
+ // Pending frame attaches callback even when no command list is open.
+ assert(HostDevice::EnqueueCompletionCallback(task(3)));
+ // D3D12 removal sentinel must not be interpreted as completed.
+ g_state->frame_fences[0]->native.completed=UINT64_MAX;
+ HostDevice::PollCompletionCallbacks();assert(ran.size()==1);
+ complete(0);HostDevice::PollCompletionCallbacks();
+ assert((ran==std::vector<int>{0,1,2,3}));
+ HostDevice::PollCompletionCallbacks();assert(ran.size()==4); // Exactly once.
+ // Newer fence completion proves every older submission on one queue done.
+ g_state->command_list_open=true;HostDevice::EnqueueCompletionCallback(task(4));submit(1);
+ g_state->command_list_open=true;HostDevice::EnqueueCompletionCallback(task(5));submit(2);
+ complete(2);HostDevice::PollCompletionCallbacks();assert(ran.back()==5&&ran.size()==6);
+ // Reentrant callback enqueue/poll must not overtake an already queued peer.
+ HostDevice::EnqueueCompletionCallback([&]{ran.push_back(6);
+  HostDevice::EnqueueCompletionCallback(task(8));HostDevice::PollCompletionCallbacks();ran.push_back(60);});
+ HostDevice::EnqueueCompletionCallback(task(7));HostDevice::PollCompletionCallbacks();
+ assert((std::vector<int>(ran.end()-4,ran.end())==std::vector<int>{6,60,7,8}));
+ // Reused ring fences must compare the newly signalled value, not the old one.
+ for(int frame=0;frame<1000;++frame){u32 slot=frame%3;
+  g_state->command_list_open=true;auto before=ran.size();HostDevice::EnqueueCompletionCallback(task(100+frame));
+  submit(slot);HostDevice::PollCompletionCallbacks();assert(ran.size()==before);
+  complete(slot);HostDevice::PollCompletionCallbacks();assert(ran.size()==before+1);}
+ assert(g_state->completion_callbacks.size()==0);
+ // Teardown discards pending guest callbacks rather than invoking on host UI thread.
+ g_state->command_list_open=true;HostDevice::EnqueueCompletionCallback(task(-1));auto count=ran.size();
+ g_state.reset();HostDevice::PollCompletionCallbacks();assert(ran.size()==count);
+ assert(!HostDevice::EnqueueCompletionCallback(task(-2)));
+}
+''')
+exe=a.output/'completion-callbacks.exe'
+subprocess.run(['clang++','-std=c++20','-DNOMINMAX','-I'+str(root/'rexlego/src'),str(h),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+assert source.count('MarkSubmissionLocked(state, slot);')==4
+hook=(root/'rexlego/src/gpu_native/hooks_device.cpp').read_text()
+insert=hook[hook.index('void InsertCallbackHook('):hook.index('}  // namespace',hook.index('void InsertCallbackHook('))]
+assert 'Synchronize(' not in insert and 'SubmitRecordedWork(' not in insert
+(a.output/'verification.json').write_text(json.dumps({'passed':True,'actual_host_functions':True,
+ 'checks':['callback waits for containing submission','pending last submission without open list',
+ 'D3D12 post-signal fenceValue - 1','device removal never completion','FIFO and exactly once',
+ 'newer queue completion covers older work','guest callback outside state AND recording mutexes','reentrant poll preserves order',
+ '1000 ring slot reuse iterations','teardown cancels guest callbacks','all four submission paths tagged',
+ 'InsertCallback neither submits nor waits'],'real_GPU_gameplay_not_proven':True},indent=2))
+print('Passed actual enqueue/poll: fence generations, FIFO, removal, reentrancy, ring reuse and shutdown.')
diff --git a/tools/gpu-aot/test_cpu_resource_wait.py b/tools/gpu-aot/test_cpu_resource_wait.py
new file mode 100644
index 0000000..665c223
--- /dev/null
+++ b/tools/gpu-aot/test_cpu_resource_wait.py
@@ -0,0 +1,80 @@
+"""Check actual resource-wait hook exclusions and completion callback ordering."""
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True)
+def body(path,sig):
+ text=(root/path).read_text();start=text.index(sig);end=text.index('{',start)+1;depth=1
+ while depth:depth+=(text[end]=='{')-(text[end]=='}');end+=1
+ return text[start:end]
+texture=body('rexlego/src/gpu_native/textures.cpp','bool IsCpuUploadedTexture(')
+hooks='\n'.join(body('rexlego/src/gpu_native/hooks_device.cpp',x) for x in [
+ 'void BlockUntilNotBusyHook(', 'void BlockUntilIdleHook(', 'void BlockOnFenceHook(', 'void InsertCallbackHook('])
+h=a.output/'cpu-resource-wait.cpp';h.write_text(r'''
+#include <cassert>
+#include <cstdlib>
+#include <memory>
+#include <mutex>
+#include <functional>
+#include <vector>
+#include <unordered_set>
+using u32=unsigned;
+#define REXLOG_ERROR(...) ((void)0)
+#define REXLOG_WARN(...) ((void)0)
+#define REXLOG_INFO(...) ((void)0)
+struct D3DDevice{};
+enum class SyncReason { kOther, kIdle, kFence, kResource, kCallback };
+struct HostDevice {
+ static inline std::recursive_mutex lock;
+ static inline u32 syncs=0,skips=0,callbacks=0;
+ static inline bool ready=true;
+ static inline bool completed=false;
+ static inline std::vector<std::function<void()>> pending;
+ static auto LockRecording(){return std::unique_lock(lock);}
+ static bool Synchronize(SyncReason = SyncReason::kOther){++syncs;return ready;}
+ static void RecordCpuResourceWaitSkip(){++skips;}
+ static bool EnqueueCompletionCallback(std::function<void()> fn){if(!ready)return false;pending.push_back(fn);return true;}
+ static void PollCompletionCallbacks(){if(!completed)return;auto tasks=std::move(pending);pending.clear();for(auto& fn:tasks)fn();}
+};
+struct Texture {std::mutex mutex;bool surface=false,resolved_on_host=false,owns_guest_memory=false;};
+std::shared_ptr<Texture> texture=std::make_shared<Texture>();
+auto FindTexture(u32 address){return address==3?texture:nullptr;}
+bool IsNativeBuffer(u32 address){return address==1;}
+bool IsNativeShader(u32 address){return address==2;}
+void Callback(u32 data){assert(HostDevice::syncs==0&&data==77);++HostDevice::callbacks;}
+struct Dispatcher {auto GetFunction(u32 addr){return addr==9?&Callback:nullptr;}};
+struct Kernel {Dispatcher d;auto function_dispatcher(){return &d;}};
+namespace rex {namespace system {auto kernel_state(){static Kernel k;return &k;}}
+ namespace ppc {template<class T> void GuestToHostFunction(void(*fn)(u32),u32 data){fn(data);}}}
+'''+texture+'\n'+hooks+r'''
+int main(){
+ const bool enabled=std::getenv("LEGO_NATIVE_ASYNC_CPU_RESOURCES")!=nullptr;
+ auto check=[&](u32 resource,bool safe){HostDevice::syncs=HostDevice::skips=0;
+  BlockUntilNotBusyHook(resource);assert(HostDevice::skips==u32(enabled&&safe));
+  assert(HostDevice::syncs==u32(!enabled||!safe));};
+ check(0,false);check(100,false);check(1,true);check(2,true);
+ for(u32 bits=0;bits<8;++bits){texture->surface=bits&1;texture->resolved_on_host=bits&2;texture->owns_guest_memory=bits&4;
+  check(3,bits==0);}
+ HostDevice::syncs=0;BlockUntilIdleHook(nullptr);assert(HostDevice::syncs==1);
+ HostDevice::syncs=0;BlockOnFenceHook(1);assert(HostDevice::syncs==1);
+ HostDevice::syncs=0;InsertCallbackHook(nullptr,0,9,77);assert(HostDevice::syncs==0&&HostDevice::callbacks==0);
+ HostDevice::completed=true;HostDevice::PollCompletionCallbacks();assert(HostDevice::callbacks==1);
+ HostDevice::ready=false;HostDevice::syncs=0;InsertCallbackHook(nullptr,0,9,77);
+ assert(HostDevice::syncs==0&&HostDevice::callbacks==1);
+}
+''')
+exe=a.output/'cpu-resource-wait.exe'
+subprocess.run(['clang++','-std=c++20',str(h),'-o',str(exe)],check=True)
+for enabled in (False,True):
+ env=os.environ.copy();env.pop('LEGO_NATIVE_ASYNC_CPU_RESOURCES',None)
+ if enabled:env['LEGO_NATIVE_ASYNC_CPU_RESOURCES']='1'
+ subprocess.run([str(exe.resolve())],check=True,env=env)
+(a.output/'verification.json').write_text(json.dumps({'actual_native_hook_bodies':True,
+ 'opt_in_and_default_tested':True,'checks':['known CPU buffers/shaders','borrowed CPU textures',
+ 'all surface/resolved/owned texture exclusions','unknown resource fallback','explicit idle and fence waits',
+ 'GPU completion before guest callback','unavailable device does not call guest'],'passed':True,
+ 'real_GPU_gameplay_not_proven':True},indent=2))
+print('Passed actual wait hooks: conservative default, CPU-only opt-in, exclusions and deferred callback ordering.')
diff --git a/tools/gpu-aot/test_depth_alias.py b/tools/gpu-aot/test_depth_alias.py
new file mode 100644
index 0000000..88c0d83
--- /dev/null
+++ b/tools/gpu-aot/test_depth_alias.py
@@ -0,0 +1,100 @@
+"""Check D24 numeric/bank mapping and execute the actual native alias tracker."""
+import argparse,json,subprocess
+from pathlib import Path
+p=argparse.ArgumentParser(description=__doc__)
+p.add_argument('output',type=Path)
+a=p.parse_args();a.output.mkdir(parents=True,exist_ok=True)
+root=Path(__file__).resolve().parents[2]
+source=(root/'rexlego/src/gpu_native/textures.cpp').read_text()
+def function(name):
+    start=source.index(name+'(')
+    start=source.rfind('\n',0,start)+1
+    end=source.index('{',start)+1;depth=1
+    while depth:
+        depth+=(source[end]=='{')-(source[end]=='}');end+=1
+    return source[start:end]
+code=r'''
+#include "gpu_native/depth_alias.h"
+#include <cassert>
+#include <cmath>
+#include <memory>
+#include <unordered_map>
+using u32=uint32_t;using u64=uint64_t;
+namespace plume {enum class RenderFormat {R8G8B8A8_UNORM,OTHER};}
+namespace legodimensions::gpu_native {
+struct TextureResource {
+  bool surface=true;u32 edram_base=1328,guest_msaa=0,guest_format=22,guest_address=1;
+  u32 width=1280,height=720,descriptor_index=7;u64 write_generation=0;
+  plume::RenderFormat format=plume::RenderFormat::OTHER;
+  std::unique_ptr<int> texture=std::make_unique<int>(42);
+  u32 HostWidth()const{return width;}u32 HostHeight()const{return height;}
+};
+u64 g_depth_alias_generation=0;
+std::unordered_map<u32,std::weak_ptr<TextureResource>> g_edram_colors;
+std::unordered_map<u32,std::shared_ptr<TextureResource>> resources;
+bool enabled=true;
+bool DepthAliasesEnabled(){return enabled;}
+auto FindTexture(u32 address){auto f=resources.find(address);return f==resources.end()?nullptr:f->second;}
+u32 transfers=0;bool transfer_ok=true;
+struct HostDevice {
+static bool TransferDepthAlias(int* src,u32 descriptor,int* dst,u32 w,u32 h,bool restore){
+assert(src&&dst&&src!=dst&&descriptor==7&&w==1280&&h==720&&restore);
+++transfers;return transfer_ok;}
+};
+#define REXLOG_INFO(...) ((void)0)
+''' + function('MarkSurfaceWritten')+'\n'+function('PrepareSurfaceDepthAlias')+r'''
+}
+int main(){
+ using namespace legodimensions::gpu_native;
+ for(u32 z=0;z<0x1000000;++z) {
+   const float actual=UnpackDepth24(z);
+   assert(std::abs(double(actual)-double(z)/double(0xFFFFFF))<=0.5/16777215.0);
+   if(z) assert(actual>UnpackDepth24(z-1));
+ }
+ assert(UnpackDepth24(0)==0 && UnpackDepth24(0xFFFFFF)==1);
+ assert(UnpackDepth24(0xC00000)==0.750000059604644775390625f);
+ for(u32 y=0;y<720;++y) for(u32 x=0;x<1280;++x){
+   const u32 alias=DepthColorAliasX(x);
+   assert(alias<1280&&DepthColorAliasX(alias)==x);
+   const u32 tile=(y/16)*16+x/80;
+   const u32 depth_address=tile*1280+(y%16)*80+((x%80+40)%80);
+   const u32 color_address=(y/16*16+alias/80)*1280+(y%16)*80+alias%80;
+   assert(depth_address==color_address);
+ }
+ DepthAliasLayout layout{0x1E4D8000,1280,720,40,1,2,1,1};
+ assert(CompatibleDepthAlias(layout,layout));
+ constexpr u32 DepthAliasLayout::* fields[]={&DepthAliasLayout::physical_base,
+   &DepthAliasLayout::width,&DepthAliasLayout::height,&DepthAliasLayout::pitch,
+   &DepthAliasLayout::tiled,&DepthAliasLayout::endian,&DepthAliasLayout::levels,
+   &DepthAliasLayout::dimension};
+ for(auto field:fields){auto different=layout;
+   different.*field^=1;assert(!CompatibleDepthAlias(layout,different));}
+ auto color=std::make_shared<TextureResource>();color->guest_address=2;
+ color->format=plume::RenderFormat::R8G8B8A8_UNORM;color->guest_format=6;
+ auto depth=std::make_shared<TextureResource>();
+ resources[1]=depth;resources[2]=color;
+ assert(PrepareSurfaceDepthAlias(1)&&transfers==0);
+ MarkSurfaceWritten(2);assert(PrepareSurfaceDepthAlias(1)&&transfers==1);
+ assert(PrepareSurfaceDepthAlias(1)&&transfers==1); // unchanged source
+ MarkSurfaceWritten(1);assert(PrepareSurfaceDepthAlias(1)&&transfers==1); // own depth write newer
+ MarkSurfaceWritten(2);transfer_ok=false;
+ assert(!PrepareSurfaceDepthAlias(1)&&transfers==2);
+ transfer_ok=true;assert(PrepareSurfaceDepthAlias(1)&&transfers==3); // failed copy never committed
+ MarkSurfaceWritten(2);depth->guest_msaa=2;
+ assert(PrepareSurfaceDepthAlias(1)&&transfers==3);depth->guest_msaa=0;
+ depth->width=640;assert(PrepareSurfaceDepthAlias(1)&&transfers==3);depth->width=1280;
+ depth->edram_base=1536;assert(PrepareSurfaceDepthAlias(1)&&transfers==3);depth->edram_base=1328;
+ depth->guest_format=23;assert(PrepareSurfaceDepthAlias(1)&&transfers==3);depth->guest_format=22;
+ enabled=false;assert(PrepareSurfaceDepthAlias(1)&&transfers==3);enabled=true;
+ resources.erase(2);color.reset();assert(PrepareSurfaceDepthAlias(1)&&transfers==3);
+}
+'''
+cpp=a.output/'depth-alias.cpp';exe=a.output/'depth-alias.exe'
+cpp.write_text(code)
+subprocess.run(['clang++','-std=c++20','-O2','-I',str(root/'rexlego/src'),str(cpp),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+report={'passed':True,'unorm24_values':16777216,'edram_coordinates':1280*720,
+ 'actual_tracker_bodies':True,'layout_rejection':True,'write_order_and_failed_retry':True,
+ 'expired_source_and_disabled_path':True,'gpu_shader_execution':'runtime capture required'}
+(a.output/'verification.json').write_text(json.dumps(report,indent=2)+'\n')
+print(json.dumps(report))
diff --git a/tools/gpu-aot/test_depth_export.py b/tools/gpu-aot/test_depth_export.py
new file mode 100644
index 0000000..bd0f0de
--- /dev/null
+++ b/tools/gpu-aot/test_depth_export.py
@@ -0,0 +1,72 @@
+"""End-to-end XenosRecomp depth-signature regression using synthetic microcode.
+
+No game assets required. Fixtures and generated outputs go in the supplied
+scratch directory, never in the shipping shader corpus/cache.
+"""
+import argparse
+import os
+from pathlib import Path
+import re
+import struct
+import subprocess
+
+
+def fixture(*, depth=False, reflection_depth=False, predicated=False,
+            scalar=False, constant_one=False):
+    data = bytearray(152)
+    struct.pack_into(">9I", data, 0, 0x102A1100, 128, 24, 0, 36, 0, 80, 0, 0)
+    struct.pack_into(">8I", data, 36, 32, 28, 0, 0, 0, 0, 0, 0)
+    struct.pack_into(">8I", data, 80, 0, 24, 0, 0xFF00, 0, 0, 0,
+                     0x10 if reflection_depth else 1)
+    # One EXEC_END at CF0 executes the ALU at instruction slot 1; CF1 is NOP.
+    struct.pack_into(">3I", data, 128, 1 | (1 << 12), 2 << 12, 0)
+    vector_mask = 0 if scalar else (1 if depth else 15)
+    scalar_mask = 1 if scalar or constant_one else 0
+    alu0 = (61 if depth else 0) | (1 << 15)
+    alu0 |= (vector_mask << 16) | (scalar_mask << 20)
+    alu1 = ((1 << 28) | (1 << 27)) if predicated else 0
+    # MAX r1, r1; scalar ADD r1, r1. Register operands, no texture/constants.
+    alu2 = 1 | (1 << 8) | (1 << 16) | (2 << 24) | (7 << 29)
+    struct.pack_into(">3I", data, 140, alu0, alu1, alu2)
+    return data
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("compiler", type=Path)
+    parser.add_argument("common", type=Path)
+    parser.add_argument("scratch", type=Path)
+    args = parser.parse_args()
+    inputs = args.scratch / "input"
+    hlsl_dir = args.scratch / "hlsl"
+    inputs.mkdir(parents=True, exist_ok=True)
+    cases = {
+        "color_only": {},
+        "reflection_only": {"reflection_depth": True},
+        "depth_vector": {"depth": True},
+        "depth_scalar": {"depth": True, "scalar": True},
+        "depth_one": {"depth": True, "constant_one": True},
+        "depth_predicated": {"depth": True, "predicated": True},
+    }
+    for name, options in cases.items():
+        (inputs / (name + ".bin")).write_bytes(fixture(**options))
+    env = os.environ.copy()
+    env["XENOS_RECOMP_DXIL_ONLY"] = "1"
+    env.pop("XENOS_RECOMP_ONLY_HASH", None)
+    subprocess.run([str(args.compiler.resolve()), str(inputs),
+                    str(args.scratch / "test-cache.cpp"), str(args.common),
+                    str(hlsl_dir)], env=env, check=True)
+    for name, options in cases.items():
+        source = (hlsl_dir / (name + ".hlsl")).read_text()
+        has_depth = options.get("depth", False)
+        assert bool(re.search(r"\boDepth\s*:\s*SV_Depth", source)) == has_depth, name
+        assert ("oDepth = 0.0;" in source) == has_depth, name
+        if has_depth:
+            assert "oDepth.x =" in source, name
+        else:
+            assert "oDepth" not in source, name
+    print(f"Depth export regression passed: {len(cases)} HLSL/DXIL fixtures")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/gpu-aot/test_descriptor_retirement.cpp b/tools/gpu-aot/test_descriptor_retirement.cpp
new file mode 100644
index 0000000..ad80196
--- /dev/null
+++ b/tools/gpu-aot/test_descriptor_retirement.cpp
@@ -0,0 +1,26 @@
+#include "../../rexlego/src/gpu_native/descriptor_retirement.h"
+#include <cassert>
+#include <vector>
+
+using legodimensions::gpu_native::DescriptorRetirement;
+int main() {
+  DescriptorRetirement retirement(8);
+  std::vector<uint32_t> released;
+  const auto free = [&](uint32_t slot) { released.push_back(slot); };
+  assert(retirement.Retire(3, 0)); // No GPU references: immediate reuse.
+  assert(!retirement.Retire(4, 0b111));
+  assert(!retirement.Retire(4, 0b111)); // Repeated release adds no tickets.
+  retirement.CompleteFrame(2, free);
+  retirement.CompleteFrame(0, free);
+  assert(released.empty()); // One older list is still using the descriptor.
+  retirement.CompleteFrame(1, free);
+  assert((released == std::vector<uint32_t>{4}));
+  retirement.CompleteFrame(1, free);
+  assert(released.size() == 1);
+  assert(!retirement.Retire(4, 0b001)); // Reuse followed by another retirement.
+  assert(!retirement.Retire(5, 0b010));
+  retirement.CompleteFrame(0, free);
+  assert(released.back() == 4 && released.size() == 2);
+  retirement.CompleteFrame(1, free);
+  assert(released.back() == 5 && released.size() == 3);
+}
diff --git a/tools/gpu-aot/test_draw_upload.py b/tools/gpu-aot/test_draw_upload.py
new file mode 100644
index 0000000..6aaedcd
--- /dev/null
+++ b/tools/gpu-aot/test_draw_upload.py
@@ -0,0 +1,113 @@
+"""Exercise actual upload allocation and fence-slot reset with a fake driver."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True)
+source=(root/'rexlego/src/gpu_native/device.cpp').read_text()
+def function(signature):
+ start=source.index(signature);pos=source.index('{',start);depth=1;end=pos+1
+ while depth:
+  depth+=(source[end]=='{')-(source[end]=='}');end+=1
+ return source[start:end]
+begin=function('plume::RenderCommandList* BeginFrameCommandsLocked')
+allocate=function('DrawUploadSlice HostDevice::AllocateDrawUpload')
+page=source[source.index('struct DrawUploadPage {'):source.index('const std::filesystem::path& SnapshotDirectory')]
+h=a.output/'draw-upload-test.cpp'
+h.write_text(r'''
+#include <array>
+#include <algorithm>
+#include <chrono>
+#include <cassert>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <vector>
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;
+#define REXLOG_WARN(...) ((void)0)
+namespace plume {
+struct RenderBufferDesc { u64 size; static auto UploadBuffer(u64 n) {return RenderBufferDesc{n};} };
+struct RenderBuffer {
+ std::vector<u8> bytes; explicit RenderBuffer(u64 n):bytes(n){}
+ void* map(){return bytes.data();}void unmap(){}
+};
+struct RenderDevice {
+ u32 allocations=0;bool fail=false;
+ auto createBuffer(RenderBufferDesc d)->std::unique_ptr<RenderBuffer> {
+  if(fail)return {}; ++allocations;return std::make_unique<RenderBuffer>(d.size);
+ }
+};
+struct RenderCommandList {u32 begins=0;void begin(){++begins;} };
+struct Fence {};
+struct Queue {u32 waits=0;void waitForCommandFence(Fence*){++waits;} };
+struct Swap {bool needsResize(){return false;}};
+}
+struct DrawUploadSlice {
+ plume::RenderBuffer* buffer=nullptr;void* mapped=nullptr;u64 offset=0;
+ explicit operator bool()const{return buffer&&mapped;}
+};
+'''+page+r'''
+struct DescriptorRetirement { void CompleteFrame(u32,auto callback){} };
+enum class Backend {kD3D12,kVulkan};
+struct State {
+ static constexpr u32 kFramesInFlight=3;
+ std::unique_ptr<plume::RenderDevice> device=std::make_unique<plume::RenderDevice>();
+ std::unique_ptr<plume::Queue> queue=std::make_unique<plume::Queue>();
+ std::unique_ptr<plume::Swap> swap_chain=std::make_unique<plume::Swap>();
+ std::vector<int> framebuffers{1,2,3};
+ std::array<std::unique_ptr<plume::RenderCommandList>,3> command_lists;
+ std::array<std::unique_ptr<plume::Fence>,3> frame_fences;
+ std::array<bool,3> frame_submitted{};
+ Backend backend=Backend::kVulkan;u64 completed_submission=0;std::array<u64,3> slot_submission{};
+ std::array<std::vector<int>,3> retired_resources,snapshots;
+ std::array<std::vector<std::unique_ptr<DrawUploadPage>>,3> draw_uploads;
+ u32 frame_slot=0,draw_upload_page=0,long_wait_log_count=0;
+ u64 draw_upload_offset=0;bool command_list_open=false;
+ DescriptorRetirement retired_descriptors;std::array<bool,2> texture_slots{};
+ State(){for(u32 s=0;s<3;++s){command_lists[s]=std::make_unique<plume::RenderCommandList>();frame_fences[s]=std::make_unique<plume::Fence>();}}
+};
+void SaveSnapshots(std::vector<int>&){}
+void RefreshCompletedSubmissionsLocked(State&){}
+void InitializeNullTextures(State&,plume::RenderCommandList*){}
+bool RebuildSwapChain(State&){return true;}
+std::unique_ptr<State> g_state=std::make_unique<State>();std::mutex g_mutex;
+struct HostDevice {
+ static auto LockRecording(){static std::recursive_mutex m;return std::unique_lock<std::recursive_mutex>(m);}
+ static DrawUploadSlice AllocateDrawUpload(u32 size);
+};
+'''+begin+'\n'+allocate+r'''
+int main() {
+ assert(!HostDevice::AllocateDrawUpload(0));
+ auto first=HostDevice::AllocateDrawUpload(8816);assert(first&&first.offset==0);
+ std::memset(first.mapped,0xA5,8816);
+ auto second=HostDevice::AllocateDrawUpload(8816);assert(second.offset==8960&&second.buffer==first.buffer);
+ std::memset(second.mapped,0x5A,8816);
+ for(int i=0;i<550;++i){auto s=HostDevice::AllocateDrawUpload(8816);assert(s&&s.offset%256==0);std::memset(s.mapped,i&255,8816);}
+ assert(g_state->device->allocations==2);assert(g_state->queue->waits==0);
+ for(int i=0;i<8816;++i) assert(static_cast<u8*>(first.mapped)[i]==0xA5);
+ g_state->command_list_open=false;g_state->frame_submitted[0]=true;g_state->frame_slot=1;
+ auto other=HostDevice::AllocateDrawUpload(8816);assert(other&&other.buffer!=first.buffer);
+ std::memset(other.mapped,0xCC,8816);assert(g_state->queue->waits==0);
+ assert(static_cast<u8*>(first.mapped)[0]==0xA5);
+ g_state->command_list_open=false;g_state->frame_submitted[1]=true;g_state->frame_slot=0;
+ auto reuse=HostDevice::AllocateDrawUpload(8816);assert(reuse.buffer==first.buffer&&reuse.offset==0);
+ assert(g_state->queue->waits==1); // Reset only after old fence wait.
+ std::memset(reuse.mapped,0xDD,8816);assert(static_cast<u8*>(other.mapped)[0]==0xCC);
+ auto large=HostDevice::AllocateDrawUpload(5*1024*1024);assert(large&&large.offset==0);
+ assert(g_state->device->allocations==4);
+ auto tail=HostDevice::AllocateDrawUpload(1);assert(tail&&tail.offset==0);
+ assert(g_state->device->allocations==5);
+ g_state->device->fail=true;assert(!HostDevice::AllocateDrawUpload(8*1024*1024));
+ g_state.reset();assert(!HostDevice::AllocateDrawUpload(8816));
+}
+''')
+exe=a.output/'draw-upload-test.exe'
+subprocess.run(['clang++','-std=c++20','-DNOMINMAX',str(h),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+(a.output/'verification.json').write_text(json.dumps({'actual_production_functions':True,'fake_driver':True,
+ 'checks':['256-byte alignment','550 immutable draws','page growth','independent slots','reset after fence wait','oversize','allocation failure','shutdown'],
+ 'passed':True},indent=2))
+print('Passed actual allocation/reset bodies: immutable draws, independent fence slots, alignment, growth and failure.')
diff --git a/tools/gpu-aot/test_hash_throughput.cpp b/tools/gpu-aot/test_hash_throughput.cpp
new file mode 100644
index 0000000..fb96cf5
--- /dev/null
+++ b/tools/gpu-aot/test_hash_throughput.cpp
@@ -0,0 +1,31 @@
+// Compare the installed SDK library with the same xxHash header implementation.
+#include <xxhash.h>
+#include <chrono>
+#include <cstdint>
+#include <iostream>
+#include <vector>
+
+static uint64_t LibraryHash(const void* p, size_t n) { return XXH3_64bits(p, n); }
+#define XXH_INLINE_ALL
+#include <xxhash.h>
+
+int main() {
+  std::vector<uint8_t> data(16 * 1024 * 1024);
+  for (size_t i = 0; i < data.size(); ++i) data[i] = uint8_t(i * 31);
+  uint64_t result = 0;
+  for (bool inlined : {false, true}) {
+    const auto start = std::chrono::steady_clock::now();
+    for (int i = 0; i < 32; ++i) {
+      data[0] = uint8_t(i);
+      const auto a = inlined ? XXH3_64bits(data.data(), data.size())
+                             : LibraryHash(data.data(), data.size());
+      result ^= a;
+    }
+    std::cout << (inlined ? "header" : "SDK library") << ": "
+              << std::chrono::duration<double, std::milli>(
+                     std::chrono::steady_clock::now() - start).count()
+              << " ms for 512 MiB, checksum=" << result << '\n';
+  }
+  // The checksum cancels when all results match.
+  return result != 0;
+}
diff --git a/tools/gpu-aot/test_inline_hash.py b/tools/gpu-aot/test_inline_hash.py
new file mode 100644
index 0000000..c9367b2
--- /dev/null
+++ b/tools/gpu-aot/test_inline_hash.py
@@ -0,0 +1,53 @@
+"""Compare inline AVX2 XXH3 with SSE2 library hashes and benchmark both."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path)
+p.add_argument('--sdk-library',type=Path);a=p.parse_args()
+root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True)
+include=root/'rexglue-sdk/thirdparty/xxHash'
+c=a.output/'reference.c';c.write_text('''#include <xxhash.h>
+unsigned long long ReferenceHash(const void* p,size_t n){return XXH3_64bits(p,n);}
+unsigned long long ReferenceSeed(const void* p,size_t n,unsigned long long s){return XXH3_64bits_withSeed(p,n,s);}
+''')
+h=a.output/'inline-hash.cpp';h.write_text(r'''
+#include <algorithm>
+#include <cassert>
+#include <chrono>
+#include <cstdio>
+#include <random>
+#include <vector>
+#define XXH_INLINE_ALL
+#include <xxhash.h>
+extern "C" unsigned long long ReferenceHash(const void*,size_t);
+extern "C" unsigned long long ReferenceSeed(const void*,size_t,unsigned long long);
+int main(){
+ std::mt19937 random(991);std::vector<unsigned char> bytes(8*1024*1024+64);
+ for(auto& b:bytes)b=random();
+ for(size_t i=0;i<5000;++i){
+  size_t size=i<300?i:random()%(8*1024*1024);size_t offset=random()%64;auto seed=(uint64_t(random())<<32)|random();
+  assert(XXH3_64bits(bytes.data()+offset,size)==ReferenceHash(bytes.data()+offset,size));
+  assert(XXH3_64bits_withSeed(bytes.data()+offset,size,seed)==ReferenceSeed(bytes.data()+offset,size,seed));
+ }
+ volatile uint64_t sink=0;
+ auto bench=[&](bool ref){auto start=std::chrono::steady_clock::now();
+  for(size_t i=0;i<2048;++i){const auto* p=bytes.data()+(i%8)*1024*1024;
+   sink=ref?ReferenceHash(p,1024*1024):XXH3_64bits(p,1024*1024);}
+  return std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count();};
+ bench(false);bench(true);
+ double ref=bench(true),inl=bench(false);
+ printf("{\"cases\":5000,\"hashed_bytes_per_benchmark\":2147483648,\"sse2_ms\":%.3f,\"avx2_ms\":%.3f,\"ratio\":%.3f,\"identical\":true}\n",ref,inl,ref/inl);
+}
+''')
+obj=a.output/'reference.obj'
+subprocess.run(['clang','-O2','-DXXH_VECTOR=XXH_SSE2','-I'+str(include),'-c',str(c),'-o',str(obj)],check=True)
+hashobj=a.sdk_library or a.output/'xxhash.obj'
+if not a.sdk_library:
+ subprocess.run(['clang','-O2','-DXXH_VECTOR=XXH_SSE2','-I'+str(include),'-c',str(include/'xxhash.c'),'-o',str(hashobj)],check=True)
+exe=a.output/'inline-hash.exe'
+subprocess.run(['clang++','-std=c++20','-O2','-march=x86-64-v3','-I'+str(include),str(h),str(obj),str(hashobj),'-o',str(exe)],check=True)
+result=subprocess.run([str(exe.resolve())],check=True,text=True,capture_output=True)
+proof=json.loads(result.stdout);proof['reference_library']=str(hashobj.resolve())
+if a.sdk_library:proof['sdk_reference_ms']=proof.pop('sse2_ms')
+(a.output/'verification.json').write_text(json.dumps(proof,indent=2));print(json.dumps(proof,indent=2))
diff --git a/tools/gpu-aot/test_long_probe.py b/tools/gpu-aot/test_long_probe.py
new file mode 100644
index 0000000..3b3546f
--- /dev/null
+++ b/tools/gpu-aot/test_long_probe.py
@@ -0,0 +1,58 @@
+"""Check opt-in tracing across translation units and concurrent producers."""
+import argparse
+import csv
+import json
+import os
+from pathlib import Path
+import subprocess
+
+parser=argparse.ArgumentParser(description=__doc__)
+parser.add_argument('output',type=Path)
+args=parser.parse_args()
+root=Path(__file__).resolve().parents[2]
+args.output.mkdir(parents=True,exist_ok=True)
+header=(root/'rexlego/src/gpu_native/long_probe.h').as_posix()
+a=args.output/'producer.cpp'
+b=args.output/'consumer.cpp'
+a.write_text(f'#include "{header}"\nvoid Produce() {{ legodimensions::gpu_native::LongProbeEvent("producer",false,42); }}\n')
+b.write_text(f'#include "{header}"\n'+r'''
+#include <thread>
+#include <cassert>
+void Produce();
+int main() {
+  using namespace legodimensions::gpu_native;
+  if(!LongProbeEnabled()) { LongProbeEvent("disabled",true,1); assert(!g_probe_capture_requested.load()); return 0; }
+  g_probe_frame.store(123);
+  std::thread first([] { for(int i=0;i<100;i++) Produce(); });
+  std::thread second([] { for(int i=0;i<100;i++) LongProbeEvent("consumer",false,i); });
+  first.join(); second.join(); LongProbeEvent("candidate",true,3);
+  assert(g_probe_capture_requested.exchange(false));
+  const uint8_t bytes[]{1,2,3,4};
+  const auto name=SaveLongProbeConstants(bytes);
+  LongProbeEvent("frame",false,"file=",name);
+  for(int i=0;i<130;i++) SaveLongProbeConstants(bytes);
+  g_probe_frame.store(124);
+  assert(!SaveLongProbeConstants(bytes).empty());
+  LongProbeEvent("frame",false,"budget_reset=1");
+}
+''')
+exe=args.output/'trace-test.exe'
+subprocess.run(['clang++','-std=c++20',str(a),str(b),'-o',str(exe)],check=True)
+env=dict(os.environ); env.pop('LEGO_NATIVE_TRACE_DIR',None)
+subprocess.run([str(exe.resolve())],env=env,check=True)
+destination=args.output/'enabled'
+assert not destination.exists(), 'Use a fresh output to preserve evidence'
+env['LEGO_NATIVE_TRACE_DIR']=str(destination.resolve())
+subprocess.run([str(exe.resolve())],env=env,check=True)
+with (destination/'events.tsv').open() as file: rows=list(csv.DictReader(file,delimiter='\t'))
+assert len(rows)==204
+assert sum(r['event']=='producer' for r in rows)==100
+assert sum(r['event']=='consumer' for r in rows)==100
+assert sum(r['anomaly_candidate']=='1' for r in rows)==1
+assert sum(r['frame']=='123' for r in rows)==203
+assert sum(r['event']=='constants_frame_budget' for r in rows)==1
+assert len(list(destination.glob('constants-f123-*.bin')))==128
+assert len(list(destination.glob('constants-f124-*.bin')))==1
+assert next(destination.glob('constants-*.bin')).read_bytes()==bytes([1,2,3,4])
+assert json.loads((destination/'trace-start.json').read_text())['unix_ms']>0
+print('Passed: disabled mode, shared writer across TUs, 200 concurrent events, anomaly request, binary payload, 128-file frame budget and reset')
diff --git a/tools/gpu-aot/test_memory_watch.py b/tools/gpu-aot/test_memory_watch.py
new file mode 100644
index 0000000..a7d1ee4
--- /dev/null
+++ b/tools/gpu-aot/test_memory_watch.py
@@ -0,0 +1,129 @@
+"""Compile the production watcher against a bounded SDK memory callback model."""
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+
+p=argparse.ArgumentParser(description=__doc__); p.add_argument('output',type=Path); a=p.parse_args()
+root=Path(__file__).resolve().parents[2]; a.output.mkdir(parents=True,exist_ok=True)
+stub=a.output/'include/rex/system'; stub.mkdir(parents=True,exist_ok=True)
+(stub.parent/'cvar.h').write_text(r'''
+#pragma once
+namespace rex::cvar {
+enum class Lifecycle { kRequiresRestart };
+struct FixtureCvar { FixtureCvar lifecycle(Lifecycle) { return *this; } };
+}
+#define REXCVAR_DEFINE_BOOL(...) static auto fixture_cvar = rex::cvar::FixtureCvar{}
+#define REXCVAR_GET(...) false
+''')
+(stub/'xmemory.h').write_text(r'''
+#pragma once
+#include <cstdint>
+#include <mutex>
+#include <utility>
+#include <unordered_set>
+#include <cassert>
+namespace rex::thread {
+struct global_critical_region {
+  static inline std::recursive_mutex mutex;
+  auto Acquire() { return std::unique_lock<std::recursive_mutex>(mutex); }
+}; }
+namespace rex::memory {
+constexpr uint32_t kMemoryAllocationCommit=1,kMemoryProtectRead=1;
+struct HeapAllocationInfo { uint32_t state=1,protect=1,region_size=4096; };
+struct Heap {
+ bool QueryRegionInfo(uint32_t at,HeapAllocationInfo* info) {
+   if(at>=0x100000) return false; *info={}; return true;
+ }
+ uint32_t heap_base() { return 0; } uint32_t page_size() { return 4096; }
+};
+struct Memory {
+ using Callback=std::pair<uint32_t,uint32_t>(*)(void*,uint32_t,uint32_t,bool);
+ Callback callback=nullptr; void* context=nullptr; Heap heap;
+ std::unordered_set<uint32_t> armed;
+ uint32_t GetPhysicalAddress(uint32_t at) {
+   if(at>=0xE0000000 && at<0xE0100000) return (at&0x1FFFFFFF)+0x1000;
+   if((at>=0xA0000000&&at<0xA0100000)||(at>=0xC0000000&&at<0xC0100000)) return at&0x1FFFFFFF;
+   return UINT32_MAX;
+ }
+ Heap* GetPhysicalHeap() { return &heap; }
+ void* RegisterPhysicalMemoryInvalidationCallback(Callback c,void* ptr) { assert(!callback);callback=c;context=ptr;return this; }
+ void UnregisterPhysicalMemoryInvalidationCallback(void*) { callback=nullptr; }
+ void EnablePhysicalMemoryAccessCallbacks(uint32_t at,uint32_t size,bool notify,bool data) {
+   assert(notify&&!data);
+   for(auto page=at/4096;page<=(at+size-1)/4096;++page) armed.insert(page);
+ }
+ void Write(uint32_t alias) {
+   auto lock=rex::thread::global_critical_region().Acquire();
+   uint32_t phys=GetPhysicalAddress(alias),page=phys/4096;
+   if(!armed.contains(page)||!callback)return;
+   auto [start,size]=callback(context,page*4096,4096,false);
+   assert(start==page*4096&&size==4096);
+   armed.erase(page);
+ }
+};
+inline Memory memory;
+}
+''')
+(stub/'kernel_state.h').write_text('#pragma once\n#include "xmemory.h"\n#define REX_KERNEL_MEMORY() (&rex::memory::memory)\n')
+h=a.output/'watch-test.cpp'
+h.write_text(r'''
+#include "gpu_native/memory_watch.h"
+#include <rex/system/xmemory.h>
+#include <cassert>
+#include <thread>
+using namespace legodimensions::gpu_native;
+CpuMemoryStamp Stamp(uint32_t at,uint32_t length) {
+ CpuMemorySpan span{at,length}; return WatchCpuMemory({&span,1});
+}
+int main(int argc,char**) {
+ if(argc>1) { assert(Stamp(0xA0002000,64).pages.empty()); return 0; }
+ auto& memory=rex::memory::memory;
+ assert(!CpuMemoryUnchanged({}));
+ assert(Stamp(0x70002000,64).pages.empty()); // Virtual-only data always hash.
+ assert(Stamp(0xA00FFFF0,128).pages.empty()); // Uncommitted end.
+ assert(Stamp(0xFFFFFFF0,128).pages.empty()); // Address overflow.
+ assert(Stamp(0xA0002000,0).pages.empty());
+ auto first=Stamp(0xA0002001,8192); // Three pages.
+ assert(first.pages.size()==3&&CpuMemoryUnchanged(first));
+ auto neighbor=Stamp(0xC0005000,4096);
+ memory.Write(0xC0003004); // Same backing, other alias.
+ assert(!CpuMemoryUnchanged(first)&&CpuMemoryUnchanged(neighbor));
+ first=Stamp(0xA0002001,8192); assert(CpuMemoryUnchanged(first));
+ memory.Write(0xE0002004); // E view's +4 KB maps to physical 0x3004.
+ assert(!CpuMemoryUnchanged(first)&&CpuMemoryUnchanged(neighbor));
+ first=Stamp(0xA0002001,8192);
+ memory.Write(0xA0003004); // Another write in the SAME frame.
+ assert(!CpuMemoryUnchanged(first));
+ const CpuMemorySpan spans[]={{0xA0002000,4096},{0xC0007000,4096}};
+ auto two=WatchCpuMemory(spans); assert(CpuMemoryUnchanged(two));
+ memory.Write(0xE0006001); assert(!CpuMemoryUnchanged(two)); // Mip tail only.
+ two=WatchCpuMemory(spans);
+ InvalidateCpuPhysicalMemory(0x2004,64); // TranslatePhysical host memmove.
+ assert(!CpuMemoryUnchanged(two));
+ auto intervening=Stamp(0xA0009000,4096);
+ std::thread writer([&]{memory.Write(0xC0009001);});writer.join();
+ assert(!CpuMemoryUnchanged(intervening)); // Arm before copy/hash.
+ auto after=Stamp(0xA0009000,4096);assert(CpuMemoryUnchanged(after));
+ ShutdownCpuMemoryWatch();assert(!CpuMemoryUnchanged(after));
+ auto recreated=Stamp(0xA0009000,4096);assert(CpuMemoryUnchanged(recreated));
+ memory.Write(0xE0008001);assert(!CpuMemoryUnchanged(recreated));
+ ShutdownCpuMemoryWatch();
+}
+''')
+exe=a.output/'memory-watch-test.exe'
+subprocess.run(['clang++','-std=c++20','-DNOMINMAX','-I'+str(a.output/'include'),
+ '-I'+str(root/'rexlego/src'),str(root/'rexlego/src/gpu_native/memory_watch.cpp'),str(h),'-o',str(exe)],check=True)
+env=dict(os.environ);env.pop('LEGO_NATIVE_NO_MEMORY_WATCH',None);env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='1'
+subprocess.run([str(exe.resolve())],env=env,check=True)
+env['LEGO_NATIVE_NO_MEMORY_WATCH']='1';subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True)
+env.pop('LEGO_NATIVE_NO_MEMORY_WATCH');env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='0'
+subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True)
+env.pop('LEGO_NATIVE_STATIC_TEXTURE_WATCH')
+subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True)
+(a.output/'verification.json').write_text(json.dumps({'production_source':True,'sdk_model':True,'checks':[
+ 'A/C/E aliases and +4KB','same-frame writes','neighbor watch retained','base+mips','physical host write',
+ 'write between arm and reuse','unknown/uncommitted/overflow fallback','device recreation','disabled fallback'],
+ 'passed':True},indent=2))
+print('Passed production watcher with SDK callback model: aliases, same-frame writes, separate mips, races, neighbor protection, physical copies and fallback.')
diff --git a/tools/gpu-aot/test_memory_watch_sdk.py b/tools/gpu-aot/test_memory_watch_sdk.py
new file mode 100644
index 0000000..7205f35
--- /dev/null
+++ b/tools/gpu-aot/test_memory_watch_sdk.py
@@ -0,0 +1,70 @@
+"""Exercise native texture watches with the installed SDK's real page faults."""
+import argparse
+import os
+from pathlib import Path
+import subprocess
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+a.output=a.output.resolve();a.output.mkdir(parents=True,exist_ok=True)
+root=Path(__file__).resolve().parents[2];sdk=root/'rexglue-sdk/out/install/win-amd64'
+stub=a.output/'include/rex/system';stub.mkdir(parents=True,exist_ok=True)
+(stub/'kernel_state.h').write_text('''#pragma once
+#include <rex/system/xmemory.h>
+extern rex::memory::Memory* tested_memory;
+#define REX_KERNEL_MEMORY() tested_memory
+''')
+cpp=a.output/'watch-sdk.cpp';exe=a.output/'watch-sdk.exe'
+cpp.write_text(r'''
+#include <rex/logging.h>
+#include <rex/cvar.h>
+#include <rex/system/xmemory.h>
+#include "gpu_native/memory_watch.h"
+#include <cassert>
+#include <iostream>
+rex::memory::Memory* tested_memory=nullptr;
+using namespace legodimensions::gpu_native;
+REXCVAR_DECLARE(bool, gpu_native_texture_watch);
+int main(int argc,char**){
+ rex::InitLogging();
+ if(argc>1) REXCVAR_SET(gpu_native_texture_watch,true);
+ rex::memory::Memory memory;tested_memory=&memory;
+ assert(memory.Initialize());
+ auto* heap=memory.LookupHeapByType(true,4096);
+ uint32_t address=0;
+ constexpr uint32_t rw=rex::memory::kMemoryProtectRead|rex::memory::kMemoryProtectWrite;
+ assert(heap->Alloc(16384,4096,rex::memory::kMemoryAllocationReserve|
+     rex::memory::kMemoryAllocationCommit,rw,false,&address));
+ auto* bytes=memory.TranslateVirtual<volatile uint8_t*>(address);
+ CpuMemorySpan spans[]={{address,4096},{address+8192,4096}};
+ auto stamp=WatchCpuMemory(spans);assert(CpuMemoryUnchanged(stamp));
+ auto* physical=memory.GetPhysicalHeap();
+ assert(physical->IsRangeCommittedReadable(memory.GetPhysicalAddress(address),16384));
+ // Re-protecting a guest allocation must not silently erase its host watch.
+ assert(heap->Protect(address,4096,rw,nullptr));
+ bytes[4]=1;assert(!CpuMemoryUnchanged(stamp));
+ for(unsigned i=0;i<100;++i){
+   stamp=WatchCpuMemory(spans);assert(CpuMemoryUnchanged(stamp));
+   bytes[(i%2)*8192+17]=uint8_t(i);assert(!CpuMemoryUnchanged(stamp));
+ }
+ stamp=WatchCpuMemory(spans);
+ auto physical_address=memory.GetPhysicalAddress(address);
+ InvalidateCpuPhysicalMemory(physical_address,16);
+ *memory.TranslatePhysical<uint8_t*>(physical_address)=42;
+ assert(!CpuMemoryUnchanged(stamp));
+ stamp=WatchCpuMemory(spans);assert(CpuMemoryUnchanged(stamp));
+ assert(heap->Decommit(address+8192,4096));assert(!CpuMemoryUnchanged(stamp));
+ assert(!physical->IsRangeCommittedReadable(physical_address,16384));
+ ShutdownCpuMemoryWatch();
+ assert(heap->Release(address,nullptr));
+ std::cout<<"PASS real SDK page writes/rearm/protect, physical copies, decommit and bounded commitment check\n";
+}
+''')
+subprocess.run(['clang++','-std=c++23','-DNOMINMAX','-fms-runtime-lib=dll','-fuse-ld=lld',
+ '-DSPDLOG_COMPILED_LIB','-DSPDLOG_FMT_EXTERNAL','-D_CRT_SECURE_NO_WARNINGS',
+ '-Xclang','--dependent-lib=msvcrt','-I'+str(a.output/'include'),'-I'+str(root/'rexlego/src'),
+ '-I'+str(sdk/'include'),str(cpp),str(root/'rexlego/src/gpu_native/memory_watch.cpp'),
+ str(sdk/'lib/rexruntime.lib'),str(sdk/'lib/fmt.lib'),str(sdk/'lib/spdlog.lib'),'-o',str(exe)],check=True)
+env=dict(os.environ);env['PATH']=str(sdk/'bin')+os.pathsep+env.get('PATH','')
+env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='1';env.pop('LEGO_NATIVE_NO_MEMORY_WATCH',None)
+subprocess.run([str(exe)],cwd=a.output,env=env,check=True,timeout=30)
+env.pop('LEGO_NATIVE_STATIC_TEXTURE_WATCH')
+subprocess.run([str(exe),'configured'],cwd=a.output,env=env,check=True,timeout=30)
diff --git a/tools/gpu-aot/test_pixel_shader_filter.cpp b/tools/gpu-aot/test_pixel_shader_filter.cpp
new file mode 100644
index 0000000..3f5c601
--- /dev/null
+++ b/tools/gpu-aot/test_pixel_shader_filter.cpp
@@ -0,0 +1,10 @@
+#include <cassert>
+#include "gpu_native/pixel_shader_filter.h"
+using namespace legodimensions::gpu_native;
+int main() {
+  assert(ParsePixelShaderFilter("").empty());
+  auto hashes = ParsePixelShaderFilter(" CFEAC7ADB912F8A9, 0x12,0X12, abcd ,bad-hash,,FFFFFFFFFFFFFFFFF");
+  assert(hashes.size() == 3 && hashes.contains(0xCFEAC7ADB912F8A9ull) &&
+         hashes.contains(0x12) && hashes.contains(0xABCD));
+  assert(ParsePixelShaderFilter("\t,0x,wat\n").empty());
+}
diff --git a/tools/gpu-aot/test_placement_resource_type.py b/tools/gpu-aot/test_placement_resource_type.py
new file mode 100644
index 0000000..09ff438
--- /dev/null
+++ b/tools/gpu-aot/test_placement_resource_type.py
@@ -0,0 +1,88 @@
+"""Execute actual TU23 GetResourceType / XGOffsetResource with native type hooks."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True)
+def body(path,signature):
+ text=(root/path).read_text();start=text.index(signature);end=text.index('{',start)+1;depth=1
+ while depth:depth+=(text[end]=='{')-(text[end]=='}');end+=1
+ return text[start:end]
+get=body('rexlego/generated/default/legodimensions_recomp.209.cpp','DEFINE_REX_FUNC(sub_83FC49E8)')
+get=get.replace('DEFINE_REX_FUNC(sub_83FC49E8)','void __imp__sub_83FC49E8(PPCContext& ctx,u8* base)')
+offset=body('rexlego/generated/default/legodimensions_recomp.314.cpp','DEFINE_REX_FUNC(sub_83F9B640)')
+offset=offset.replace('DEFINE_REX_FUNC(sub_83F9B640)','void sub_83F9B640(PPCContext& ctx,u8* base)')
+hook=body('rexlego/src/gpu_native/hooks_buffer.cpp','REX_HOOK_RAW(sub_83FC49E8)')
+hook=hook.replace('REX_HOOK_RAW(sub_83FC49E8)','void sub_83FC49E8(PPCContext& ctx,u8* base)')
+buf=body('rexlego/src/gpu_native/buffers.cpp','u32 NativeBufferType(')
+tex=body('rexlego/src/gpu_native/textures.cpp','u32 NativeTextureType(')
+h=a.output/'placement-resource-type.cpp'
+h.write_text(r'''
+#include <array>
+#include <cassert>
+#include <cstdint>
+#include <memory>
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;
+union PPCRegister {u64 u64;int64_t s64;u32 u32;int32_t s32;u8 u8;};
+struct XER {u8 ca=0;};
+struct CR {bool eq=false,gt=false;template<class T>void compare(T a,T b,XER){eq=a==b;gt=a>b;}};
+struct PPCContext {
+'''+''.join(f' PPCRegister r{i}{{}};\n' for i in range(32))+r'''
+ u64 lr=0;PPCRegister ctr{};XER xer{};CR cr0{},cr6{};
+};
+#define REX_FUNC_PROLOGUE() ((void)0)
+#define REXLOG_INFO(...) ((void)0)
+std::array<u8,65536> ram{};
+u32 Load(u32 p){return (u32(ram[p])<<24)|(u32(ram[p+1])<<16)|(u32(ram[p+2])<<8)|ram[p+3];}
+void Store(u32 p,u32 v){for(u32 i=0;i<4;++i)ram[p+i]=u8(v>>(24-8*i));}
+void Store64(u32 p,u64 v){Store(p,u32(v>>32));Store(p+4,u32(v));}
+u64 Load64(u32 p){return (u64(Load(p))<<32)|Load(p+4);}
+#define REX_LOAD_U32(p) Load(p)
+#define REX_STORE_U32(p,v) Store(p,v)
+#define REX_LOAD_U64(p) Load64(p)
+#define REX_STORE_U64(p,v) Store64(p,v)
+u32 texture_offset_calls=0,last_texture=0,last_base_delta=0,last_mip_delta=0;
+void sub_83F9B5D8(PPCContext& c,u8*){++texture_offset_calls;last_texture=c.r3.u32;last_base_delta=c.r4.u32;last_mip_delta=c.r5.u32;}
+namespace legodimensions::gpu_native {
+struct Buffer {bool owns_guest_memory;u32 kind;};struct Texture {bool owns_guest_memory;u32 d3d_type;};
+std::shared_ptr<Buffer> buffer;std::shared_ptr<Texture> texture;
+auto FindBuffer(u32)->std::shared_ptr<Buffer>{return buffer;}
+auto FindTexture(u32)->std::shared_ptr<Texture>{return texture;}
+bool IsNativeBuffer(u32){return bool(buffer);}
+'''+buf+'\n'+tex+'\n}\n'+get+'\n'+hook+'\n'+offset+r'''
+void Reset(u32 common,u32 fetch5=0){ram.fill(0);Store(1024,common);Store(1048,0x2003);Store(1056,0xAABBCCDD);Store(1072,fetch5);texture_offset_calls=0;}
+PPCContext RunOffset(){PPCContext c{};c.r1.u32=60000;c.r3.u32=1024;c.r4.u32=0x1000;sub_83F9B640(c,ram.data());return c;}
+int main(){
+ using namespace legodimensions::gpu_native;
+ // The original owner relocation must update +24 for adopted VB, not +32.
+ buffer=std::make_shared<Buffer>(Buffer{false,6});Reset(0x200001);RunOffset();
+ assert(Load(1048)==0x3003&&Load(1056)==0xAABBCCDD);
+ // Native-owned compatibility override remains as before.
+ buffer->owns_guest_memory=true;Reset(6);RunOffset();
+ assert(Load(1048)==0x2003&&Load(1056)==0xAABBDCDD);
+ buffer->owns_guest_memory=false;buffer->kind=7;Reset(0x80000002);RunOffset();
+ assert(Load(1048)==0x3003&&Load(1056)==0xAABBCCDD);
+ buffer.reset();texture=std::make_shared<Texture>(Texture{false,3});
+ const std::array<u32,4> types{20,3,17,18};
+ for(u32 dimension=0;dimension<4;++dimension){
+  Reset(3,dimension<<9);Store(1056,0);PPCContext c{};c.r3.u32=1024;sub_83FC49E8(c,ram.data());assert(c.r3.u32==types[dimension]);
+  RunOffset();assert(texture_offset_calls==1&&last_texture==1024&&last_base_delta==0x1000&&last_mip_delta==0);
+ }
+ Reset(3,(1u<<9));Store(1056,0x400);PPCContext c{};c.r3.u32=1024;sub_83FC49E8(c,ram.data());assert(c.r3.u32==19);
+ texture->owns_guest_memory=true;texture->d3d_type=18;c.r3.u32=1024;sub_83FC49E8(c,ram.data());assert(c.r3.u32==18);
+ // Deliberately recreate the old adopted-VB override to demonstrate wrong write.
+ texture.reset();buffer=std::make_shared<Buffer>(Buffer{true,6});Reset(1);RunOffset();
+ assert(Load(1048)==0x2003&&Load(1056)==0xAABBDCDD);
+}
+''')
+exe=a.output/'placement-resource-type.exe'
+subprocess.run(['clang++','-std=c++20',str(h),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+(a.output/'verification.json').write_text(json.dumps({'actual_generated_functions':['83FC49E8','83F9B640'],
+ 'actual_native_functions':['NativeBufferType','NativeTextureType','83FC49E8 hook'],
+ 'checks':['borrowed VB correct +24','owned VB compatibility +32','borrowed IB +24',
+ 'four texture dimensions','stacked texture','owned texture compatibility','old bug reproduced'],
+ 'runtime_title_idle_not_proven':True,'passed':True},indent=2))
+print('Passed actual generated relocation/type functions: borrowed VB/IB use +24; owned-header overrides preserved.')
diff --git a/tools/gpu-aot/test_pool_copy.cpp b/tools/gpu-aot/test_pool_copy.cpp
new file mode 100644
index 0000000..a49e465
--- /dev/null
+++ b/tools/gpu-aot/test_pool_copy.cpp
@@ -0,0 +1,34 @@
+#include "gpu_native/pool_copy.h"
+
+#include <cassert>
+#include <limits>
+#include <iostream>
+
+using namespace legodimensions::gpu_native;
+
+int main() {
+  // Touching allocations must not evict each other's cached resources.
+  assert(!CopyRangesOverlap({0x1000, 64}, {0x1040, 64}));
+  assert(CopyRangesOverlap({0x1000, 64}, {0x103F, 64}));
+  assert(CopyRangesOverlap({0x103F, 64}, {0x1000, 64}));
+  assert(!CopyRangesOverlap({0x1000, 0}, {0x1000, 64}));
+  assert(CopyRangesOverlap({0xFFFFFFF0, 32}, {0xFFFFFFFE, 1}));
+  assert(!CopyRangesOverlap({0xFFFFFFF0, 32}, {0, 32}));
+
+  PoolCopyDraws copy{32769};
+  assert(!copy.Consume(4, 0, 16384));  // A triangle pass never matches.
+  assert(!copy.Consume(1, 1, 16384));  // Only consecutive records are consumed.
+  assert(!copy.Consume(1, 0, 16385));
+  assert(copy.consumed == 0);
+  assert(copy.Consume(1, 0, 16384));
+  assert(!copy.Consume(1, 0, 16384)); // Repeated batch must not disappear.
+  assert(copy.Consume(1, 16384, 16384));
+  assert(!copy.Consume(1, 32768, 2));
+  assert(copy.Consume(1, 32768, 1));
+  assert(copy.consumed == copy.records);
+  assert(!copy.Consume(1, 32769, 1));
+  assert(!copy.Consume(1, 32769, 0));
+  PoolCopyDraws empty{};
+  assert(!empty.Consume(1, 0, 1));
+  std::cout << "pool copy overlap and POINTLIST batch checks passed\n";
+}
diff --git a/tools/gpu-aot/test_query_retirement.py b/tools/gpu-aot/test_query_retirement.py
new file mode 100644
index 0000000..071489e
--- /dev/null
+++ b/tools/gpu-aot/test_query_retirement.py
@@ -0,0 +1,150 @@
+"""Execute native query generation/release/readback bodies with fake GPU fences."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+root = Path(__file__).resolve().parents[2]
+a.output.mkdir(parents=True, exist_ok=True)
+
+def body(path, signature):
+    text = (root / path).read_text()
+    start = text.index(signature)
+    end = text.index('{', start) + 1
+    depth = 1
+    while depth:
+        depth += (text[end] == '{') - (text[end] == '}')
+        end += 1
+    return text[start:end]
+
+query_path = 'rexlego/src/gpu_native/queries.cpp'
+functions = '\n'.join(body(query_path, sig) for sig in
+                      ['u32 IssueQuery(', 'u32 GetQueryData(', 'u32 ReleaseQuery('])
+assert 'Synchronize' not in functions
+retire = body('rexlego/src/gpu_native/device.cpp', 'void HostDevice::RetireResource(')
+harness = r'''
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+using u32=uint32_t;using u64=uint64_t;using be_u32=u32;
+#define REXLOG_INFO(...) ((void)0)
+#define FAILED(x) ((x)!=0)
+#define SUCCEEDED(x) ((x)==0)
+#define IID_PPV_ARGS(x) (x)
+constexpr int D3D12_FENCE_FLAG_NONE=0;
+constexpr u32 kInvalid=0x8876086C,kUnsupported=0x80004001,kFailure=0x80004005;
+template<class T>struct ComPtr {
+ std::shared_ptr<T> p;
+ T* Get()const{return p.get();} T* operator->()const{return p.get();}
+ ComPtr* operator&(){return this;}
+};
+struct ID3D12Fence {u64 completed=0;u64 GetCompletedValue(){return completed;}};
+struct D3D12_RANGE {size_t Begin,End;};
+struct Readback {
+ u64 value=0;bool fail=false;
+ int Map(int,const D3D12_RANGE* r,void** out){assert(r->End==8);*out=&value;return fail?1:0;}
+ void Unmap(int,const D3D12_RANGE* r){assert(r->End==0);}
+};
+struct Samples {std::shared_ptr<Readback> readback=std::make_shared<Readback>();};
+struct Query {
+ bool active=false,ended=false,failed=false;
+ std::vector<std::shared_ptr<Samples>> draws;
+ ComPtr<ID3D12Fence> completion;
+};
+std::mutex mutex,g_mutex;
+std::unordered_map<u32,std::shared_ptr<Query>> queries;
+std::atomic<u32> diagnostics{100};
+std::array<u32,256> header{};
+be_u32* Header(u32 a){return header.data()+a;}
+u32 Unsupported(const char*,u32){return kUnsupported;}
+struct Memory {int frees=0;void SystemHeapFree(u32 a){assert(a==1);++frees;}} memory;
+#define REX_KERNEL_MEMORY() (&memory)
+struct DeviceImpl {
+ bool fail=false;
+ int CreateFence(int,int,ComPtr<ID3D12Fence>* out){if(fail)return 1;out->p=std::make_shared<ID3D12Fence>();return 0;}
+};
+struct QueueImpl {bool fail=false;int Signal(ID3D12Fence* f,int value){assert(f&&value==1);return fail?1:0;}};
+namespace plume {struct D3D12Device {DeviceImpl* d3d;};struct D3D12CommandQueue {QueueImpl* d3d;};}
+DeviceImpl device_impl;QueueImpl queue_impl;
+plume::D3D12Device device{&device_impl};plume::D3D12CommandQueue queue{&queue_impl};
+struct State {
+ static constexpr u32 kFramesInFlight=3;
+ std::array<bool,3> frame_submitted{true,false,false};
+ bool command_list_open=true;u32 frame_slot=1;
+ std::array<std::vector<std::shared_ptr<void>>,3> retired_resources;
+};
+std::unique_ptr<State> g_state=std::make_unique<State>();
+struct HostDevice {
+ static inline std::recursive_mutex recording;
+ static inline bool submit_ok=true;
+ static auto LockRecording(){return std::unique_lock(recording);}
+ static auto Device(){return &device;} static auto Queue(){return &queue;}
+ static bool SubmitRecordedWork(){return submit_ok;}
+ static void RetireResource(std::shared_ptr<void>);
+};
+'''
+tests = r'''
+std::shared_ptr<Samples> sample(u64 value){auto s=std::make_shared<Samples>();s->readback->value=value;return s;}
+void empty_slots(){for(auto& s:g_state->retired_resources)s.clear();}
+int main(){
+ auto initial=std::make_shared<Query>();initial->completion.p=std::make_shared<ID3D12Fence>();
+ initial->draws.push_back(sample(999));initial->ended=true;
+ std::weak_ptr<Query> old=initial;queries[1]=initial;initial.reset();Header(1)[3]=1;
+ assert(IssueQuery(1,2)==0);auto current=queries[1];
+ assert(current->active&&current->draws.empty()&&!current->ended&&!current->failed);
+ assert(!old.expired()); // Previous submission and open list each hold it.
+ g_state->retired_resources[0].clear();assert(!old.expired());
+ g_state->retired_resources[1].clear();assert(old.expired());
+ assert(IssueQuery(1,2)==kInvalid); // Active generation cannot be restarted.
+ current->draws={sample(7),sample(5)};
+ assert(IssueQuery(1,1)==0);assert(!current->active&&current->ended);
+ assert(GetQueryData(1,100,4,0)==1); // Never invent a ready result.
+ current->completion->completed=1;assert(GetQueryData(1,100,4,0)==0&&*Header(100)==12);
+ current->completion->completed=UINT64_MAX;assert(GetQueryData(1,100,4,0)==kFailure);
+ current->completion->completed=1;
+ assert(GetQueryData(1,100,8,0)==kUnsupported);
+ current->draws[0]->readback->fail=true;assert(GetQueryData(1,100,4,0)==kFailure);
+ current->draws[0]->readback->fail=false;
+ current->draws={sample(UINT32_MAX),sample(5)};
+ assert(GetQueryData(1,100,4,0)==0&&*Header(100)==UINT32_MAX);
+ device_impl.fail=true;assert(IssueQuery(1,2)==kFailure&&queries[1]==current&&current->ended);
+ device_impl.fail=false;
+ std::weak_ptr<Query> ended=current;current.reset();
+ assert(IssueQuery(1,2)==0);assert(!ended.expired());empty_slots();assert(ended.expired());
+ auto active=queries[1];active->draws={sample(19)};std::weak_ptr<Query> releasing=active;
+ Header(1)[3]=2;assert(ReleaseQuery(1)==1&&queries.contains(1)&&active->active&&memory.frees==0);
+ active.reset();assert(ReleaseQuery(1)==0); // Release before END is safe.
+ assert(!queries.contains(1)&&memory.frees==1&&!releasing.expired());
+ assert(GetQueryData(1,100,4,0)==kInvalid&&ReleaseQuery(1)==0&&memory.frees==1);
+ g_state->retired_resources[0].clear();assert(!releasing.expired());
+ g_state->retired_resources[1].clear();assert(releasing.expired());
+ // No pending work needs retention. Same guest address may be reused.
+ g_state->frame_submitted.fill(false);g_state->command_list_open=false;
+ auto next=std::make_shared<Query>();queries[1]=next;Header(1)[3]=1;
+ next->active=true;next->completion.p=std::make_shared<ID3D12Fence>();
+ queue_impl.fail=true;assert(IssueQuery(1,1)==kFailure&&next->failed);
+ assert(GetQueryData(1,100,4,0)==kFailure);
+ std::weak_ptr<Query> done=next;next.reset();assert(ReleaseQuery(1)==0&&done.expired());
+}
+'''
+source = a.output / 'query-retirement.cpp'
+source.write_text(harness + retire + '\n' + functions + '\n' + tests)
+exe = a.output / 'query-retirement-test.exe'
+subprocess.run(['clang++', '-std=c++20', '-DNOMINMAX', str(source), '-o', str(exe)], check=True)
+subprocess.run([str(exe.resolve())], check=True)
+report = {'passed': True, 'actual_native_functions': ['IssueQuery','GetQueryData','ReleaseQuery','HostDevice::RetireResource'],
+          'checks': ['no query drain', 'fresh generations', 'retain at every submitted/open slot',
+                     'release before END', 'guest header reuse', 'nonfinal refcount', 'fence readiness',
+                     'no fabricated result', 'removal failure', 'readback failure', 'saturating sum',
+                     'allocation/signal failures'], 'real_GPU_gameplay_not_proven': True}
+(a.output / 'verification.json').write_text(json.dumps(report, indent=2))
+print('PASS: native query generations, fence-gated results and all-live-slot retirement.')
diff --git a/tools/gpu-aot/test_readable_range.py b/tools/gpu-aot/test_readable_range.py
new file mode 100644
index 0000000..1c6ff11
--- /dev/null
+++ b/tools/gpu-aot/test_readable_range.py
@@ -0,0 +1,68 @@
+"""Execute the production bounded heap check with holes and boundary cases."""
+import argparse
+from pathlib import Path
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+a.output.mkdir(parents=True, exist_ok=True)
+root = Path(__file__).resolve().parents[2]
+source = (root / 'rexglue-sdk/src/system/xmemory.cpp').read_text()
+start = source.index('bool BaseHeap::IsRangeCommittedReadable(')
+end = source.index('\nbool BaseHeap::QuerySize(', start)
+method = source[start:end]
+code = r'''
+#include <cstdint>
+#include <mutex>
+#include <vector>
+#include <cassert>
+#include <random>
+namespace memory { constexpr uint32_t kMemoryAllocationCommit=2, kMemoryProtectRead=1; }
+struct Entry { uint32_t state=2, current_protect=1; };
+struct BaseHeap {
+ uint32_t heap_base_=0x100000, heap_size_=128*4096, page_size_shift_=12;
+ std::recursive_mutex heap_mutex_;
+ std::vector<Entry> page_table_=std::vector<Entry>(128);
+ bool IsRangeCommittedReadable(uint32_t address,uint32_t length);
+};
+''' + method + r'''
+int main() {
+ BaseHeap h;
+ assert(h.IsRangeCommittedReadable(h.heap_base_,h.heap_size_));
+ assert(!h.IsRangeCommittedReadable(h.heap_base_-1,1));
+ assert(!h.IsRangeCommittedReadable(h.heap_base_+h.heap_size_,1));
+ assert(!h.IsRangeCommittedReadable(0xfffffff0u,32));
+ assert(h.IsRangeCommittedReadable(0,0));
+ h.page_table_[5].state=0; // A hole inside a readable allocation.
+ assert(!h.IsRangeCommittedReadable(h.heap_base_+4096*4+4095,2));
+ assert(h.IsRangeCommittedReadable(h.heap_base_+4096*4,4096));
+ h.page_table_[5].state=1; // Reserved but not committed; read flag still set.
+ assert(!h.IsRangeCommittedReadable(h.heap_base_+4096*5,1));
+ h.page_table_[5]={2,0};
+ assert(!h.IsRangeCommittedReadable(h.heap_base_+4096*5,1));
+ std::mt19937 rng(0x1080);
+ for (unsigned n=0;n<10000;++n) {
+   auto& e=h.page_table_[rng()%128]; e.state=rng()%4;e.current_protect=rng()%4;
+   uint32_t offset=rng()%(h.heap_size_+4096), length=rng()%8193;
+   bool expected=true;
+   for(uint32_t i=0;i<length;++i) {
+     if(offset+i>=h.heap_size_){expected=false;break;}
+     const auto& entry=h.page_table_[(offset+i)/4096];
+     if(!(entry.state&2)||!(entry.current_protect&1)){expected=false;break;}
+   }
+   assert(h.IsRangeCommittedReadable(h.heap_base_+offset,length)==expected);
+ }
+ // An address range ending at 4 GiB is valid in the last virtual heap.
+ h.heap_base_=0xfff80000;
+ for(auto& e:h.page_table_)e={2,1};
+ assert(h.IsRangeCommittedReadable(0xfffffff0u,16));
+ assert(!h.IsRangeCommittedReadable(0xfffffff0u,17));
+}
+'''
+cpp = a.output / 'readable-range.cpp'
+cpp.write_text(code)
+exe = a.output / 'readable-range.exe'
+subprocess.run(['clang++', '-std=c++20', str(cpp), '-o', str(exe)], check=True)
+subprocess.run([str(exe.resolve())], check=True)
+print('PASS: production range check, holes/reservation/protection, unaligned boundaries, 4 GiB edge, 10000 differential cases')
diff --git a/tools/gpu-aot/test_renderer_route.py b/tools/gpu-aot/test_renderer_route.py
new file mode 100644
index 0000000..a8a9245
--- /dev/null
+++ b/tools/gpu-aot/test_renderer_route.py
@@ -0,0 +1,83 @@
+"""Exercise the actual hook routing macros for every native GPU replacement."""
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+a.output.mkdir(parents=True, exist_ok=True)
+root = Path(__file__).resolve().parents[2]
+sources = root/'rexlego/src/gpu_native'
+hooks = []
+for path in sources.glob('*.cpp'):
+    text = path.read_text()
+    found = re.findall(r'REX_HOOK(_RAW)?\(\s*(sub_[0-9A-Fa-f]+)', text)
+    if found:
+        assert '#include "gpu_native/renderer_route.h"' in text, path
+    hooks.extend((bool(raw), name) for raw, name in found)
+assert len(hooks) == len({name for _,name in hooks}) and hooks
+fake = a.output/'include/rex/hook.h'
+fake.parent.mkdir(parents=True, exist_ok=True)
+fake.write_text('''#pragma once
+#include <cstdint>
+using u8=uint8_t;
+struct PPCContext { uint64_t r3=0, canary=0x123456789ABCDEF0ull; };
+#define REX_FUNC(name) void name(PPCContext& ctx, u8* base)
+#define REXLOG_INFO(...) ((void)0)
+namespace rex::ppc { template<auto Function> void HostToGuestFunction(PPCContext& ctx,u8* base) { Function(ctx,base); } }
+''')
+(fake.parent/'logging.h').write_text('#pragma once\n#define REXLOG_INFO(...) ((void)0)\n#define REXLOG_ERROR(...) ((void)0)\n')
+(fake.parent/'cvar.h').write_text('#pragma once\n#define REXCVAR_DECLARE(type,name) type& FLAGS_##name##_storage_()\n#define REXCVAR_GET(name) FLAGS_##name##_storage_()\n')
+app_source = (root/'rexlego/src/legodimensions_app.h').read_text()
+def app_body(name):
+    start = app_source.index('void '+name+'(')
+    pos = app_source.index('{',start)
+    end, depth = pos+1, 1
+    while depth:
+        depth += (app_source[end]=='{')-(app_source[end]=='}')
+        end += 1
+    return app_source[start:end].replace(' override','')
+code = ['#include <cassert>\n#include "gpu_native/renderer_route.h"',
+        'bool& FLAGS_gpu_native_pm4_storage_() { static bool setting=std::getenv("TEST_PM4_CONFIG")!=nullptr; return setting; }',
+        '#include <memory>\n#include <string>',
+        'namespace rex { struct RuntimeConfig { std::unique_ptr<int> graphics; std::string gpu_plugin; }; }',
+        'namespace legodimensions::gpu_native { struct HostDevice { inline static int creates=0; static bool Create(int) { ++creates; return true; } }; }',
+        'struct App { int window(){return 0;} App& app_context(){return *this;} void QuitFromUIThread(){assert(false);}',
+        app_body('OnPreSetup'),app_body('OnPreLaunchModule'),'};',
+        'static int originals=0,natives=0; static u8* expected_base;',
+        'static void native_call(PPCContext& ctx,u8* base) { assert(base==expected_base); ++natives; ctx.r3=22; }']
+for raw,name in hooks:
+    code.append(f'extern "C" REX_FUNC(__imp__{name}) {{ assert(base==expected_base); ++originals; ctx.r3=11; }}')
+    code.append(f'REX_HOOK_RAW({name}) {{ native_call(ctx,base); }}' if raw else f'REX_HOOK({name},native_call);')
+code.append('int main(int argc,char**) { bool reference=argc>1; u8 memory[8]{}; expected_base=memory; PPCContext ctx;')
+code.append('App app; rex::RuntimeConfig config; config.graphics=std::make_unique<int>(7); config.gpu_plugin="xenos"; app.OnPreSetup(config); assert(bool(config.graphics)==reference); assert(config.gpu_plugin==(reference?"xenos":"")); app.OnPreLaunchModule(); assert(legodimensions::gpu_native::HostDevice::creates==(reference?0:1)); rex::RuntimeConfig empty; app.OnPreSetup(empty); assert(empty.gpu_plugin==(reference?"xenos":""));')
+for _,name in hooks:
+    code.append(f'{name}(ctx,memory); assert(ctx.r3==(reference?11:22)); assert(ctx.canary==0x123456789ABCDEF0ull);')
+code.append(f'assert(originals==(reference?{len(hooks)}:0)); assert(natives==(reference?0:{len(hooks)}));')
+# Selection remains locked for resource lifetime, even if the environment changes.
+code.append('_putenv_s("LEGO_NATIVE_PM4_REFERENCE",reference?"0":"1"); assert(legodimensions::gpu_native::UsePm4Reference()==reference); }')
+cpp = a.output/'route.cpp'
+cpp.write_text('\n'.join(code))
+exe = a.output/'route.exe'
+subprocess.run(['clang++','-std=c++20','-I'+str(a.output/'include'),'-I'+str(root/'rexlego/src'),str(cpp),'-o',str(exe)],check=True)
+for configured in (False,True):
+    for value in (None,'0','1','false','yes',''):
+        env = dict(os.environ)
+        env.pop('LEGO_NATIVE_PM4_REFERENCE',None)
+        env.pop('TEST_PM4_CONFIG',None)
+        if configured: env['TEST_PM4_CONFIG']='1'
+        if value is not None: env['LEGO_NATIVE_PM4_REFERENCE']=value
+        selected = value=='1' if value in ('0','1') else configured
+        subprocess.run([str(exe.resolve())]+(['reference'] if selected else []),env=env,check=True)
+report = {'passed':True,'hooks':len(hooks),'raw_hooks':sum(raw for raw,_ in hooks),
+          'environment_cases':12,'actual_header_macros':True,'mock_ppc_and_marshaler':True,
+          'checks':['all native GPU hook sources covered','original path bypasses native body',
+                    'configured mode and explicit environment override','ctx/base forwarded','register canary preserved',
+                    'route immutable after first use','actual app setup preserves SDK plugin in reference',
+                    'actual app prelaunch bypasses native initialization in reference']}
+(a.output/'verification.json').write_text(json.dumps(report,indent=2)+'\n')
+print(json.dumps(report,indent=2))
diff --git a/tools/gpu-aot/test_resize_fences.py b/tools/gpu-aot/test_resize_fences.py
new file mode 100644
index 0000000..756001f
--- /dev/null
+++ b/tools/gpu-aot/test_resize_fences.py
@@ -0,0 +1,172 @@
+"""Exercise actual Plume wait and native resize bodies without a game/window.
+
+Deterministic events/fences cover old auto-reset wakeups and partial retries.
+"""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+root = Path(__file__).resolve().parents[2]
+a.output.mkdir(parents=True, exist_ok=True)
+
+def body(path, signature):
+    source = (root / path).read_text()
+    start = source.index(signature)
+    end = source.index('{', start) + 1
+    depth = 1
+    while depth:
+        depth += (source[end] == '{') - (source[end] == '}')
+        end += 1
+    return source[start:end]
+
+plume = body('thirdparty/plume/plume_d3d12.cpp',
+             'void D3D12CommandQueue::waitForCommandFence(')
+resize = body('thirdparty/plume/plume_d3d12.cpp', 'bool D3D12SwapChain::resize(')
+native = '\n'.join(body('rexlego/src/gpu_native/device.cpp', sig) for sig in (
+    'void RefreshCompletedSubmissionsLocked(', 'bool WaitForSubmittedFramesLocked(',
+    'bool RebuildSwapChain('))
+source = r'''
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <cstdint>
+#include <cstdio>
+#include <deque>
+#include <memory>
+#include <string>
+#include <vector>
+using u32=uint32_t;using u64=uint64_t;
+#define REXLOG_ERROR(...) ((void)0)
+constexpr int WAIT_OBJECT_0=0,INFINITE=-1,FALSE=0;
+enum class Backend {kD3D12,kVulkan};
+namespace plume {
+struct NativeFence {u64 completed=0;u64 GetCompletedValue(){return completed;}};
+struct RenderCommandFence {virtual ~RenderCommandFence()=default;};
+struct D3D12CommandFence:RenderCommandFence {
+ NativeFence native;NativeFence* d3d=&native;u64 fenceValue=2;
+ D3D12CommandFence* fenceEvent=this;std::deque<u64> wakes;int event_waits=0;
+};
+struct D3D12CommandQueue {
+ int calls=0;
+ void waitForCommandFence(RenderCommandFence*);
+};
+}
+int WaitForSingleObjectEx(plume::D3D12CommandFence* event,int timeout,int alertable) {
+ assert(timeout==INFINITE&&alertable==FALSE);++event->event_waits;
+ // An empty wake queue would block forever with the old event-only wait.
+ assert(!event->wakes.empty());
+ event->native.completed=event->wakes.front();event->wakes.pop_front();
+ return WAIT_OBJECT_0;
+}
+namespace plume {
+'''+plume+r'''
+using HRESULT=long;
+#define FAILED(x) ((x)!=0)
+constexpr int DXGI_FORMAT_UNKNOWN=0;
+struct BackBuffer {int releases=0;void Release(){++releases;}};
+struct FakeDxgi {bool fail=false;int calls=0;HRESULT ResizeBuffers(int,int,int,int,u32){++calls;return fail?1:0;}};
+struct D3D12SwapChain {
+ struct {u32 textureCount=3;} desc;
+ struct Texture {BackBuffer* d3d=nullptr;};
+ std::array<Texture,3> textures;std::array<BackBuffer,3> buffers;
+ FakeDxgi dxgi;FakeDxgi* d3d=&dxgi;u32 swapChainFlags=0;
+ u32 width=1280,height=720,requested_width=1920,requested_height=1080;
+ int gets=0;bool device_removed=false;
+ D3D12SwapChain(){setTextures();}
+ void getWindowSize(u32& w,u32& h){w=requested_width;h=requested_height;}
+ void setTextures(){++gets;for(u32 i=0;i<3;++i)textures[i].d3d=device_removed?nullptr:&buffers[i];}
+ bool resize();
+};
+'''+resize+r'''
+}
+struct SwapChain {bool resize_ok=true,empty=false;int resizes=0;bool resize();bool isEmpty(){return empty;}};
+struct State {
+ static constexpr u32 kFramesInFlight=3;
+ Backend backend=Backend::kD3D12;
+ std::array<bool,3> frame_submitted{};
+ std::array<std::unique_ptr<plume::D3D12CommandFence>,3> frame_fences;
+ std::array<u64,3> slot_submission{1,2,3};u64 completed_submission=0;
+ std::unique_ptr<plume::D3D12CommandQueue> queue=std::make_unique<plume::D3D12CommandQueue>();
+ std::unique_ptr<SwapChain> swap_chain=std::make_unique<SwapChain>();
+ std::vector<int> framebuffers{1,2,3},render_semaphores{1,2,3};
+ bool command_list_open=false;int rings=0;
+ State(){for(auto& f:frame_fences)f=std::make_unique<plume::D3D12CommandFence>();}
+};
+State* active=nullptr;
+void AssertGpuIdle(){for(u32 i=0;i<3;++i){const auto& f=*active->frame_fences[i];
+ assert(!active->frame_submitted[i]);assert(f.native.completed!=UINT64_MAX&&f.native.completed>=f.fenceValue-1);}}
+bool SwapChain::resize(){AssertGpuIdle();++resizes;return resize_ok;}
+bool CreateFrameRing(State& s){AssertGpuIdle();++s.rings;s.framebuffers={4,5,6};s.render_semaphores={4,5,6};return true;}
+'''+native+r'''
+int main(){
+ {
+  plume::D3D12SwapChain s;s.requested_width=0;
+  assert(!s.resize()&&s.width==1280&&s.height==720&&s.dxgi.calls==0);
+  assert(s.textures[0].d3d&&s.buffers[0].releases==0);
+  s.requested_width=1920;s.dxgi.fail=true;assert(!s.resize());
+  assert(s.width==1280&&s.height==720&&s.textures[0].d3d&&s.gets==2);
+  s.dxgi.fail=false;assert(s.resize());assert(s.width==1920&&s.height==1080);
+  assert(s.buffers[0].releases==2&&s.textures[0].d3d);
+  s.device_removed=true;s.dxgi.fail=true;assert(!s.resize());
+  assert(!s.textures[0].d3d);assert(!s.resize()); // No null Release on retry.
+ }
+ // Complete fence whose auto-reset event has already been consumed: no wait.
+ plume::D3D12CommandQueue q;plume::D3D12CommandFence f;
+ f.native.completed=1;q.waitForCommandFence(&f);assert(f.event_waits==0);
+ // A stale event from submission 1 cannot satisfy submission 2.
+ f.fenceValue=3;f.wakes={1,2};q.waitForCommandFence(&f);
+ assert(f.native.completed==2&&f.event_waits==2);
+ // Device removal must never be treated as a healthy completion or block.
+ f.native.completed=UINT64_MAX;q.waitForCommandFence(&f);assert(f.event_waits==2);
+ {
+  State s;active=&s;s.frame_submitted.fill(true);
+  s.frame_fences[0]->wakes={1};s.frame_fences[1]->wakes={1};s.frame_fences[2]->wakes={1};
+  assert(RebuildSwapChain(s));assert(s.completed_submission==3&&s.rings==1&&s.swap_chain->resizes==1);
+  for(auto& fence:s.frame_fences)assert(fence->event_waits==1);
+ }
+ {
+  State s;active=&s;s.command_list_open=true;s.frame_submitted.fill(true);
+  assert(!RebuildSwapChain(s));assert(s.rings==0&&s.swap_chain->resizes==0&&s.framebuffers.size()==3);
+  for(auto& fence:s.frame_fences)assert(fence->event_waits==0);
+ }
+ {
+  State s;active=&s;s.frame_submitted.fill(true);
+  s.frame_fences[0]->native.completed=1;s.frame_fences[1]->native.completed=UINT64_MAX;
+  assert(!RebuildSwapChain(s));assert(!s.frame_submitted[0]&&s.frame_submitted[1]);
+  assert(s.framebuffers[0]==1&&s.rings==0&&s.swap_chain->resizes==0);
+  // Retry must not wait again on slot 0's already-consumed event.
+  s.frame_fences[1]->native.completed=1;s.frame_fences[2]->wakes={1};
+  assert(RebuildSwapChain(s));assert(s.completed_submission==3);
+ }
+ {
+  State s;active=&s;s.frame_submitted.fill(true);
+  for(auto& fence:s.frame_fences)fence->native.completed=1;
+  s.swap_chain->resize_ok=false;assert(!RebuildSwapChain(s));
+  assert(s.rings==0);s.swap_chain->resize_ok=true;s.swap_chain->empty=true;
+  assert(!RebuildSwapChain(s));assert(s.rings==0);
+  s.swap_chain->empty=false;assert(RebuildSwapChain(s));assert(s.rings==1);
+  for(auto& fence:s.frame_fences)assert(fence->event_waits==0);
+ }
+ {
+  State s;active=&s;s.frame_submitted[0]=true;s.frame_fences[0]->wakes={UINT64_MAX};
+  assert(!RebuildSwapChain(s));assert(s.frame_submitted[0]&&s.rings==0&&s.framebuffers[0]==1);
+ }
+}
+'''
+h = a.output / 'resize-fences.cpp'
+h.write_text(source)
+exe = a.output / 'resize-fences.exe'
+subprocess.run(['clang++', '-std=c++20', '-DNOMINMAX', str(h), '-o', str(exe)], check=True)
+subprocess.run([str(exe.resolve())], check=True)
+(a.output / 'verification.json').write_text(json.dumps({
+    'actual_production_bodies': True, 'fake_gpu': True, 'passed': True,
+    'checks': ['completed consumed event', 'stale event and newer fence', 'all three submissions before resize',
+               'open list rejection', 'device removal before/during wait', 'partial retry',
+               'failed/minimized resize retry', 'DXGI dimensions only after success',
+               'backbuffer references restored on failure', 'null backbuffer retry',
+               'no DXGI latency wait']}, indent=2))
+print('PASS: actual fence wait and resize ordering; no game launched.')
diff --git a/tools/gpu-aot/test_resolve_region.cpp b/tools/gpu-aot/test_resolve_region.cpp
new file mode 100644
index 0000000..c162ee4
--- /dev/null
+++ b/tools/gpu-aot/test_resolve_region.cpp
@@ -0,0 +1,23 @@
+#include <cassert>
+#include "gpu_native/resolve_region.h"
+using namespace legodimensions::gpu_native;
+int main() {
+  auto padded = CheckedResolveRegion(640, 368, 640, 360);
+  assert(padded && padded->width == 640 && padded->height == 360);
+  ResolveRect tile{0,0,960,960};
+  for (int i = 0; i < 4; ++i) {
+    ResolvePoint dst{0,960*i};
+    auto r = CheckedResolveRegion(960,960,960,3840,&tile,&dst);
+    assert(r && r->y == 960*i && r->height == 960);
+  }
+  ResolvePoint overflow{0,2881}, negative{-1,0};
+  assert(!CheckedResolveRegion(960,960,960,3840,&tile,&overflow));
+  assert(!CheckedResolveRegion(960,960,960,3840,&tile,&negative));
+  ResolveRect outside{0,0,961,960}, inverted{5,0,4,960};
+  assert(!CheckedResolveRegion(960,960,960,3840,&outside));
+  assert(!CheckedResolveRegion(960,960,960,3840,&inverted));
+  ResolveRect crop{16,32,80,96}; ResolvePoint offset{128,256};
+  auto r = CheckedResolveRegion(960,960,960,3840,&crop,&offset);
+  assert(r && r->left == 16 && r->top == 32 && r->width == 64 &&
+         r->height == 64 && r->x == 128 && r->y == 256);
+}
diff --git a/tools/gpu-aot/test_sampler_state.cpp b/tools/gpu-aot/test_sampler_state.cpp
new file mode 100644
index 0000000..2e28bdc
--- /dev/null
+++ b/tools/gpu-aot/test_sampler_state.cpp
@@ -0,0 +1,119 @@
+// Differential oracle is the actual generated TU23 83FB58A8 body, extracted
+// by extract_sampler_oracle.py. No hand-written copy of its merge algorithm.
+#include <cassert>
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <vector>
+
+#include "gpu_native/sampler_state.h"
+
+using namespace legodimensions::gpu_native;
+
+union OracleRegister {
+  uint64_t u64 = 0;
+  int64_t s64;
+  uint32_t u32;
+  int32_t s32;
+};
+struct OracleCondition {
+  bool eq = false, gt = false, lt = false;
+  template<typename T> void compare(T a, T b, int) { eq = a == b; gt = a > b; lt = a < b; }
+};
+struct OracleContext {
+  OracleRegister r1,r3,r4,r5,r6,r7,r8,r9,r10,r11,r12,r20,r21,r22,r23,r24,r25,r26,r27,r28,r29,r30,r31;
+  uint64_t lr = 0;
+  OracleCondition cr0, cr6;
+  int xer = 0;
+};
+uint32_t Read32(const uint8_t* base, uint32_t at) {
+  assert(at + 4ull <= 0x40000); uint32_t value;
+  std::memcpy(&value, base + at, 4); return __builtin_bswap32(value);
+}
+void Write32(uint8_t* base, uint32_t at, uint32_t value) {
+  assert(at + 4ull <= 0x40000); value = __builtin_bswap32(value);
+  std::memcpy(base + at, &value, 4);
+}
+uint64_t Read64(const uint8_t* base, uint32_t at) {
+  return uint64_t(Read32(base, at)) << 32 | Read32(base, at + 4);
+}
+void Write64(uint8_t* base, uint32_t at, uint64_t value) {
+  Write32(base, at, uint32_t(value >> 32)); Write32(base, at + 4, uint32_t(value));
+}
+#define DEFINE_REX_FUNC(name) void name(OracleContext& ctx, uint8_t* base)
+#define REX_FUNC_PROLOGUE() ((void)0)
+#define REX_LOAD_U32(x) Read32(base, uint32_t(x))
+#define REX_STORE_U32(x,v) Write32(base, uint32_t(x), uint32_t(v))
+#define REX_LOAD_U64(x) Read64(base, uint32_t(x))
+#define REX_STORE_U64(x,v) Write64(base, uint32_t(x), uint64_t(v))
+#define REX_LOAD_U8(x) base[uint32_t(x)]
+// Save/restore only affects registers outside the compared device bytes.
+void __savegprlr_20(OracleContext&, uint8_t*) {}
+void __restgprlr_20(OracleContext&, uint8_t*) {}
+void sub_83FC7048(OracleContext&, uint8_t*) { assert(false && "Xbox queue path entered"); }
+#include "tu23_set_texture_oracle.inc"
+
+int main() {
+  std::vector<uint8_t> memory(0x40000);
+  std::mt19937 random(0x83FB58A8);
+  constexpr uint32_t device = 0x10000, texture = 0x20000;
+  for (uint32_t n = 0; n < 100000; ++n) {
+    const uint32_t slot = n % 26;
+    const uint32_t address = device + 1152 + 24 * slot;
+    TextureFetchWords before, header;
+    for (uint32_t w = 0; w < 6; ++w) {
+      before[w] = random(); header[w] = random();
+      Write32(memory.data(), address + 4*w, before[w]);
+      Write32(memory.data(), texture + 28 + 4*w, header[w]);
+    }
+    const uint8_t min = uint8_t(random()), max = uint8_t(random());
+    memory[device + 12356 + slot] = min; memory[device + 12382 + slot] = max;
+    Write32(memory.data(), device + 12920 + 4*slot, 0);
+    const uint64_t dirty_before = uint64_t(random()) << 32 | random();
+    Write64(memory.data(), device + 24, dirty_before);
+    OracleContext ctx;
+    ctx.r1.u32 = 0x3F000; ctx.r3.u32 = device; ctx.r4.u32 = slot;
+    ctx.r5.u32 = n & 1 ? texture : 0; ctx.r6.u64 = uint64_t(1) << (31 - slot);
+    auto expected = before;
+    if (ctx.r5.u32) expected = MergeTextureFetch(before, header, min, max);
+    else ClearTextureFetch(expected);
+    const uint64_t expected_dirty = ctx.r5.u32 ? dirty_before | ctx.r6.u64 : dirty_before;
+    sub_83FB58A8(ctx, memory.data());
+    for (uint32_t w = 0; w < 6; ++w) assert(Read32(memory.data(), address + 4*w) == expected[w]);
+    assert(Read64(memory.data(), device + 24) == expected_dirty);
+    assert(Read32(memory.data(), device + 12920 + 4*slot) == (n & 1 ? texture : 0));
+  }
+
+  // Known Xenos modes and edge cases beyond a round-trip of our own bitfields.
+  TextureFetchWords f{};
+  f[0] = 2 | (1 << 10) | (6 << 13) | (3 << 16);
+  f[1] = 0x1000; f[2] = 255 | (127 << 13); f[5] = 0x2000 | (1 << 9) | 1;
+  f[3] = (1 << 19) | (0 << 21) | (1 << 23);
+  f[4] = (2 << 2) | (7 << 6) | (uint32_t(-48) & 1023) << 12;
+  auto d = DecodeTextureSampler(f);
+  assert(d.addressU == plume::RenderTextureAddressMode::MIRROR);
+  assert(d.addressV == plume::RenderTextureAddressMode::BORDER);
+  assert(d.addressW == plume::RenderTextureAddressMode::CLAMP);
+  assert(d.minFilter == plume::RenderFilter::NEAREST);
+  assert(d.magFilter == plume::RenderFilter::LINEAR);
+  assert(d.minLOD == 2 && d.maxLOD == 7 && d.mipLODBias == -1.5f);
+  assert(d.borderColor == plume::RenderBorderColor::OPAQUE_WHITE);
+  const auto key = TextureSamplerKey(d);
+  // Moving an identical texture must not allocate a new sampler descriptor.
+  f[1] += 0x100000; f[5] += 0x100000;
+  assert(TextureSamplerKey(DecodeTextureSampler(f)) == key);
+  f[3] |= 5 << 25;
+  d = DecodeTextureSampler(f);
+  assert(d.anisotropyEnabled && d.maxAnisotropy == 16);
+  assert(d.minFilter == plume::RenderFilter::LINEAR);
+  f[5] = (3 << 9); f[4] |= 15 << 6;
+  d = DecodeTextureSampler(f);
+  assert(d.addressU == plume::RenderTextureAddressMode::CLAMP);
+  assert(d.addressV == plume::RenderTextureAddressMode::CLAMP);
+  assert(d.addressW == plume::RenderTextureAddressMode::CLAMP);
+  assert(d.minLOD == 0 && d.maxLOD == 0); // no mip page
+  f[3] = 2 << 23;
+  d = DecodeTextureSampler(f);
+  assert(d.maxLOD == 0.25f && d.mipmapMode == plume::RenderMipmapMode::NEAREST);
+  std::puts("TU23 SetTexture differential: 100000 cases passed; sampler modes passed");
+}
diff --git a/tools/gpu-aot/test_shader_lookup_hashes.py b/tools/gpu-aot/test_shader_lookup_hashes.py
new file mode 100644
index 0000000..0654c6b
--- /dev/null
+++ b/tools/gpu-aot/test_shader_lookup_hashes.py
@@ -0,0 +1,55 @@
+"""Run the production lookup against colliding prefixes and same-call hash reuse."""
+import argparse
+from pathlib import Path
+import subprocess
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+a.output.mkdir(parents=True,exist_ok=True)
+root=Path(__file__).resolve().parents[2]
+src=(root/'rexlego/src/gpu_native/shader_archive.cpp').read_text()
+start=src.index('const ShaderCacheEntry* FindShaderByMicrocode(')
+end=src.index('\nuint64_t HashShaderContainer(',start)
+body=src[start:end].replace('XXH3_64bits(', 'CountHash(')
+code=r'''
+#include <cassert>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+#include <array>
+#define XXH_INLINE_ALL
+#include <xxhash.h>
+#include "gpu_native/shader_archive.h"
+std::vector<ShaderMicrocodeEntry> entries;
+namespace {
+ShaderMicrocodeEntry* g_shaderMicrocodeEntries;
+size_t g_shaderMicrocodeEntryCount;
+unsigned calls=0;
+uint64_t CountHash(const void* p,size_t n){++calls;return XXH3_64bits(p,n);}
+ShaderCacheEntry hit{9,0,0,0,0,0};
+const ShaderCacheEntry* FindShader(uint64_t id){return id==9?&hit:nullptr;}
+''' + body + r'''
+}
+int main(){
+ std::array<uint8_t,512> bytes{};
+ bytes[20]=23;
+ auto prefix=XXH3_64bits(bytes.data(),8);
+ for(unsigned i=0;i<3000;++i)
+   entries.push_back({prefix,0,100+i,128u<<(i%3),i%2,0});
+ entries.push_back({prefix,XXH3_64bits(bytes.data(),512),9,512,0,0});
+ g_shaderMicrocodeEntries=entries.data();g_shaderMicrocodeEntryCount=entries.size();
+ assert(FindShaderByMicrocode(bytes.data(),0)==&hit);
+ assert(calls<=4); // Prefix plus at most three lengths, not 1501 hashes.
+ calls=0;
+ assert(FindShaderByMicrocode(bytes.data(),1)==nullptr);assert(calls==4);
+ bytes[20]^=1;calls=0;
+ assert(FindShaderByMicrocode(bytes.data(),0)==nullptr);assert(calls==4);
+ bytes[20]^=1;calls=0;
+ assert(FindShaderByMicrocode(bytes.data(),0)==&hit);assert(calls<=4);
+ assert(!FindShaderByMicrocode(nullptr,0));
+}
+'''
+code=code.replace('g_shaderMicrocodeEntries','fixtureEntries').replace('g_shaderMicrocodeEntryCount','fixtureCount')
+cpp=a.output/'lookup.cpp';cpp.write_text(code);exe=a.output/'lookup.exe'
+subprocess.run(['clang++','-std=c++20','-I'+str(root/'rexlego/src'),
+ '-I'+str(root/'rexglue-sdk/thirdparty/xxHash'),str(cpp),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+print('PASS: actual lookup, 3001 colliding prefixes, stage/missing/success and same-address rewrites; <=4 hashes per lookup')
diff --git a/tools/gpu-aot/test_shader_texture_usage.cpp b/tools/gpu-aot/test_shader_texture_usage.cpp
new file mode 100644
index 0000000..7fd83a5
--- /dev/null
+++ b/tools/gpu-aot/test_shader_texture_usage.cpp
@@ -0,0 +1,35 @@
+#include <array>
+#include <cassert>
+#include <vector>
+#include "../../rexlego/src/gpu_native/shader_texture_usage.h"
+using namespace legodimensions::gpu_native;
+void write(std::vector<uint8_t>& code, size_t at, uint32_t value) {
+  for (int i = 0; i < 4; ++i) code[at + i] = uint8_t(value >> (24 - i * 8));
+}
+void cf(std::vector<uint8_t>& code, size_t pair, uint32_t a, uint32_t ah,
+        uint32_t b = 0, uint32_t bh = 0) {
+  write(code, pair * 12, a);
+  write(code, pair * 12 + 4, ah | (b << 16));
+  write(code, pair * 12 + 8, (b >> 16) | (bh << 16));
+}
+int main() {
+  constexpr uint32_t all = ~uint32_t{0};
+  assert(ShaderTextureUsage({}) == all);
+  std::vector<uint8_t> code(60);
+  // A conditional END must not hide the second exec clause. Include slot 31.
+  cf(code, 0, 2 | (1 << 12) | (1 << 16), 4 << 12,
+              3 | (2 << 12) | (5 << 16), 2 << 12);
+  write(code, 24, 1 | (7 << 20));
+  write(code, 36, 19 | (31u << 20));
+  write(code, 48, 0); // vertex fetch does not consume a texture slot
+  assert(ShaderTextureUsage(code) == ((1u << 7) | (1u << 31)));
+  write(code, 24, 3); // unknown fetch opcode: conservative fallback
+  assert(ShaderTextureUsage(code) == all);
+  cf(code, 0, 9 | (1 << 12) | (1 << 16), 1 << 12);
+  assert(ShaderTextureUsage(code) == all);
+  code.resize(59);
+  assert(ShaderTextureUsage(code) == all);
+  code.assign(36, 0);
+  cf(code, 0, 2 | (1 << 12), 2 << 12); // ALU-only program
+  assert(ShaderTextureUsage(code) == 0);
+}
diff --git a/tools/gpu-aot/test_stencil_state.py b/tools/gpu-aot/test_stencil_state.py
new file mode 100644
index 0000000..532f7f9
--- /dev/null
+++ b/tools/gpu-aot/test_stencil_state.py
@@ -0,0 +1,162 @@
+"""Execute TU23 stencil leaves and the native decoder without launching a game."""
+import json
+from pathlib import Path
+import re
+import struct
+import subprocess
+import sys
+
+root = Path(__file__).resolve().parents[2]
+out = Path(sys.argv[1]).resolve()
+out.mkdir(parents=True, exist_ok=True)
+def body(s, sig):
+    start = s.index(sig)
+    end = s.index('{', start) + 1
+    depth = 1
+    while depth:
+        depth += (s[end] == '{') - (s[end] == '}')
+        end += 1
+    return s[start:end]
+
+image = (root / 'xexdump/dump/default.bin').read_bytes()
+table = [struct.unpack_from('>III', image, 0x847F9B18-0x82000000+12*k)
+         for k in range(27, 43)]
+functions = {}
+for p in (root / 'rexlego/generated/default').glob('*.cpp'):
+    s = p.read_text()
+    for _, setter, _ in table:
+        sig = f'DEFINE_REX_FUNC(sub_{setter:08X})'
+        if sig in s:
+            f = body(s, sig)
+            instructions = re.findall(r'^\s*// ([^\n]+)', f, re.M)
+            for instruction in instructions:
+                branch = re.match(r'b[l]? 0x([a-fA-F0-9]+)', instruction)
+                if branch:
+                    assert setter <= int(branch[1],16) < setter+4*len(instructions)
+                assert not re.match(r'bctrl|bctr', instruction)
+            assert not re.search(r'\bsub_\w+\(ctx, base\)', f)
+            functions[setter] = f
+assert len(functions) == 16
+initializer = body((root/'rexlego/src/gpu_native/hooks_device.cpp').read_text(),
+                   'bool InitializeStencilDefaults(')
+rebind = body((root/'rexlego/generated/default/legodimensions_recomp.324.cpp').read_text(),
+              'DEFINE_REX_FUNC(sub_83FBAE38)')
+start = rebind.index('\t// lwz r9,12308(r31)')
+end = rebind.index('\t// ld r11,16(r31)', rebind.index('\t// stw r11,10548(r31)', rebind.index('\t// lwz r11,12312(r31)', start)))
+oracle = 'void OriginalRebind(PPCContext& ctx,u8* base) { PPCRegister temp{};\n' + rebind[start:end] + '\n}\n'
+assert 'RebindDepthStencilControl' in (root/'rexlego/src/gpu_native/hooks_state.cpp').read_text()
+source = r'''
+#include <array>
+#include <cassert>
+#include <cstdint>
+#include <vector>
+#include "gpu_native/stencil_state.h"
+#include "gpu_native/depth_state.h"
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;
+union PPCRegister {uint64_t u64;int64_t s64;uint32_t u32;int32_t s32;uint8_t u8;};
+struct XER {u8 ca=0;};
+struct CR {bool eq=false;template<class T>void compare(T a,T b,XER){eq=a==b;}};
+struct PPCContext {
+''' + ''.join(f'PPCRegister r{i}{{}};\n' for i in range(32)) + r'''
+ XER xer;CR cr0,cr6;
+};
+std::array<u8,65536> ram{};
+u32 Load(u32 p){return (u32(ram[p])<<24)|(u32(ram[p+1])<<16)|(u32(ram[p+2])<<8)|ram[p+3];}
+void Store(u32 p,u32 v){for(u32 i=0;i<4;++i)ram[p+i]=u8(v>>(24-8*i));}
+u64 Load64(u32 p){return (u64(Load(p))<<32)|Load(p+4);}
+void Store64(u32 p,u64 v){Store(p,u32(v>>32));Store(p+4,u32(v));}
+#define REX_LOAD_U32(p) Load(p)
+#define REX_LOAD_U8(p) (ram[p])
+#define REX_STORE_U32(p,v) Store(p,v)
+#define REX_LOAD_U64(p) Load64(p)
+#define REX_STORE_U64(p,v) Store64(p,v)
+#define REX_STORE_U8(p,v) (ram[p]=u8(v))
+#define REX_FUNC_PROLOGUE() ((void)0)
+#define DEFINE_REX_FUNC(n) void n(PPCContext& ctx,u8* base)
+#define REXLOG_INFO(...) ((void)0)
+''' + '\n'.join(functions.values()) + oracle + r'''
+struct be_u32 {u32 raw;operator u32()const{return __builtin_bswap32(raw);}};
+constexpr u32 kRenderStateTable=4096;
+std::vector<u32> calls;
+struct Dispatcher {u32 missing=0;u32 GetFunction(u32 p){return p==missing?0:p;}} dispatcher;
+struct Kernel {auto function_dispatcher(){return &dispatcher;}} fixture_kernel;
+#define REX_KERNEL_STATE() (&fixture_kernel)
+namespace rex::ppc {
+template<class T>void GuestToHostFunction(u32 f,u32 device,u32 value){
+ PPCContext c{};c.r3.u32=device;c.r4.u32=value;calls.push_back(f);
+''' + '\n'.join((' if' if i==0 else ' else if') +
+                f'(f==0x{t[1]:08X})sub_{t[1]:08X}(c,ram.data());'
+                for i,t in enumerate(table)) + r'''
+ else assert(false);
+}
+}
+''' + initializer + r'''
+using namespace legodimensions::gpu_native;
+int main(){
+ constexpr u32 device=12288;
+''' + ''.join(f'Store(kRenderStateTable+12*{27+i}+4,{t[1]}u);Store(kRenderStateTable+12*{27+i}+8,{t[2]}u);\n'
+               for i,t in enumerate(table)) + r'''
+ assert(InitializeStencilDefaults(device,ram.data()));assert(calls.size()==16);
+ assert(Load(device+10496)==0x00FFFF00 && Load(device+10492)==0x00FFFF00);
+ assert(Load(device+10548)==0x00700700); // ALWAYS both faces, KEEP, disabled.
+ dispatcher.missing=0x83FB8828;calls.clear();
+ assert(!InitializeStencilDefaults(device,ram.data()));assert(calls.empty());
+ dispatcher.missing=0;
+ // Actual enable leaf preserves requested enable even with no depth target.
+ PPCContext c{};c.r3.u32=device;c.r4.u32=1;
+ sub_83FB8570(c,ram.data());assert(Load(device+12312)==1);
+ assert((Load(device+10548)&1)==0);
+ Store(device+12832,1);sub_83FB8570(c,ram.data());
+ assert(Load(device+10548)&1);
+ for(u32 ref=0;ref<256;++ref){
+  c.r4.u32=ref;sub_83FB8788(c,ram.data());
+  assert(DecodeStencilState(Load(device+10548),Load(device+10496),true,true).reference==ref);
+ }
+ // Exercise independent front/back compare and op fields through real setters.
+ for(u32 value=0;value<8;++value){
+  c.r4.u32=value;
+  sub_83FB85E8(c,ram.data());sub_83FB8618(c,ram.data());
+  sub_83FB8650(c,ram.data());sub_83FB8688(c,ram.data());
+  auto s=DecodeStencilState(Load(device+10548),Load(device+10496),true,true);
+  assert(s.enabled&&s.front.comparison==value&&s.front.fail==value&&s.front.pass==value&&s.front.depth_fail==value);
+  assert(s.back==s.front);
+  c.r4.u32=7-value;
+  sub_83FB86B8(c,ram.data());sub_83FB86E8(c,ram.data());
+  sub_83FB8720(c,ram.data());sub_83FB8758(c,ram.data());
+  c.r4.u32=1;sub_83FB85B0(c,ram.data());
+  s=DecodeStencilState(Load(device+10548),Load(device+10496),true,true);
+  assert(s.back.comparison==7-value&&s.back.fail==7-value&&s.back.pass==7-value&&s.back.depth_fail==7-value);
+  assert(DecodeStencilState(Load(device+10548),Load(device+10496),true,false).back==s.front);
+  assert(!DecodeStencilState(Load(device+10548),Load(device+10496),false,true).enabled);
+  c.r4.u32=0;sub_83FB85B0(c,ram.data());
+ }
+ for(u32 mask=0;mask<256;++mask){
+  c.r4.u32=mask;sub_83FB87A8(c,ram.data());
+  c.r4.u32=255-mask;sub_83FB87C8(c,ram.data());
+  auto s=DecodeStencilState(Load(device+10548),Load(device+10496),true,true);
+  assert(s.read_mask==mask&&s.write_mask==255-mask);
+ }
+ // Execute the original DS-rebind tail, including its PPC masking, against
+ // the host helper. Preserve requested bits across unbind/rebind sequences.
+ for(u32 control : {0u,0x00700700u,0xC07E07B3u,0xFFFFFFFFu})
+ for(u32 depth_request : {0u,1u,2u,3u,0xFFFFFFFFu})
+ for(u32 stencil_request : {0u,1u,2u,3u,0xFFFFFFFFu})
+ for(u32 surface : {0u,1u,0xAB001234u}) {
+  Store(device+10548,control);Store(device+12308,depth_request);
+  Store(device+12312,stencil_request);Store(device+12832,surface);
+  c.r31.u32=device;OriginalRebind(c,ram.data());
+  assert(Load(device+10548)==RebindDepthStencilControl(control,depth_request,stencil_request,surface!=0));
+  assert(Load(device+12308)==depth_request&&Load(device+12312)==stencil_request);
+ }
+}
+'''
+cpp = out/'test.cpp'
+cpp.write_text(source)
+exe = out/'test.exe'
+subprocess.run(['clang++','-std=c++20','-O2','-I'+str(root/'rexlego/src'),str(cpp),'-o',str(exe)],check=True)
+subprocess.run([str(exe)],check=True)
+(out/'verification.json').write_text(json.dumps({'passed':True,'game_launched':False,
+    'original_setters':16,'reference_cases':256,'mask_cases':256,'operation_values':8,
+    'original_rebind_cases':300,
+    'note':'CPU semantics/defaults only; no GPU pixels or missing-mesh claim'},indent=2))
+print('Stencil TU23 leaf/default/decoder tests PASS')
diff --git a/tools/gpu-aot/test_texture_header_refresh.py b/tools/gpu-aot/test_texture_header_refresh.py
new file mode 100644
index 0000000..91de100
--- /dev/null
+++ b/tools/gpu-aot/test_texture_header_refresh.py
@@ -0,0 +1,120 @@
+"""Exercise the production refresh body with real before/after TU23 headers.
+
+The bounded harness stubs GPU allocation, not the cache mutation under test.
+GPU fence completion still requires the separate descriptor-retirement test
+and the in-game AFK reproduction.
+"""
+import argparse
+import json
+from pathlib import Path
+import re
+import subprocess
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('log', type=Path)
+parser.add_argument('output', type=Path)
+parser.add_argument('--compiler', default='clang++')
+args = parser.parse_args()
+root = Path(__file__).resolve().parents[2]
+text = (root / 'rexlego/src/gpu_native/textures.cpp').read_text()
+begin = text.index('void RefreshTextureHeader(u32 guest_address) {')
+end = text.index('\nTextureResourceView ResolveTextureResource', begin)
+body = text[begin:end]
+rows = re.findall(r'header changed guest=([0-9A-F]+).*?cached=([0-9A-F,]+) current=([0-9A-F,]+)',
+                  args.log.read_bytes().decode('utf-8', errors='replace'))
+assert rows, 'Need actual runtime headers'
+args.output.mkdir(parents=True, exist_ok=True)
+harness = r'''
+#include <array>
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+#include <string>
+bool LongProbeEnabled() { return false; }
+template<class... T> void LongProbeEvent(T&&...) {}
+template<class T> std::string LongProbeHex(const T&) { return {}; }
+using u32=uint32_t;
+struct D3DTexture { struct { u32 common=3; } resource; struct { std::array<u32,6> dword; } format; } header;
+struct TextureResource {
+  bool owns_guest_memory=false,surface=false,resolved_on_host=false,guest_uploaded=true,upload_source_key_valid=true;
+  u32 adopted_header_type=3,descriptor_index=4,resolved_descriptor_index=5,sampled_descriptor_index=6,mirror_address=0;
+  std::vector<u32> previous_surface_descriptors{7};
+  std::mutex mutex; std::array<u32,6> guest_fetch;
+};
+std::mutex g_textures_mutex;
+std::unordered_map<u32,std::shared_ptr<TextureResource>> g_textures;
+std::vector<std::shared_ptr<TextureResource>> retired;
+std::vector<u32> descriptors;
+u32 framebuffer_invalidations=0;
+struct Memory { void SystemHeapFree(u32) { assert(false); } } memory;
+#define REX_KERNEL_MEMORY() (&memory)
+#define REXLOG_INFO(...) ((void)0)
+struct HostDevice {
+  static int LockRecording() { return 0; }
+  static void UnregisterTexture(u32 slot) { descriptors.push_back(slot); }
+  static void RetireResource(std::shared_ptr<TextureResource> value) { retired.push_back(value); }
+};
+template<class T> T* GuestAt(u32) { return &header; }
+std::shared_ptr<TextureResource> FindTexture(u32 a) { auto i=g_textures.find(a); return i==g_textures.end()?nullptr:i->second; }
+void InvalidateFramebufferReferences(u32) { ++framebuffer_invalidations; }
+'''
+harness += body + r'''
+void Check(std::array<u32,6> before,std::array<u32,6> after,bool replace,bool host=false,bool owned=false,u32 type=3) {
+  g_textures.clear(); retired.clear(); descriptors.clear(); framebuffer_invalidations=0;
+  auto old=std::make_shared<TextureResource>(); old->guest_fetch=before;
+  old->resolved_on_host=host; old->owns_guest_memory=owned;
+  header.format.dword=after; header.resource.common=type; g_textures.emplace(1,old);
+  RefreshTextureHeader(1);
+  if(owned || (before==after && type==3)) {
+    assert(FindTexture(1)==old && old->guest_fetch==before && old->upload_source_key_valid);
+    assert(retired.empty() && descriptors.empty()); return;
+  }
+  if(replace) {
+    assert(!FindTexture(1)); assert(retired.size()==1 && retired[0]==old);
+    assert((descriptors==std::vector<u32>{4,5,6,7}) && framebuffer_invalidations==1);
+  } else {
+    assert(FindTexture(1)==old && old->guest_fetch==after);
+    assert(!old->upload_source_key_valid && old->guest_uploaded==host);
+    assert(retired.empty() && descriptors.empty());
+  }
+}
+int main() {
+'''
+relocations = replacements = 0
+for guest, cached, current in rows:
+    old, new = [[int(w, 16) for w in value.split(',')] for value in (cached, current)]
+    assert len(old) == len(new) == 6
+    # Observed shape change is independently visible in the runtime words.
+    replace = any(old[i] != new[i] for i in (0, 2, 3, 4))
+    assert (old[1] & 0xFFF) == (new[1] & 0xFFF)
+    assert (old[5] & 0xFFF) == (new[5] & 0xFFF)
+    replacements += replace
+    relocations += not replace
+    a, b = ['{' + ','.join(hex(w) + 'u' for w in value) + '}' for value in (old, new)]
+    for host in ('false', 'true'):
+        harness += f'Check({a},{b},{str(replace).lower()},{host}); // guest {guest}\n'
+    harness += f'Check({a},{b},false,false,true);\nCheck({a},{a},false);\n'
+# Address-only-looking transitions may add/remove the complete mip allocation.
+harness += r'''
+Check({2,0xF4000052u,0x001FE0FFu,0xD10,0x200,0xA00},
+      {2,0xF4000052u,0x001FE0FFu,0xD10,0x200,0xF4100A00u},true);
+Check({2,0xF4000052u,0x001FE0FFu,0xD10,0x200,0xF4100A00u},
+      {2,0xF4000052u,0x001FE0FFu,0xD10,0x200,0xA00},true);
+Check({2,3,4,5,6,7},{2,3,4,5,6,7},true,false,false,6);
+}
+'''
+source = args.output / 'refresh-harness.cpp'
+source.write_text(harness, encoding='utf-8')
+exe = args.output / 'refresh-harness.exe'
+subprocess.run([args.compiler, '-std=c++20', '-DNOMINMAX', str(source), '-o', str(exe)], check=True)
+subprocess.run([str(exe.resolve())], check=True)
+report = {'runtime_header_pairs': len(rows), 'relocations': relocations,
+          'shape_replacements': replacements, 'fixture_calls': len(rows) * 4 + 3,
+          'production_body_extracted': True, 'gpu_fences_and_gameplay_verified': False}
+(args.output / 'verification.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
+print(json.dumps(report))
diff --git a/tools/gpu-aot/test_texture_lod.py b/tools/gpu-aot/test_texture_lod.py
new file mode 100644
index 0000000..8aa86bb
--- /dev/null
+++ b/tools/gpu-aot/test_texture_lod.py
@@ -0,0 +1,98 @@
+"""End-to-end explicit-LOD fixtures for the isolated compiler candidate.
+
+Checks actual microcode -> HLSL -> validated DXIL, using only synthetic inputs.
+All output stays in scratch; this does not establish game image correctness.
+"""
+import argparse
+import os
+from pathlib import Path
+import re
+import struct
+import subprocess
+
+
+def fixture(*, computed=False, register=True, predicated=False, bias=0,
+            vertex=False, relative=False):
+    data = bytearray(176)
+    struct.pack_into('>9I', data, 0, 0x102A1100 | int(vertex), 128, 48, 0, 36, 0, 80, 0, 0)
+    struct.pack_into('>8I', data, 36, 32, 28, 0, 0, 0, 0, 0, 0)
+    struct.pack_into('>8I', data, 80, 0, 48, 0, 0xFF00, 0, 0, 0, 1)
+    # EXEC_END: setTexLOD, tfetch2D, ALU export. CF1 is NOP.
+    struct.pack_into('>3I', data, 128, 1 | (3 << 12) | (5 << 16), 2 << 12, 0)
+    set0 = 24 | (7 << 5) | (int(relative) << 11) | (2 << 26)  # source r7.z
+    set1 = int(predicated) << 31
+    set2 = int(predicated) << 31
+    struct.pack_into('>3I', data, 140, set0, set1, set2)
+    fetch0 = 1 | (8 << 5) | (2 << 12) | (3 << 20) | (4 << 26)
+    fetch1 = 0x688 | (3 << 12) | (3 << 14) | (3 << 16)
+    fetch1 |= int(computed) << 28 | int(register) << 29
+    fetch2 = ((bias & 127) << 2) | (1 << 14)
+    struct.pack_into('>3I', data, 152, fetch0, fetch1, fetch2)
+    # MAX r2, r2 exported as color0 or oPos; ordinary register operands.
+    struct.pack_into('>3I', data, 164, (62 if vertex else 0) | (1 << 15) | (15 << 16), 0,
+                     2 | (2 << 8) | (2 << 16) | (2 << 24) | (7 << 29))
+    return data
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('compiler', type=Path)
+    parser.add_argument('common', type=Path)
+    parser.add_argument('scratch', type=Path)
+    args = parser.parse_args()
+    inputs = args.scratch / 'input'
+    hlsl = args.scratch / 'hlsl'
+    inputs.mkdir(parents=True, exist_ok=True)
+    cases = {
+        'explicit_register': {},
+        'explicit_zero': {'register': False},
+        'predicated_set': {'predicated': True},
+        'negative_instruction_bias': {'bias': -8},
+        'positive_instruction_bias': {'bias': 24},
+        'computed_pixel_lod': {'computed': True, 'register': False},
+        'vertex_computed_is_explicit': {'computed': True, 'register': False, 'vertex': True},
+    }
+    for name, opts in cases.items():
+        (inputs / f'{name}.bin').write_bytes(fixture(**opts))
+    env = os.environ.copy()
+    env['XENOS_RECOMP_DXIL_ONLY'] = '1'
+    env['XENOS_RECOMP_LEGO_NATIVE_SCALE'] = '1'
+    env['XENOS_RECOMP_VERBOSE_DXC'] = '1'
+    env.pop('XENOS_RECOMP_ONLY_HASH', None)
+    result = subprocess.run([str(args.compiler.resolve()), str(inputs.resolve()),
+                             str((args.scratch / 'fixture-cache.cpp').resolve()),
+                             str(args.common.resolve()), str(hlsl.resolve())],
+                            env=env, text=True, capture_output=True)
+    if result.returncode:
+        raise RuntimeError(result.stdout + result.stderr)
+    for name, opts in cases.items():
+        source = (hlsl / f'{name}.hlsl').read_text()
+        assert 'float textureLod = 0.0;' in source, name
+        assert 'textureLod = r7.z;' in source, name
+        assert 'g_FetchLodBiasArr[8] : packoffset(c39)' in source, name
+        computed = opts.get('computed', False) and not opts.get('vertex', False)
+        call = re.search(r'r2\.xyzw = (tfetch\w+)\(([^\n]+)', source)
+        assert call, name
+        if computed:
+            assert call.group(1) == 'tfetch2D', name
+            assert 'g_FetchLodBias(3)' not in call.group(2), name
+        else:
+            assert call.group(1) == 'tfetchLevel2D', name
+            lod = 'textureLod' if opts.get('register', True) else '0.0'
+            assert f'{lod} + g_FetchLodBias(3)' in call.group(2), name
+            assert str(opts.get('bias', 0) / 16).rstrip('0').rstrip('.') in call.group(2), name
+        if opts.get('predicated'):
+            assert re.search(r'if \(p0\)\s*\{\s*textureLod = r7.z;', source), name
+    # Reject a source addressing mode that this candidate cannot translate.
+    rejected = args.scratch / 'rejected-input'
+    rejected.mkdir(parents=True, exist_ok=True)
+    (rejected / 'relative.bin').write_bytes(fixture(relative=True))
+    failure = subprocess.run([str(args.compiler.resolve()), str(rejected.resolve()),
+                              str((args.scratch / 'rejected-cache.cpp').resolve()),
+                              str(args.common.resolve())], env=env, text=True, capture_output=True)
+    assert failure.returncode != 0 and 'relative setTexLOD unsupported' in failure.stderr
+    print(f'Explicit LOD regression passed: {len(cases)} HLSL/DXIL fixtures, relative-source refusal')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/gpu-aot/test_texture_number.cpp b/tools/gpu-aot/test_texture_number.cpp
new file mode 100644
index 0000000..d6169d7
--- /dev/null
+++ b/tools/gpu-aot/test_texture_number.cpp
@@ -0,0 +1,28 @@
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include "../../rexlego/src/gpu_native/texture_number.h"
+using namespace legodimensions::gpu_native;
+float transfer(float value, int resolve_exp, int sample_exp, bool integer) {
+  const float unorm = std::clamp(value * Unsigned16ResolveScale(resolve_exp, integer), 0.0f, 1.0f);
+  const float stored = std::round(unorm * 65535.0f) / 65535.0f;
+  return stored * Unsigned16SampleScale(sample_exp, integer);
+}
+int main() {
+  assert(ColorOutputScale(5u << 16) == 1.0f / 32);
+  assert(ColorOutputScale(4u << 16) == 1.0f / 32);
+  assert(ColorOutputScale(7u << 16) == 1);
+  assert(ColorExponentBias(59u << 20) == -5);
+  assert(ColorOutputScale((5u << 16) | (5u << 20)) == 1);
+  assert(ColorOutputScale((7u << 16) | (59u << 20)) == 1.0f / 32);
+  // Captured HDR: UINT16 resolve +11, fetch integer -13. Expected /4,
+  // retaining the 16-bit quantization, not the former saturate(value*2048).
+  for (float value : {0.0f, 0.1f, 1.0f, 16.0f, 30.391f})
+    assert(std::abs(transfer(value, 11, -13, true) - value * 0.25f) < 1.0f / 8192.0f);
+  assert(transfer(-1, 11, -13, true) == 0);
+  assert(std::abs(transfer(100, 11, -13, true) - 65535.0f / 8192.0f) < 0.00001f);
+  assert(Unsigned16ResolveScale(0, false) == 1);
+  assert(Unsigned16SampleScale(0, false) == 1);
+  assert(Unsigned16SampleScale(5, false) == 32); // half-resolution color fetch
+  assert(std::abs(transfer(0.5f, 0, 5, false) - 16) < 0.001f);
+}
diff --git a/tools/gpu-aot/test_texture_upload_cache.cpp b/tools/gpu-aot/test_texture_upload_cache.cpp
new file mode 100644
index 0000000..88661ae
--- /dev/null
+++ b/tools/gpu-aot/test_texture_upload_cache.cpp
@@ -0,0 +1,36 @@
+#define XXH_INLINE_ALL
+#include "../../rexlego/src/gpu_native/texture_upload_cache.h"
+
+#include <array>
+#include <cassert>
+#include <iostream>
+
+int main() {
+  using namespace legodimensions::gpu_native;
+  std::array<uint8_t, 6 * 256> base{};  // All cube faces, not only face zero.
+  std::array<uint8_t, 512> mips{};
+  std::array<uint8_t, 24> fetch{};
+  const auto initial = MakeTextureUploadSourceKey(base, mips, fetch);
+  assert(initial == MakeTextureUploadSourceKey(base, mips, fetch));
+  for (size_t offset : {size_t{0}, size_t{256}, base.size() - 1}) {
+    ++base[offset];
+    assert(initial != MakeTextureUploadSourceKey(base, mips, fetch));
+    --base[offset];
+  }
+  for (size_t offset : {size_t{0}, mips.size() - 1}) {
+    ++mips[offset];
+    assert(initial != MakeTextureUploadSourceKey(base, mips, fetch));
+    --mips[offset];
+  }
+  ++fetch[20];  // Changing fetch semantics must invalidate identical bytes.
+  assert(initial != MakeTextureUploadSourceKey(base, mips, fetch));
+  --fetch[20];
+  const auto other_allocation = base;
+  assert(initial == MakeTextureUploadSourceKey(other_allocation, mips, fetch));
+  const auto base_only = MakeTextureUploadSourceKey(base, {}, fetch);
+  assert(initial != base_only);
+  assert(base_only == MakeTextureUploadSourceKey(base, {}, fetch));
+  assert(initial != MakeTextureUploadSourceKey(
+      std::span<const uint8_t>(base).first(base.size() - 1), mips, fetch));
+  std::cout << "Texture upload source-cache regressions passed\n";
+}
diff --git a/tools/gpu-aot/test_texture_upload_capture.py b/tools/gpu-aot/test_texture_upload_capture.py
new file mode 100644
index 0000000..b69cc1c
--- /dev/null
+++ b/tools/gpu-aot/test_texture_upload_capture.py
@@ -0,0 +1,67 @@
+"""Execute native DDS capture with padded BC1/BC3/RGBA rows, no GPU or game."""
+import argparse
+import json
+from pathlib import Path
+import struct
+import subprocess
+
+p=argparse.ArgumentParser(description=__doc__)
+p.add_argument('output',type=Path)
+a=p.parse_args()
+a.output.mkdir(parents=True,exist_ok=True)
+root=Path(__file__).resolve().parents[2]
+s=(root/'rexlego/src/gpu_native/textures.cpp').read_text()
+start=s.index('void DumpTextureUpload(')
+end=s.index('\nbool EnsureMirror',start)
+function=s[start:end]
+cpp=a.output/'test.cpp'
+cpp.write_text(r'''
+#include <cstdint>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <mutex>
+#include <string>
+#include <unordered_set>
+#include <vector>
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;
+namespace plume {enum class RenderFormat {BC1_UNORM,BC3_UNORM,R8G8B8A8_UNORM};}
+enum class ShaderStage {kVertex,kPixel};
+u64 vs=0,ps=0;
+u64 BoundShaderHash(ShaderStage stage){return stage==ShaderStage::kVertex?vs:ps;}
+struct Fetch {u32 base_address=0,mip_address=0,swizzle=0x688,endianness=0,pitch=1,tiled=1,packed_mips=0;};
+struct TextureResource {plume::RenderFormat format;u32 guest_address,width=5,height=7,guest_format=20;Fetch guest_fetch;};
+u32 RowPitch(plume::RenderFormat,u32){return 256;}
+u32 RowCount(plume::RenderFormat f,u32 h){return f==plume::RenderFormat::R8G8B8A8_UNORM?h:(h+3)/4;}
+#define REXLOG_INFO(...) ((void)0)
+'''+function+r'''
+int main(int argc,char**argv){
+ _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS",argv[1]);
+ _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY","1");
+ std::vector<u8> data(256*7);
+ for(size_t i=0;i<data.size();++i)data[i]=u8(i/256*31+i%256);
+ TextureResource r{plume::RenderFormat::BC3_UNORM,999};
+ DumpTextureUpload(r,data.data()); // Unrelated shader must not consume quota.
+ vs=0x91007ACB3E640E3D;ps=0xEEE573BE160E037D;
+ r.guest_address=1;DumpTextureUpload(r,data.data());
+ r.format=plume::RenderFormat::BC1_UNORM;r.guest_address=2;DumpTextureUpload(r,data.data());
+ vs=0xC3958E2D1B795ED9;ps=0x0C1840BF35E84F2F;
+ r.format=plume::RenderFormat::R8G8B8A8_UNORM;r.guest_address=3;DumpTextureUpload(r,data.data());
+}
+''')
+exe=a.output/'test.exe'
+subprocess.run(['clang++','-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True)
+data=a.output/'dds'
+subprocess.run([str(exe.resolve()),str(data.resolve())],check=True)
+assert not (data/'999.dds').exists()
+for name,code,row_bytes,rows in [('1',b'DXT5',32,2),('2',b'DXT1',16,2),('3',b'\0'*4,20,7)]:
+    b=(data/(name+'.dds')).read_bytes()
+    assert b[:4]==b'DDS ' and b[84:88]==code
+    assert struct.unpack_from('<II',b,12)==(7,5)
+    assert len(b)==128+row_bytes*rows
+    expected=bytes((row*31+column)&255 for row in range(rows) for column in range(row_bytes))
+    assert b[128:]==expected
+(a.output/'verification.json').write_text(json.dumps({'passed':True,'formats':['BC3/DXT5','BC1/DXT1','RGBA8'],
+    'checks':['odd dimensions','256-byte row padding stripped','exact payload preserved','both TT shader variants','unrelated shader filtered'],
+    'game_launched':False},indent=2)+'\n')
+print('PASS: DXT5/DXT1/RGBA capture preserves exact rows and filters unrelated shaders')
diff --git a/tools/gpu-aot/test_texture_volume_upload.cpp b/tools/gpu-aot/test_texture_volume_upload.cpp
new file mode 100644
index 0000000..0ca43d7
--- /dev/null
+++ b/tools/gpu-aot/test_texture_volume_upload.cpp
@@ -0,0 +1,65 @@
+#include "../../rexlego/src/gpu_native/texture_volume_upload.h"
+
+#include <cassert>
+#include <iostream>
+#include <vector>
+
+using legodimensions::gpu_native::CopyTextureVolumeBlocks;
+
+void TestLinearVolume(uint32_t width, uint32_t height) {
+  // Test both the observed 64x32x16 LUT and a 16^3 regression whose guest
+  // Z slices have 32 padded rows while host upload slices have only 16.
+  constexpr uint32_t depth = 16, pitch = 256;
+  auto offset = [](uint32_t x, uint32_t y, uint32_t z) -> int64_t {
+    return (uint64_t(z) * 32 + y) * pitch + x * 4;
+  };
+  for (uint32_t endian = 0; endian < 4; ++endian) {
+    std::vector<uint8_t> source(size_t(offset(width - 1, height - 1, depth - 1)) + 4, 0xCD);
+    std::vector<uint8_t> destination(pitch * height * depth, 0xFE);
+    for (uint32_t z = 0; z < depth; ++z)
+      for (uint32_t y = 0; y < height; ++y)
+        for (uint32_t x = 0; x < width; ++x) {
+          const uint8_t rgba[] = {uint8_t(x * 17), uint8_t(y * 17), uint8_t(z * 17), 255};
+          for (uint32_t b = 0; b < 4; ++b)
+            source[(size_t(offset(x, y, z)) + b) ^ endian] = rgba[b];
+        }
+    assert(CopyTextureVolumeBlocks(source, destination, width, height, depth, 4, pitch, endian, offset));
+    for (uint32_t z = 0; z < depth; ++z)
+      for (uint32_t y = 0; y < height; ++y) {
+        const size_t row = (z * height + y) * pitch;
+        for (uint32_t x = 0; x < width; ++x) {
+          assert(destination[row + x * 4] == uint8_t(x * 17));
+          assert(destination[row + x * 4 + 1] == uint8_t(y * 17));
+          assert(destination[row + x * 4 + 2] == z * 17);
+          assert(destination[row + x * 4 + 3] == 255);
+        }
+        if (width * 4 < pitch)
+          assert(destination[row + width * 4] == 0xFE); // Do not copy guest padding.
+      }
+    auto short_source = std::span<const uint8_t>(source).first(source.size() - 1);
+    assert(!CopyTextureVolumeBlocks(short_source, destination, width, height, depth, 4, pitch, endian, offset));
+    auto short_dest = std::span<uint8_t>(destination).first(destination.size() - 1);
+    assert(!CopyTextureVolumeBlocks(source, short_dest, width, height, depth, 4, pitch, endian, offset));
+    assert(!CopyTextureVolumeBlocks(source, destination, width, height, depth, 4, 4, endian, offset));
+    assert(!CopyTextureVolumeBlocks(source, destination, width, height, depth, 4, pitch, endian,
+                                   [](auto, auto, auto) { return int64_t{-1}; }));
+  }
+}
+
+int main() {
+  TestLinearVolume(16, 16);
+  TestLinearVolume(64, 32);
+  // A callback can address a packed mip / non-linear block order too. The
+  // production callback uses the SDK's GetTiledOffset3D, not a 2D slice tiler.
+  std::vector<uint8_t> packed(128, 0), output(8);
+  auto packed_offset = [](uint32_t x, uint32_t y, uint32_t z) {
+    return int64_t(64 + z + 4 * y + 2 * x);
+  };
+  for (uint32_t z = 0; z < 2; ++z)
+    for (uint32_t y = 0; y < 2; ++y)
+      for (uint32_t x = 0; x < 2; ++x)
+        packed[size_t(packed_offset(x, y, z))] = uint8_t(z * 4 + y * 2 + x);
+  assert(CopyTextureVolumeBlocks(packed, output, 2, 2, 2, 1, 2, 0, packed_offset));
+  for (uint32_t i = 0; i < 8; ++i) assert(output[i] == i);
+  std::cout << "Volume upload regressions passed\n";
+}
diff --git a/tools/gpu-aot/test_tile_extent.cpp b/tools/gpu-aot/test_tile_extent.cpp
new file mode 100644
index 0000000..5885754
--- /dev/null
+++ b/tools/gpu-aot/test_tile_extent.cpp
@@ -0,0 +1,26 @@
+#include "../../rexlego/src/gpu_native/tile_extent.h"
+#include <array>
+#include <cassert>
+#include <iostream>
+
+using namespace legodimensions::gpu_native;
+int main() {
+  std::array tiles{NativeTileRect{0, 0, 1280, 512}, NativeTileRect{0, 512, 1280, 720}};
+  auto extent = LogicalTileExtent(tiles);
+  assert(extent.width == 1280 && extent.height == 720);
+  std::swap(tiles[0], tiles[1]);
+  assert(LogicalTileExtent(tiles).height == 720);
+  tiles[0].top = 513; // Hole between the guest tiles.
+  assert(!LogicalTileExtent(tiles).width);
+  tiles[0].top = 511; // Overlap is not a full-frame partition.
+  assert(!LogicalTileExtent(tiles).width);
+  tiles[0].top = -1;
+  assert(!LogicalTileExtent(tiles).width);
+  assert(!LogicalTileExtent({}).width);
+  std::array grid{NativeTileRect{0,0,640,360}, NativeTileRect{640,0,1280,360},
+                  NativeTileRect{0,360,640,720}, NativeTileRect{640,360,1280,720}};
+  assert(LogicalTileExtent(grid).height == 720);
+  grid[3].right = 16385;
+  assert(!LogicalTileExtent(grid).width);
+  std::cout << "tile extent tests passed\n";
+}
diff --git a/tools/gpu-aot/test_tonemap_dof.cpp b/tools/gpu-aot/test_tonemap_dof.cpp
new file mode 100644
index 0000000..4281203
--- /dev/null
+++ b/tools/gpu-aot/test_tonemap_dof.cpp
@@ -0,0 +1,29 @@
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <cstring>
+#include "gpu_native/tonemap_dof.h"
+using legodimensions::gpu_native::ApplyDisabledTonemapDof;
+int main() {
+  std::array<float,1024> original;
+  for (unsigned i=0;i<original.size();++i) original[i]=float(i)*0.125f;
+  original[48]=1;original[49]=0;original[50]=0.2f;original[51]=7;
+  auto host=original;
+  constexpr std::uint64_t hash=0x3A47E5DDE66B42C6ull;
+  assert(!ApplyDisabledTonemapDof(hash,false,host.data(),sizeof(host))&&host==original);
+  assert(!ApplyDisabledTonemapDof(hash+1,true,host.data(),sizeof(host))&&host==original);
+  assert(!ApplyDisabledTonemapDof(hash,true,host.data(),199)&&host==original);
+  assert(!ApplyDisabledTonemapDof(hash,true,nullptr,sizeof(host)));
+  assert(ApplyDisabledTonemapDof(hash,true,host.data(),sizeof(host)));
+  for(unsigned i=0;i<host.size();++i)
+    assert(host[i]==((i==48||i==49)?0:original[i]));
+  // Actual translated mix, including fixed16's finite fetch exponent range.
+  for(int i=-4096;i<=4096;++i) {
+    const float alpha=float(i)/128;
+    const float w=std::clamp(2*(alpha*host[48]+host[49])-1,0.0f,1.0f);
+    assert(w==0);
+  }
+  assert(original[48]==1&&original[50]==0.2f); // Guest/source never touched.
+  host=original; // Re-enabled setting uses a fresh draw snapshot.
+  assert(!ApplyDisabledTonemapDof(hash,false,host.data(),sizeof(host))&&host==original);
+}
diff --git a/tools/gpu-aot/test_vertex_byte_order.cpp b/tools/gpu-aot/test_vertex_byte_order.cpp
new file mode 100644
index 0000000..a02ed4e
--- /dev/null
+++ b/tools/gpu-aot/test_vertex_byte_order.cpp
@@ -0,0 +1,39 @@
+#include "../../rexlego/src/gpu_native/vertex_byte_order.h"
+#include <array>
+
+using namespace legodimensions::gpu_native;
+
+constexpr bool TestColor() {
+  // Two vertices: untouched float bits followed by the observed logo tint.
+  std::array<uint8_t, 16> data{0, 0, 0, 0xBF, 0x1A, 0xFE, 0xFE, 0xFE,
+                              0, 0, 0, 0x3F, 0x34, 0xFE, 0xFE, 0xFE};
+  const std::array<uint32_t, 1> fields{4};
+  return ApplyVertexByteOrder(data.data(), data.size(), {8, 0, fields}) &&
+      data == std::array<uint8_t, 16>{0, 0, 0, 0xBF, 0xFE, 0xFE, 0xFE, 0x1A,
+                                    0, 0, 0, 0x3F, 0xFE, 0xFE, 0xFE, 0x34};
+}
+
+constexpr bool TestOffsetAndTail() {
+  std::array<uint8_t, 11> data{9, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+  const std::array<uint32_t, 1> fields{0};
+  return ApplyVertexByteOrder(data.data(), data.size(), {4, 2, fields}) &&
+      data == std::array<uint8_t, 11>{9, 9, 4, 3, 2, 1, 8, 7, 6, 5, 9};
+}
+
+constexpr bool TestInvalid() {
+  std::array<uint8_t, 4> data{1, 2, 3, 4};
+  const std::array<uint32_t, 1> fields{1};
+  return !ApplyVertexByteOrder(data.data(), data.size(), {4, 0, fields}) &&
+      !ApplyVertexByteOrder(data.data(), data.size(), {0, 0, fields}) &&
+      !ApplyVertexByteOrder(data.data(), data.size(), {8, 5, fields}) &&
+      ApplyVertexByteOrder(data.data(), data.size(), {}) &&
+      data == std::array<uint8_t, 4>{1, 2, 3, 4};
+}
+
+static_assert(TestColor());
+static_assert(TestOffsetAndTail());
+static_assert(TestInvalid());
+static_assert(((0x00014C86 >> 10) & 0xFFF) == 0x053); // WZYX.
+static_assert((0x00014C86 & 0x300) == 0); // Unsigned, normalized.
+static_assert((0x00014E86 & 0x300) == 0x200); // Unsigned, integer.
+int main() {}
diff --git a/tools/gpu-aot/test_viewport_state.py b/tools/gpu-aot/test_viewport_state.py
new file mode 100644
index 0000000..94761e0
--- /dev/null
+++ b/tools/gpu-aot/test_viewport_state.py
@@ -0,0 +1,96 @@
+"""Run TU23 viewport setter and emitted HLSL math offline, without a game."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+
+
+def body(s, signature):
+    start = s.index(signature)
+    end = s.index('{', start) + 1
+    depth = 1
+    while depth:
+        depth += (s[end] == '{') - (s[end] == '}')
+        end += 1
+    return s[start:end]
+
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('common', type=Path)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+root = Path(__file__).resolve().parents[2]
+a.output.mkdir(parents=True, exist_ok=True)
+original = body((root/'rexlego/generated/default/legodimensions_recomp.70.cpp').read_text(),
+                'DEFINE_REX_FUNC(sub_83FB8FD0)')
+helper = body(a.common.read_text(), 'float4 LegoViewportPosition(')
+draw = (root/'rexlego/src/gpu_native/draw.cpp').read_text()
+key = body(draw, 'struct PipelineKey {')+';\n'+body(draw, 'struct PipelineKeyHash {')+';\n'
+cpp = a.output/'test.cpp'
+cpp.write_text(r'''
+#include <array>
+#include <cassert>
+#include <cmath>
+#include <cstdint>
+#include <unordered_map>
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using uint=uint32_t;
+constexpr u32 kNativeRenderTargets=4,kNativeVertexStreams=16;
+union PPCRegister {u64 u64;int64_t s64;u32 u32;int32_t s32;};
+struct CR {bool eq=false;template<class T>void compare(T a,T b,int){eq=a==b;}};
+struct PPCContext {PPCRegister r3{},r4{},r10{},r11{};CR cr6;int xer=0;};
+std::array<u8,65536> ram{};
+u32 Load(u32 p){return u32(ram[p])<<24|u32(ram[p+1])<<16|u32(ram[p+2])<<8|ram[p+3];}
+void Store(u32 p,u32 v){for(u32 i=0;i<4;++i)ram[p+i]=u8(v>>(24-8*i));}
+u64 Load64(u32 p){return (u64(Load(p))<<32)|Load(p+4);}
+void Store64(u32 p,u64 v){Store(p,u32(v>>32));Store(p+4,u32(v));}
+#define REX_LOAD_U32(p) Load(p)
+#define REX_STORE_U32(p,v) Store(p,v)
+#define REX_LOAD_U64(p) Load64(p)
+#define REX_STORE_U64(p,v) Store64(p,v)
+#define REX_FUNC_PROLOGUE() ((void)0)
+#define DEFINE_REX_FUNC(n) void n(PPCContext& ctx,u8* base)
+struct float2 {float x,y;};
+float2 operator*(float2 a,float2 b){return {a.x*b.x,a.y*b.y};}
+float2 operator*(float2 a,float b){return {a.x*b,a.y*b};}
+float2 operator+(float2 a,float2 b){return {a.x+b.x,a.y+b.y};}
+struct float4 {float2 xy;float z,w;};
+'''+original+'\n'+helper+'\n'+key+r'''
+int main(){
+ constexpr u32 dev=12288;
+ for(u32 initial:{0u,0x80000u,0xFFFFFFFFu,0xA5317788u})
+ for(u32 enable:{0u,1u,2u,0xFFFFFFFFu}){
+  Store(dev+10564,initial);Store64(dev+16,0xF010000000000001);
+  PPCContext c;c.r3.u32=dev;c.r4.u32=enable;sub_83FB8FD0(c,ram.data());
+  assert(Load(dev+10572)==(enable?0x43Fu:0x400u));
+  assert(Load(dev+10564)==((initial&~0x10000u)|(enable?0:0x10000u)));
+  assert(Load64(dev+16)==0xF0100000000000A1);
+ }
+ for(float width:{128.f,720.f,1280.f,1920.f})
+ for(float height:{128.f,720.f,1080.f})
+ for(float w:{0.5f,1.f,2.f})
+ for(float x:{-64.f,0.f,0.5f,33.f,1280.f})
+ for(float y:{-64.f,0.f,0.5f,55.f,720.f}){
+  float4 input={{x*w,y*w},0.3f*w,w};
+  auto out=LegoViewportPosition(input,{2/width,-2/height},1);
+  // Independent D3D12 viewport maps NDC back to the supplied pixel position.
+  float pixel_x=(out.xy.x/out.w+1)*width/2;
+  float pixel_y=(1-out.xy.y/out.w)*height/2;
+  assert(std::abs(pixel_x-x)<0.001f&&std::abs(pixel_y-y)<0.001f);
+  assert(out.z==input.z&&out.w==input.w);
+  auto unchanged=LegoViewportPosition(input,{2/width,-2/height},0);
+  assert(unchanged.xy.x==input.xy.x&&unchanged.xy.y==input.xy.y);
+ }
+ PipelineKey a,b;b.clip_disable=1;
+ std::unordered_map<PipelineKey,int,PipelineKeyHash> cache;
+ cache[a]=1;cache[b]=2;assert(cache.size()==2&&cache[a]==1&&cache[b]==2);
+}
+''')
+exe = a.output/'test.exe'
+subprocess.run(['clang++','-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True)
+subprocess.run([str(exe)],check=True)
+proof = {'passed': True, 'original_setter_cases': 16, 'pixel_mapping_cases': 900,
+         'modes': ['TU23 0x43F', 'TU23 0x400'], 'shared_bytes': 624,
+         'depth_clip_pipeline_cache': True, 'game_launched': False,
+         'scope': 'CPU setter, exact emitted HLSL arithmetic, host viewport mapping and PSO identity; no pixel/character repair claim'}
+(a.output/'verification.json').write_text(json.dumps(proof,indent=2)+'\n')
+print('PASS: TU23 viewport setter, 900 screen coordinates, unchanged normal mode and distinct clip PSOs')
diff --git a/tools/gpu-aot/tiling-answer-20260930.md b/tools/gpu-aot/tiling-answer-20260930.md
new file mode 100644
index 0000000..e0a1bca
--- /dev/null
+++ b/tools/gpu-aot/tiling-answer-20260930.md
@@ -0,0 +1,1212 @@
+# Ответ на вопрос tiling-reference-20260929.md (TU23, 2026-09-30)
+
+Метод: скан сгенерированного кода (rexlego/generated/default) на load/store по
+смещениям device +13124/+13128/+13368/+13556/+13560/+13564 в диапазоне
+0x83F00000–0x84100000, потом ручной просмотр каждой ветки. Совпадения в
+0x840xxxxx — это другие структуры с теми же смещениями, их не учитывать.
+Статический анализ, в игре НЕ проверялось.
+
+## Что пишет оригинал
+
+BeginTiling (83FBCD28): +13100, +13104, +13108, +13112, +13116 (RT на время
+тайлинга), +13120 (DS на время тайлинга), +13124 (count), rects с +13128,
+origins с +13368, +13556/+13560 (extent), +13564 (flags), +13588.
+Сам вызывает 83FBA618, 83FBA9F8, 83FBC8A0 (→ Clear-путь).
+
+EndTiling (83FBD098): из этого набора сбрасывает ТОЛЬКО +13100 (=0). count
+(+13124), flags (+13564), extent и rects остаются устаревшими после End.
+Значит «тайлинг активен» потребители определяют НЕ по count != 0, а по
++13100 и/или по совпадению текущих RT/DS (+12828/+12832) с +13116/+13120.
+Внутри End: SetPredication (83FBCBB8), 83FB94F8, Resolve (83FBF0D8),
+InsertCallback (83FC0F38), KickOff (83FC0C10), 83FC0540, стор +13772,
++13084..+13092, +11060/+11064/+11068/+11071, в конце 83FCBA68(count).
+
+## Незахваченные потребители
+
+| Функция | Поле | Как читает | Кто вызывает |
+|---|---|---|---|
+| 83FBA710 | extent | если RT==+13116 и DS==+13120 → берёт extent вместо размера поверхности | 83FBA978 (SetViewport, ЗАХВАЧЕН), 83FBA9F8 (не захвачен, зовут 83FB2898, 83FB3B48, Begin) |
+| 83FBC8A0 | extent | тот же флаг (байт-аргумент) → прямоугольник клира = extent | Clear 83FBC9D8 (захвачен), Begin |
+| 83FBC220 → 83FBBC80 | count, origins (+13368/+13372), +13548/+13552 | цикл по тайлам, эмитит пакеты на каждый тайл, дальше 83FBBA18 и 83FD0BF8 | 83FBC8A0 (клир-путь) |
+| 83FB7AB8 | count, rects-массив с +13132 | цикл по тайлам, внутри KickOff 83FC0C10 | 83FB62F8 (←83FB6DD0), SetScissor 83FBA068 (захвачен), 83FBA618, 83FBC220, ResolveEx 83FBDF80, 83FC5048 (←EndExport, захвачен) |
+| 83FD0BF8 | count | цикл count-1..0, эмиссия в командный буфер | 83FBBC80, 83FBC220, 83FBDD20, ResolveEx 83FBDF80, 83FC2C80, 83FC6180, Draw* (захвачены), 83FCFBB8 |
+| 83FAFB08 | count | если бит 0x20 в флагах → count, иначе 1 (множитель резерва места в ринге) | 83FC7580 ← Swap (захвачен) |
+| 83FCAB10 StartWorkerQueue | count, flags, origins | — | именно он крашил (lr=83FCABCC) |
+| 83FBDF80 ResolveEx | origins, count | через 83FB7AB8/83FD0BF8 | только Resolve 83FBF0D8 (захвачен) |
+
+Опасные при отсутствии очередей: всё, что ведёт в 83FB7AB8 (KickOff внутри),
+83FD0BF8 (пишет в командный буфер) и 83FCAB10. Незахваченные входы туда:
+83FBA618, 83FB62F8/83FB6DD0, 83FC2C80/83FC38D0, 83FC6180/83FB2898,
+83FCFBB8/83FD0480, 83FBDD20. 83FBA9F8→83FBA710 только читает extent — безопасно.
+
+## Вывод / контракт
+
+1. Минимально безопасно: ничего из +13124..+13564 не писать. Все известные
+   читатели extent/count из захваченных точек (Viewport, Clear, Scissor,
+   Draw*, Resolve, EndExport, Swap) уже подменены хуками; оставшиеся
+   незахваченные читатели count — как раз эмиттеры пакетов/KickOff, которым
+   ненулевой count даёт повод работать.
+2. Если писать (например, чтобы 83FBA9F8/83FBA710 давали полный viewport):
+   достаточно +13116/+13120 (RT/DS тайлинга) и +13556/+13560 (extent);
+   count оставить 0 — 83FB7AB8/83FBBC80/83FD0BF8 при count==0 пропускают циклы.
+   В End сбросить +13100=0 как оригинал и дополнительно (отклонение от
+   оригинала, но безопаснее) +13116/+13120=0.
+3. flags +13564 читают только Begin/End/83FCAB10 — не писать.
+4. До реализации проверить: вызываются ли 83FBA618 и 83FB6DD0 в рантайме во
+   время кадра (лог/брейкпоинт) — если да, их хукать или гарантировать count==0.
+
+Непроверено: точная семантика +13100 (фаза? флаг?), потребители через
+указатели/vtable (скан ловит только прямые смещения от базы device).
+
+# Дополнение 2 (2026-09-30): порядок clear и привязок вокруг BeginTiling
+
+Статический анализ, в игре не проверено.
+
+## Кто вызывает Begin/End
+
+Единственный игровой вызов идёт через `sub_82B6A7B0` (переключение группы
+render targets; группа — объект игры, текущая лежит в глобале 0x849505F8).
+Тонкие обёртки: `82B648C8` → b BeginTiling, `82B648E8` → b EndTiling.
+
+Порядок внутри 82B6A7B0(obj, bool direct):
+1. если у старой группы тайлинг (поле +308) → EndTiling (82B648E8), иначе заглушка 837F4088.
+2. current = obj.
+3. если obj+292 (dirty) или direct → 82B6A0C8(obj) — (пере)создание поверхностей группы.
+4. direct=1 → 83FBB110 (пакетная установка 5 поверхностей obj+20/76/132/188/244), Begin НЕ вызывается.
+   direct=0 → цикл 5× `82B698A0` (шаг 56 байт) → 82B64778 = b 83FBB758 = b SetRenderTarget 83FBAAA8
+   и 82B64818 = b SetDepthStencil 83FBAE38.
+5. direct=0 и obj+308 указывает на тайлинг → BeginTiling(flags=1, rects=obj+312) через 82B648C8.
+
+Итого: RT0–RT3 и DS **привязаны ДО Begin**, clear в самом Begin пропущен (flags=1).
+
+## Где clear
+
+Пример прохода сцены `82B685A8`: 82B6A7B0 (Set RTs → Begin) → `82B64FD0` (обёртка clear)
+→ 82B6A980 / 82B69FB0 / … → отрисовка. То есть **clear идёт ПОСЛЕ Begin, внутри тайлинга,
+до геометрии**. Пересоздание attachment в BeginTilingHook уже выполненный clear НЕ теряет —
+при условии, что пересоздание делается синхронно в Begin, а не лениво на первом draw/clear.
+Остальные вызыватели 82B64FD0 (20 штук, 82B65230…82BCB010) — не проверял поштучно.
+
+Что очищает 82B64FD0 (все вызовы Clear 83FBC9D8 с count=0, rects=0 = весь extent):
+- основной: flags собираются из аргумента: 0x1 (RT0, цвет из аргумента r6),
+  0x30 (Z+stencil), 0x20 (stencil), 0x40/0x80 (hi-stencil), вокруг — 83FB9560/83FB9590.
+- если группа obj+728 == 1 (MRT-режим): ещё два Clear:
+  flags=0x2 (RT1) цвет 0x0080FF00, flags=0x4 (RT2) цвет 0.
+Это совпадает с резолвами RT0/RT1/RT2/depth из трейса. Биты flags трактованы по
+заголовкам X360 SDK (TARGET0..3 = 1/2/4/8, ZBUFFER 0x10, STENCIL 0x20) — сверить с ClearHook.
+
+## Viewport — поправка принята
+
+Да, 83FBA710 сравнивает все пять: RT0–RT3 (+12816..+12828) с +13104..+13116 и DS +12832 с +13120;
+Begin пишет все пять +13104..+13120. Мой прошлый ответ (+13116/+13120) был неполным.
+Хранить логический extent только на host и не писать гостевые count/flags — согласен:
+все читатели extent (83FBA710 через SetViewport, 83FBC8A0 через Clear) на захваченных путях.
+Host-правило «тайлинг активен» = снимок пяти привязок в Begin совпадает с текущими.
+
+# Дополнение 3 (2026-09-30): 0x1A20AB55 и Resolve(0x100)
+
+## 0x1A20AB55 — разбор D3DFORMAT
+
+биты 0–5 = 21 = `k_16_16_16_16_EDRAM`; 6–7 = 1 (endian 8in16); бит 8 = tiled;
+знаки X/Y/Z/W = 1 (SIGNED) у всех четырёх; бит 17 NumFormat = 0 (fraction);
+swizzle XYZW. Экспоненты в самом D3DFORMAT нет — она в RB_COLOR_INFO[20:25]
+(при записи) и в exp_adjust fetch-константы (при чтении).
+
+Семантика (xenia, xenos.h:307 и d3d12/texture_cache.cpp:184): цветовой RT
+`k_16_16_16_16` = **фиксированная точка −32…+32** (0x8001 = −32, 0x7FFF = +32),
+НЕ 0…1. `_EDRAM`-текстура — тот же формат, «−32…32 range».
+
+Как делает xenia в режиме host RT (command_processor.cpp:4702–4719, cvar
+snorm16_render_target_full_range=true по умолчанию):
+- host-формат **R16G16B16A16_SNORM** (render_target_cache.cpp:1465), не UNORM;
+- на выходе PS цвет умножается на `2^(color_exp_bias − 5)`, т.е. RB_COLOR_INFO
+  exp_bias и **дополнительно /32**, чтобы −32…32 легло в −1…1 SNORM;
+- resolve копирует сырые биты; для тракта truncated −1…1 xenia добавляет +5 к
+  copy_dest_exp_bias (util/draw.cpp:1066–1079), в full-range — нет;
+- чтение: знаковый fetch даёт −1…1, fetch exp_adjust **+5** (= ×32) восстанавливает
+  −32…32. Поэтому у tonemap-fetch exp=+5.
+
+У вас: UNORM без /32 → значения >1 клипаются в 1.0 при записи, потом fetch ×32 →
+**всё ×32 = белый экран**. Плюс UNORM теряет отрицательные значения и неверно
+читает signed-биты. Фикс: RT → R16G16B16A16_SNORM, в PS для этого RT умножать
+вывод на 2^(exp_bias)/32 (exp_bias из RB_COLOR_INFO, у 360 это signed 6 бит);
+при чтении как текстуры — SNORM view + exp_adjust из fetch как есть.
+То же самое для RG16 (`k_16_16`, тоже −32…32).
+Минус (признан в xenia): блендинг в /32-пространстве не точен, для HDR-сцены обычно терпимо.
+
+## Resolve(flags=0x100)
+
+0x100 = D3DRESOLVE_CLEARRENDERTARGET (резолв RT0 + очистка), по SDK.
+Игровые вызовы Resolve идут через обёртку `82B64860` (флаги в r3 игры) из:
+- `82BD5BE8` — общий «резолв рендер-текстуры»: к флагам добавляет exponent-bias в
+  старших битах — `0x2C000000` если тип объекта ([obj+12]) == 117, `0xEC000000` если 118;
+  **dst = obj+120 + 52*[obj+116]** — массив 52-байтных D3D-заголовков текстур ВНУТРИ
+  игрового объекта (кольцо по +116).
+- `82B65F28` — flags=0, dst тоже [..+44]+120+52*[+116].
+Вызыватели 82BD5BE8: 82BD6800, 82BD69A0 (флаги = база|индекс RT или |4 для depth,
+256 добавляется условно), 82B66200 (flags = r11+256 | маска), 82B66AC0, 82B68F30,
+82B68988, 82B690D8 (эти четыре с flags=0).
+
+Почему dst AC4B9A48 «не текстура»: заголовок не создан через CreateTexture — он
+встроен в игровой объект и, вероятно, заполняется напрямую (кандидат — 83FB57C8,
+его зовёт 82B6A0C8 при (пере)создании поверхностей группы; не проверено, что это
+XGSetTextureHeader). Нужно хукать инициализацию заголовка, а не только CreateTexture.
+
+Экспозиция/readback: статически не доказано. Чтобы отличить, залогируй lr цепочки
+(82B64860 → 82BD5BE8 → кто) и [obj+12] для этого вызова, и есть ли потом Lock/чтение
+CPU по base этой текстуры. RG16 1-пиксельный/маленький → типично адаптация яркости;
+она тоже `k_16_16` (−32…32) — при UNORM+×32 экспозиция развалится так же.
+
+# Addendum 4 (2026-09-30): signed EDRAM RT -> unsigned texture, RB_COLOR_INFO location
+
+Static analysis of generated TU23 code + rexglue-sdk xenia sources. Not verified in game.
+
+## 1. Resolve must convert numerically, not copy SNORM bits
+
+Xenos resolve is a format conversion, not a memcpy: it reads the EDRAM value in the
+RT's own range (k_16_16_16_16 = fixed -32..32), multiplies by 2^copy_dest_exp_bias,
+then packs into copy_dest_format per copy_dest_number (unsigned repeating fraction
+= UNORM, clamp 0..1). xenia only takes the raw-copy path when exp_bias == 0 AND the
+dest number format matches (rexglue-sdk src/graphics/util/draw.cpp:1132-1178,
+ColorResolveNumberFormatMatches); otherwise it runs the full conversion shader.
+
+Where the scale comes from: the game's resolve wrapper `82BD5BE8` ORs an exponent
+bias into the top 6 bits of the D3D Resolve flags (bits 31:26, signed):
+- object type [obj+12] == 118 -> `oris 0xEC00` -> 0xEC000000 -> bias = 0b111011 = **-5**
+- object type [obj+12] == 117 -> `oris 0x2C00` -> 0x2C000000 -> bias = 0b001011 = +11
+  (meaning unclear; possibly a different packing or a deliberate large bias - check the trace)
+
+So the consistent chain for the half-colour texture is:
+EDRAM value v in -32..32 -> resolve x2^-5 -> v/32 -> clamp to 0..1 -> UNORM16 ->
+fetch unsigned, exp_adjust +5 -> x32 -> v (negatives lost, which the game accepts).
+That matches ACEAEF2C: format 26, num 0, sign 0,0,0,0, exp +5.
+
+Consequence for native: if the host RT stores v/32 in SNORM (the xenia full-range
+convention), the resolve into the unsigned texture must apply (copy_exp_bias + 5),
+i.e. for bias -5 it's a plain SNORM->UNORM re-encode with clamp at 0 (net factor 1),
+NOT a bit copy (SNORM bits read as UNORM are wrong for negatives and off by 2x).
+Take the bias from the actual Resolve flags (top 6 bits) per call; don't hardcode.
+
+## 2. RB_COLOR_INFO shadow in the guest device
+
+Written by SetRenderTarget `83FBAAA8(device, index, surface)` (hooked by native):
+- device+12816+4*i = surface pointer (RT0..RT3; DS at +12832).
+- device[(2593 + k)*4] = **surface+28** (u32, big-endian), where k = 0 for RT0, i+1 otherwise:
+  - RT0 -> device+**10372**
+  - RT1 -> device+**10380**
+  - RT2 -> device+**10384**
+  - RT3 -> device+**10388**
+  (gap after RT0 matches the register order RB_COLOR_INFO=0x2001, RB_COLOR1..3_INFO=0x2003..0x2005.)
+- Then a dirty bit is set in the 64-bit mask at device+16.
+- For formats 2/3/10/12 (2_10_10_10 family) the stored word is patched according to a
+  per-slot value at device+12408+4*i (gamma/float variant switch); 16_16_16_16 is not patched.
+- Also RB_COLOR_MASK shadow: device+10460, one nibble per RT, taken from per-slot
+  masks at device+12292+4*i (zeroed when the slot's surface is null).
+
+Field layout of that word (same as RB_COLOR_INFO): bits 19:16 = ColorRenderTargetFormat,
+bits 25:20 = color_exp_bias (signed 6-bit), low bits = EDRAM base. The source is
+surface+28, i.e. it's filled at surface creation from D3DSURFACE_PARAMETERS.ColorExpBias.
+
+Practical: in SetRenderTargetHook read the big-endian u32 at surface+28 directly
+(no need to read the device shadow); PS output scale for slot i =
+2^(exp_bias_i) * (format == 16_16 or 16_16_16_16 ? 1/32 : 1). Only fixed-16 formats get the /32.
+
+# Addendum 5 (2026-09-30): D3DSURFACE_PARAMETERS and how 83FB52C0 builds surface+28
+
+Correction: 83FB57C8 is GetSurfaceDesc (per the GPU agent), not a texture-header
+initializer. The destination-header initializer for obj+120+52*idx is still unknown.
+
+Signature as used (inferred from register use in TU23 `83FB52C0`):
+`83FB52C0(width r3, height r4, D3DFORMAT r5, msaa r6, const D3DSURFACE_PARAMETERS* r7,
+D3DSurface* r8, u32* out_tiles r9, u32* out_hiz_size r10)`
+
+## D3DSURFACE_PARAMETERS (r7), all big-endian 32-bit
+| off | field | used as |
+|---|---|---|
+| +0 | Base (EDRAM tile base) | `& 0xFFF` into surface+28 |
+| +4 | HierarchicalZBase | depth only, surface+32 |
+| +8 | ColorExpBias (signed int) | colour only, `& 0x3F` -> bits 25:20 of surface+28 |
+| +12 | HiZ func/flags (depth only) | if nonzero: (value-1)&1 into surface+32 |
+
+**parameters == NULL defaults:** Base = 0, ColorExpBias = 0, HierarchicalZBase = -1
+(no HiZ) for colour; for depth formats HiZ base = -1 only if the global u32 at
+0x84A6A644 is nonzero, otherwise 0. (Verify that address: lis -31577 / -22972.)
+
+## surface+28 for colour formats (D3DFORMAT low 6 bits != 22/23)
+```
+tf        = D3DFORMAT & 0x3F            // 54 is remapped to 7 first
+rt_fmt    = (u16 table @0x824C9660[tf] >> 8) & 0xF   // ColorRenderTargetFormat
+if rt_fmt == 0 (8_8_8_8) && (D3DFORMAT & 0x0003FE00) == 0x00007E00
+   (sign X,Y,Z = 3 = GAMMA, sign W = 0, NumFormat bit 17 = 0; tests at 83FB54C8..83FB54FC)
+   -> rt_fmt = 1 (8_8_8_8_GAMMA)
+surface+28 = (Base & 0xFFF) | (rt_fmt << 16) | ((ColorExpBias & 0x3F) << 20)
+```
+That is exactly the RB_COLOR_INFO layout; SetRenderTarget copies it verbatim to the
+device shadow (+10372/+10380/+10384/+10388, see Addendum 4).
+For 0x1A20AB55 (tf 21, 16_16_16_16_EDRAM) the table should give rt_fmt 5
+(k_16_16_16_16) - read the table in guest memory to confirm, don't assume.
+
+## Depth formats (tf 22 = 24_8, tf 23 = 24_8_FLOAT)
+surface+28 = (Base & 0xFFF) | (depth_is_float_from_table << 16)
+surface+32 = HiZ info: 0 if HiZ base == -1, else (HiZBase << 13 | func bit) << 4 | 1.
+
+## Other fields written (for a faithful simplified surface)
++0 = 4 (resource type: surface), +4 = 1 (refcount), +20 = 0xFFFF0000,
++24 = (msaa | pitch_tiles<<2) << 16 | pitch-derived value (80- or 40-sample tile rounding by msaa),
++36 = (width-1) | (height-1) << 14 | old low 3 bits kept,
++40 = D3DFORMAT, +44 = size_in_tiles * 5120 (size from 82B64CB0), *out_tiles = size_in_tiles.
+
+## Recommendation for CreateSurfaceHook (83FB5690)
+Don't reimplement the table guess: fill surface+28 with the formula above using
+params (or the NULL defaults) and rt_fmt taken from the same u16 table at 0x824C9660.
+Then SetRenderTargetHook can read exp_bias/format from surface+28 consistently, and
+the PS output scale is 2^ColorExpBias (/32 extra only for 16_16 and 16_16_16_16).
+
+# Addendum 6 (2026-09-30): who fills placement-shader physical sections
+
+Static analysis of TU23 generated code. Not verified at runtime.
+
+## Answer: CPU memcpy by the game's own pool allocator. Not memexport, not code generation.
+
+Only one game function creates placement shaders: **`82BD73F0`** (the only caller of
+XGSetVertexShaderHeader `83FB7618` and XGSetPixelShaderHeader `83FB74F0`).
+Arguments: r3 = owner object (its [+8] leads to the pool), r4 = target record (shader
+objects stored at +104 VS, +108 PS), r6 = VS container, r10 = PS container (optional),
+stack byte 327(r1) picks the path.
+
+### Path A — fresh load (byte 327(r1) != 0)
+Per stage (VS first, then PS):
+1. `83FB67A0` = XGGetShaderInfo(container, info): info+0 = container, +4 = virtual size,
+   **+8 = physical source = container + [container+4]**, **+12 = physical size = [container+8]**.
+2. Allocate the SDK object + virtual section (82A4BB98, align 4, tag 55),
+   store it in record+104/+108, then XGSet*ShaderHeader(obj, size, info) copies the virtual section.
+3. Physical section, pool = `82CE86C0(ctx[+8]) + 3092`:
+   - `82CEC738(pool, phys_size, obj, phys_src)` -> allocates a handle (82CEC2B8), then if
+     phys_src != 0: lock the block (82CEBFC0), **memcpy(dst, phys_src, phys_size)** (83E45090),
+     unlock (82CEC170). This is the ONLY writer of the microcode.
+   - `82CEC7D0(pool, handle)` -> lock again, returns the CPU pointer into the block.
+   - `83FB7658(obj, ptr)` = XGRegisterVertexShader (PS: `83FB67F0` = XGRegisterPixelShader);
+     stores ptr at obj+0x20 (VS) / +0x18 (PS) — the "backing" your hook reads.
+   - `82CEC838(pool, handle)` -> unlock.
+### Path B — reuse (byte 327(r1) == 0)
+`82B7A5D0(owner)` returns an existing entry; the shader objects are **embedded in an
+already-filled block** at [entry+4]->[0] + 64; the pool handle is entry+12. The code
+locks it (82CEC7D0), re-registers only if a flag byte is set (VS: arg r9, PS: 311(r1)),
+unlocks. No memcpy on this path — it relies on the block content from an earlier path-A fill.
+It also accumulates sizes into globals 0x8499667C/0x84996680 (lis -31591 +26236/+26240).
+
+## Pool semantics (ownership / lifetime)
+Pool = array of 1568-byte block descriptors at [pool+4]; handle = (block << 23) | offset.
+Blocks are 64 KB physical allocations (82CEA680(0x10000, 7, 25, 384) on first use).
+- lock `82CEBFC0`: under a lock at pool+16, refcount u16 at desc+20 ++; if desc+12 (CPU
+  mapping) is 0 it maps via `82CBC020(desc[+0], 16, 0)` and stores the pointer at desc+12.
+- unlock `82CEC170`: refcount --; **when it reaches 0 it calls `82CEA7D0(desc[+0], -1, 0)`
+  and clears desc+12** (mapping dropped).
+So the pointer stored in the shader object is only guaranteed to be *mapped* while the
+block is locked; the game releases the lock right after XGRegister*. On real hardware
+the GPU uses the physical address, which stays valid; the CPU mapping can go away.
+
+## What to check (in order)
+1. Log in 83FB7658/83FB67F0 (or right before): r4 and the first 16 bytes at r4, plus
+   the first 16 bytes at info+8 (phys_src). Three outcomes:
+   - src zero -> the container's physical section wasn't loaded yet (streaming) — look at
+     who produced the container (caller of 82BD73F0);
+   - src non-zero, r4 zero -> the memcpy went to a different view than r4 (compare the
+     address memcpy wrote to in 82CEC738 with r4; see point 2);
+   - r4 non-zero at register time but zero at bind -> content dropped/remapped after
+     unlock (82CEA7D0 path) or path B reused a block that was never filled.
+2. Physical-view aliasing: the pointer is in the 0xE0000000 view. In xenia that heap has
+   a +0x1000 host offset; make sure your read of the backing uses the same
+   TranslateVirtual path as the CPU memcpy, NOT (addr & 0x1FFFFFFF) into a physical base
+   without the offset. A 0x1000 shift would read neighbouring (possibly zero) pages.
+3. Hash at bind time only after confirming content is non-zero; never cache a zero section (agreed).
+
+## Memexport VS A58EBA123EFB541C / BeginExport(8496AF20, format 0)
+Not related to the shader fill — nothing in 82BD73F0 or the pool touches GPU export.
+Treat it as a separate issue.
+
+# Addendum 7 (2026-09-30): depth resolve flags=4 into the 960x3840 atlas; Resolve ABI
+
+Static analysis of TU23 generated code + shader-identity-game.log. Not verified at runtime.
+I could NOT pin down statically which caller produces the atlas copies — see "What to log".
+
+## Resolve ABI (game side)
+Game resolves go: caller -> `82BD6800` or `82BD69A0` -> `82BD5BE8` -> `82B64860` -> Resolve `83FBF0D8`
+(ResolveEx `83FBDF80` is only reached from inside 83FBF0D8).
+`82B64860` only shifts registers and inserts the device, so Resolve receives, XDK order:
+`Resolve(device r3, Flags r4, pSourceRect r5, pDestTexture r6, pDestPoint r7, DestLevel r8,
+DestSliceOrFace r9, pClearColor r10, ClearZ f1, ClearStencil [sp+0x5C], pParameters [sp+0x64])`.
+
+Flags as the game builds them (82BD6800 / 82BD69A0):
+- low bits = source: RT index 0..3, or **4 = depth/stencil** (when the group's slot type [slot+0] == 4);
+- **0x10** added when [group+296] != 0 — per SDK this is D3DRESOLVE_FRAGMENT0 (resolve sample 0 only, MSAA);
+- **0x100 = D3DRESOLVE_CLEARRENDERTARGET**, added ONLY on the colour path and ONLY when the caller
+  passed a clear colour; pClearColor then points to a stack copy of **4 floats (D3DVECTOR4, RGBA)**.
+  On the depth path 0x100 is never set and pClearColor = NULL;
+- bits 31:26 = exponent bias added in 82BD5BE8 by object type (Addendum 3: 117 -> +11, 118 -> -5);
+- ClearZ = float constant at 0x82005CD4 (likely 1.0 — read it), ClearStencil = 0, pParameters = NULL
+  in both helpers.
+DestSliceOrFace = u32 table at 0x8211377C indexed by the caller's face argument.
+
+## The 82BD6800 helper (single resolve)
+`82BD6800(slot r3, pClearColor r4, holder r5, level r6, face r7, pDestPoint r8, pSourceRect r9)`;
+dst texture = holder[0][+44] -> the obj+120+52*idx header (Addendum 3). SourceRect and DestPoint are
+**passed straight through from the caller** (D3DRECT {x1,y1,x2,y2} / D3DPOINT {x,y}, s32).
+After a colour resolve it calls 82BD5C60 (advances the obj ring index); not after depth.
+It also rebinds the slot before/after via 82B698A0 when [group+1042] is set.
+
+## Callers that loop (candidates for the 4 transfers)
+- `82B690D8`: loop over group tiles (+308 count, rects at [group+1036]+16*i), clamps each rect
+  to the texture size (u16 at tex+40/+42), sets predication `3 << 2i` (82B64930) and calls
+  82BD6800 with **srcRect = destPoint = the same rect** -> copies tile i to the same place.
+- `82B69A78`: tiles x 5 slots (RT0..RT3 + depth at k==4), rects from group+792+16*k.
+- `82B6A4C0`, `82B69CE0`: same pattern with explicit DestPoint built on the stack.
+None of these obviously produces "960x960 src -> y = 960*i dst" — the atlas placement probably
+comes from a caller of these (shadow code) passing a DestPoint. So:
+
+## What to log (cheap, settles it)
+In ResolveHook print lr and the ORIGINAL guest pointers' contents:
+`*pSourceRect` (4 x s32), `*pDestPoint` (2 x s32), level, slice, and the two frames up
+(82B64860's caller = 82BD5BE8, then its caller). Expected for a 4-cascade atlas:
+srcRect (0,0,960,960), destPoint (0, 960*i) for i = 0..3, one depth render between each.
+If destPoint is always (0,0) the atlas is filled differently (e.g. viewport offsets on a tall RT).
+
+## Destination format / how the PS reads it
+- Source DS: D3DFORMAT 0x2D200196 -> tf 22 = **k_24_8 (D24S8, unorm depth)**.
+- Destination 0xAC585174: xenos_format 22 (k_24_8), tiled, num 0, sign 0, exp 0,
+  swizzle 0xB48 = X, Y, 1, 1 (Xenos swizzle codes 0..3 = XYZW, 4 = 0, 5 = 1).
+- Depth resolve copies the 24_8 bits as-is (no numeric conversion; xenia: depth resolve is raw).
+- A tfetch of k_24_8 returns depth as unorm24 -> float in X; xenia loads it with
+  kLoadShaderIndexDepthUnorm into **R32_FLOAT, swizzle RRRR** (d3d12/texture_cache.cpp:191).
+  So the PS sees a plain float depth 0..1 and does its own compare (no hardware PCF on Xenos).
+- For native: host D32_FLOAT_S8 source -> write the atlas region as float depth in an R32_FLOAT
+  (or D32 SRV) texture at destPoint; if you keep guest-memory layout, convert float -> unorm24 << 8.
+  Watch the clear value: ClearZ default 1.0 = far = "not in shadow"; an unwritten atlas (current
+  skip) reads as 0 = everything in shadow -> black, which fits the symptom but is NOT proven.
+
+# Addendum 8 (2026-09-30): the memexport pass is a GPU memmove
+
+Static analysis of TU23 generated code. Not verified at runtime; not yet matched against the log.
+
+## Call chain
+- BeginExport `83FC53D0` / EndExport `83FC54C8` are called from game code in ONE place:
+  **`82BD2868`** (the other caller, 83FC3438, is D3D-internal).
+- `82BD2868` is only called by **`82BD2AA0(dst r3, src r4, size r5)`**.
+- `82BD2AA0` is called by **`82B57E88`** and **`82B583C0`** — they sit next to the physical
+  block pool code from Addendum 6 (82B58928, 82B58DD8, 82B590E8). Most likely the pool's
+  compaction/relocation path. (Inference from address proximity, not proven.)
+
+## 82BD2AA0 = GPU memmove(dst, src, size)
+1. `83FC5A58(device, src, size, 0)` and `83FC5A58(device, dst, size, 0)` — probably
+   D3D InvalidateGpuCache/flush for both ranges (not hooked by native).
+2. Binds a fixed VS/PS pair from globals: VS = [0x84992E40], PS = [0x84992E44]
+   (SetVertexShader 83FB6A30 / SetPixelShader 83FB6828), then 82B64708([0x84992E48]).
+   The VS is presumably A58EBA123EFB541C — check that [0x84992E40] resolves to it.
+3. size is rounded up to 64. Copy is done in chunks **from the end backwards**, each chunk at
+   most (dst - src) bytes -> overlap-safe for dst > src, i.e. memmove semantics.
+4. Final `83FC5A58(device, dst, size_rounded, 0)`.
+
+## 82BD2868 = one chunk (dst r3, src r4, bytes r5)
+- Takes the next slot from a ring of **2048** (index at global 0x84992E4C, wraps at 2048).
+- Two 32-byte buffer headers per slot, built with `83F9AFD0` (header init, size = bytes)
+  + `83F9B640` (set base address):
+  - **dst header: 0x8496AE40 + 32*slot** — this is the "export resource" you see
+    (8496AE40..8496AF20 = slots 0..7). Plain XG buffer header, not a D3DBuffer created
+    through CreateVertexBuffer, so your buffer registry doesn't know it.
+  - src header: 0x8497AE40 + 32*slot.
+- Builds a 16-byte export stream constant at 0x8498AE40 + 16*slot and copies it into the
+  device's VS float-constant shadow at **device+1920..1932** (that's the eA memexport
+  address constant): word0 = 0x40000000 | (dst address bits), word1 = 0x4B000000,
+  word2 = (old & mask) | 0x4B021A01 (export format/endian), word3 = 0x4B000000 | (size >> 3).
+  Dirty bit 63 set in the 64-bit mask at device+0.
+- SetStreamSource(0, src header, offset 0, stride 64) via 83FBA160.
+- device+1936..1948 = second constant: x = float at 0x82005CD0, yzw = stack leftovers
+  (uninitialised in the caller frame — the shader presumably ignores them).
+- Loop in batches of <= 16384: BeginExport(0, dst header, 0) ->
+  **DrawVertices(POINTLIST, start, count)** with count = bytes / 64 -> EndExport(0, dst header, 0).
+- Unbinds stream 0, then `83FC5A58(device, src, bytes, 0)`.
+So each point = one 64-byte record (four float4 fetches), exported unchanged to dst + 64*index.
+
+## Readiness / ordering
+It's a GPU copy in command order: everything submitted before it that writes src must finish
+first, and every later draw that reads dst sees the new data. The CPU side does not wait
+(apart from the cache ops). If this pass is skipped, the relocated memory at dst keeps stale
+or zero contents while the game already uses the new address -> garbage vertex/index data
+-> "huge triangles" is consistent with this, but NOT proven.
+
+## CPU equivalent
+`memmove(dst, src, round_up(size, 64))` in guest memory at the point in the command stream
+where 82BD2AA0 runs, then invalidate any host mirrors (vertex/index/texture/shader caches)
+covering dst. Since native executes draws immediately, doing it synchronously in a hook on
+82BD2AA0 matches the GPU ordering, as long as no queued host work still has to read the old
+src/dst contents. Byte order: export format 0x1A01 plus the vertex fetch are expected to be a
+straight bit copy (same endian in and out) — confirm on one call by comparing src and dst
+after a real run (e.g. xenia stock) before relying on it.
+
+## Suggested check
+Hook 82BD2AA0 (log dst/src/size and lr) and see whether the addresses match the vertex/index
+buffers of the characters and menu background that break.
+
+# Addendum 9 (2026-09-30): sampler state = the Xenos fetch constant shadow
+
+Static analysis of TU23 generated code. Bit positions below are LSB-numbered (xenia
+xe_gpu_texture_fetch_t, rexglue-sdk include/rex/graphics/xenos.h:1168).
+
+## Key finding
+TU23 D3D has NO separate sampler object. SetSamplerState edits the **real Xenos texture fetch
+constant** in place: `F(s) = device + 0x480 + 24*s` (6 dwords d0..d5, s = D3D sampler index,
+26 slots). SetTexture merges the texture header's fetch words into the same 6 dwords while
+keeping the sampler bits. So the correct native solution is NOT to translate D3D sampler
+states one by one: read the 6 dwords at F(s) at draw time and derive the host sampler exactly
+like xenia does from a fetch constant (clamp_x/y/z, mag/min/mip filter, aniso, lod_bias,
+mip_min/max_level, border_color). Unleashed does the same thing from its data0/3/5.
+
+Dirty tracking: every setter ORs `(1<<63) >> (s+32)` into the u64 at device+24.
+
+## Setup (device init 83FC9450, NOT hooked)
+Table at **0x847F9FD8**: 20 entries x 12 bytes {+0 getter, +4 setter, +8 default}.
+For k = 0..19: device+0x3B8+4k (952) = getter, **device+0x1D4+4k (468) = setter**; then for
+every sampler s it calls setter_k(device, s, default_k); finally SetTexture(s, NULL).
+The defaults are data — read the 20 u32 at 0x847F9FE0 + 12k from guest memory at runtime.
+Setter k order follows D3DSAMPLERSTATETYPE; the addresses below identify each by the bits it writes.
+
+## Setters (all `(device r3, sampler r4, value r5)`), what they write in F(s)
+| Setter | Field | Bits | Rule |
+|---|---|---|---|
+| 83FB9DB8 | AddressU | d0 10..12 clamp_x | raw value (X360 D3DTADDRESS == Xenos ClampMode 0..7) |
+| 83FB9E08 | AddressV | d0 13..15 clamp_y | raw |
+| 83FB9E58 | AddressW | d0 16..18 clamp_z | raw |
+| 83FB97E8 | MagFilter | d3 19..20 mag_filter, d3 25..27 aniso, d4 10 mag_aniso_walk | value&3 -> filter; (value>>2) = anisotropic flag; aniso from table 0x824CAF70[byte device+10864+s] |
+| 83FB9640 | MinFilter | d3 21..22 min_filter, d3 25..27 aniso, d4 11 min_aniso_walk | same scheme |
+| 83FB9988 | MipFilter | d3 23..24 mip_filter | raw |
+| 83FB9A88 | MaxAnisotropy | d3 25..27 aniso_filter | table 0x824CAF70[value] (log2), only if an aniso walk bit is set; value kept in byte device+10864+s |
+| 83FB9BA8 | MipMapLodBias | d4 12..21 lod_bias | float value * const @0x8202776C, truncated (s10) |
+| 83FB9C48 | MaxMipLevel | d4 2..5 mip_min_level | max(value, bound texture's level); request kept in byte device+12356+s |
+| 83FB9CC8 | MinMipLevel/"max level" | d4 6..9 mip_max_level | min(value, texture's max); request in byte device+12382+s |
+| 83FB9B00 | AnisotropyBias | d5 5..8 aniso_bias | float * const @0x8204EA60 |
+| 83FB9D48 | BorderColor | d5 0..1 border_color | (value != 0) -> 1 (Xenos only has black/white-ish presets) |
+| 83FB9EA8 | TrilinearThreshold | d5 3..4 tri_clamp | raw |
+| 83FB9FB0 | (force BC W to max) | d5 2 | raw |
+| 83FB9F00 / 83FB9F58 | H / V gradient exp bias | d4 22..26 / 27..31 | raw (s5) |
+| 83FBA008 | (clamp policy) | d1 11 nearest_clamp_policy | = (value == 0) |
+| 83FB98E8 / 83FB9740 / 83FB99E0 | separate-Z / volume filter flags | byte device+10890+s bits 0/1/2, folded into d4 0..1 vol_mag/vol_min | |
+Getters 83FB99C8, 83FB9B60, 83FB9C08, 83FB9D90, 83FB9DF0, 83FB9E40, 83FB9E90, 83FB9EE8,
+83FB9F40, 83FB9F98, 83FB9FF0, 83FBA050 read the same fields back. (Names for the last few rows
+are inferred from the bit they touch — confirm the k index against the table if it matters.)
+
+## What SetTexture (83FB58A8, which native hooks) must preserve
+`SetTexture(device, s, tex r5, dirty r6)`; texture header fetch words are tex+28..tex+48 (T0..T5).
+Stores texture pointer at device+0x3278+4s (device+12920). Merge rules, bit-exact:
+- d0 = T0 with **bits 10..21 kept from F** (clamp_x/y/z + pad).
+- d1 = (T1 & 0x1FFFFFFF) + 0x1000-in-address if T1's address >= 0xE0000000
+  (i.e. `((T1>>20)+0x200) & 0x1000`), with **bit 11 kept from F** (nearest_clamp_policy).
+  -> D3D itself applies the 0xE0000000 physical +4 KB offset here (relevant to Addendum 6).
+- d2 = T2 (size).
+- d3 = T3 with **bits 19..30 kept from F** (mag/min/mip/aniso/arbitrary filters);
+  texture supplies num_format, swizzle, exp_adjust (0..18) and border_size (31).
+- d4 = T4 bits 2..9 (mip levels) + **bits 0..1 and 10..31 kept from F**; then
+  mip_min_level = max(T4.min, byte 12356+s), mip_max_level = min(T4.max, byte 12382+s).
+- d5 = T5 bits 9..28 (dimension, packed_mips, mip_address, same 0xE0000000 fixup)
+  + **bits 0..8 kept from F** (border_color, force_bc_w, tri_clamp, aniso_bias).
+- tex == NULL: F.d0 &= ~3 (fetch type 0 = invalid), nothing else changes.
+- device+24 |= r6 (caller-supplied dirty bit).
+
+## Recommended native fix
+1. Keep the original setters running (they are not hooked) — the guest shadow at F(s) is then
+   always correct for sampler bits.
+2. In SetTextureHook either call the original merge or reproduce the rules above exactly;
+   the current hook presumably overwrites/ignores F(s), losing sampler bits.
+3. At draw: for each sampler the shader uses, read F(s), build a sampler key from
+   clamp_x/y/z, mag/min/mip_filter, aniso_filter, lod_bias, mip_min/max_level, border_color
+   and reuse xenia's fetch-constant -> host sampler mapping (D3D12 TextureCache::GetSamplerParameters
+   in rexglue-sdk). Cache host samplers by key; there are few distinct ones.
+4. VS vs PS: setters and SetTexture index F(s) by the D3D sampler index unchanged; which s
+   values the VS uses (D3DVERTEXTEXTURESAMPLER*) and how s maps to the shader's tfetch slot
+   was not determined here — take it from the shader's fetch-constant indices in your AOT data.
+
+# Addendum 10 (2026-09-30): replaying sampler init without 83FC9450; VS sampler path
+
+Static analysis of TU23 generated code. Not verified at runtime.
+
+## Exact original order (83FC9450, after the render-state part)
+```
+for s in 0..25:                      // cmplwi r28,26
+    for k in 0..19:                  // table 0x847F9FD8, 12-byte entries
+        device[0x3B8 + 4k] = entry_k.getter   // written every s (harmless repeat)
+        device[0x1D4 + 4k] = entry_k.setter
+        entry_k.setter(device, s, entry_k.default)   // default = u32 at 0x847F9FE0 + 12k
+    SetTexture(device, s, NULL, (1<<63) >> (s+32))   // 83FB58A8
+```
+So it's sampler-major (all 20 states for slot 0, then slot 1, ...), each slot ended by
+SetTexture(NULL). Your plan matches; keep the SetTexture(NULL) step via your hook
+(its only effect on a zeroed device: F(s).d0 &= ~3, store NULL at device+0x3278+4s,
+device+24 |= dirty). Right after the loop the original continues with render-state
+setup (12568 = 5, 12572 = 1, 83FAFC78(2), 83FB7840(2), ...) — you're skipping that, fine.
+
+## Queue safety of the 20 setters with texture == NULL
+All setters I identified (Addendum 9 table) are leaf functions: they only read/modify
+F(s) = device+0x480+24s, the bytes device+10864+s / +10890+s / +12356+s / +12382+s,
+the u64 dirty mask at device+24, and static data (0x824CAF70 table, float constants at
+0x8202776C / 0x8204EA60). The only calls are the GPR save/restore helpers
+(0x83E44F7C / 0x83E44FCC) in 83FB9640 and 83FB97E8. No command-buffer (13928), fence,
+KickOff or queue access.
+MaxMipLevel/MinMipLevel setters (83FB9C48 / 83FB9CC8) read the bound texture pointer at
+device+0x3278+4s and **skip the write when it is NULL** — so on a zeroed device those two
+only store the request byte (12356+s / 12382+s) and leave d4 mip fields 0 until SetTexture.
+Caveat: I did not read the table itself (it's guest data). Before relying on this, check at
+runtime that all 20 setter pointers are within the set listed in Addendum 9
+(83FB9640..83FBA008 range). Any pointer outside it — inspect before calling.
+Note the original SetTexture (not the setters) does touch the command buffer, but only when
+the PREVIOUS texture is non-NULL (fence/reference path at device+11036/11040/13928).
+
+## VS / PS sampler -> tfetch constant index
+device+0x480 is the start of the **Xenos fetch constant file** itself, not a D3D-private copy:
+SetVertexShader 83FB6A30 and SetPixelShader 83FB6828 both memcpy the shader's embedded
+fetch-constant patches into `device+1152 + offset` (records {u16 byte offset, u16 dword count}
+from the shader object, second loop merges masked words). That is how vertex-fetch constants
+(vfetch streams) get written into the same file.
+Consequences:
+- Texture fetch slot n occupies bytes 1152 + 24n .. +23. D3D sampler index s == fetch
+  constant index n for BOTH VS and PS. The tfetch "const" index in the microcode is the slot
+  to read, no remapping.
+- Slots 0..25 are D3D samplers (26 = what init touches); bytes of slots 26..31
+  (device+1776..1919) are left for vertex fetch constants written by shader binds.
+  (Memexport address constants start at device+1920 = the float constant file, Addendum 8.)
+- For PS, 0..15 as you expect. For VS, take the tfetch const index straight from the VS
+  microcode in your AOT data and read F(that index); by the rule above the D3D-side slot is
+  the same number (on 360 the vertex texture samplers sit above the 16 PS slots — expect
+  16..19 — but I did not find the constant in code, so trust the microcode index).
+- Watch for the shader-bind memcpy: if native hooks SetVertexShader/SetPixelShader, those
+  embedded fetch patches must still be applied to device+1152, or vfetch (and any
+  texture fetch a shader pre-fills) will read stale words.
+
+# Addendum 11 (2026-09-30): dynamic (CPU-written) textures rebuild their header on every lock
+
+Static analysis of TU23 generated code. Not verified at runtime (compare headers as you planned).
+
+## VS slots 16/17 come from the game's render-texture objects
+`82BCE7F8(stage r4, slot r5, obj r6, sampler_desc r7)` is the game's "bind texture + sampler":
+- stage 0 (vertex) -> **D3D slot = slot + 16** (confirms VS samplers 16..19, Addendum 10);
+- it writes AddressU/V/W straight into F(s) and calls the Addendum 9 setters for the rest
+  (MinFilter 83FB9640, MagFilter 83FB97E8, MipFilter bits, LOD bias, MaxAnisotropy,
+  BorderColor, Min/MaxMipLevel);
+- then SetTexture(s, **obj + 120 + 52*[obj+116]**) — the same header ring as in Addendum 3.
+Callers: 82BCB7F0, 82BCBAA8, 82BCBB78.
+So AB1AE148 / AB1AE238 are header slots inside two such objects, not CreateTexture textures.
+
+## Header lifetime: rebuilt on every CPU lock — found the initializer missing in Addendum 5
+`82BD6088(obj, flags)` = lock-for-CPU-write (callers 82BD4520, 82BD5498):
+1. if obj+36 bit0 (double-buffered): **obj+116 = 1 - obj+116** (ping-pong between headers
+   at obj+120 and obj+172);
+2. BlockUntilNotBusy(header) 83FBF900 (waits for GPU to finish with that buffer);
+3. **`82BD58D0(obj, idx, ...)` rebuilds the whole header** at obj+120+52*idx through XG:
+   2D `83F9B490`, cube `83F9B508`, volume `83F9B578`; size from u16 obj+40/+42(/+44),
+   levels byte obj+46, format obj+96; exp bias arg = -13 if type [obj+12] == 117, +5 if 118
+   (matches the resolve exp-bias pairing in Addendum 3);
+4. locks the backing pool block (handle obj+16, 82B58898 — the same pool family as Addendum 6/8)
+   and gets the CPU address via 82BD5C80(handle, 4096) -> obj+20;
+5. **`83F9B5D8(header, base, mip_base)`** sets base (and mip) address in the header;
+   also stored at obj+224+4*idx;
+6. optional 82BD5D20(handle, -1) afterwards.
+Unlock/commit is `82BD5AA8` (records obj, size, idx and the written range in globals
+0x849965D0..0x849965E4; also calls 82BD58D0 on some paths).
+
+So between two binds of "the same" header address:
+- the header fetch words T0..T5 (header+28..+48) are rewritten (base address can change
+  when the pool relocates the block — see Addendum 8's GPU memmove), and
+- the texel data at base is rewritten by the CPU every lock (bone/instance data for
+  VS F5871392703E06C0, 1024x128 32_32_32_32_FLOAT, tiled).
+With double buffering the two headers alternate each frame.
+
+## What native should do
+- Identity: key host textures on the decoded fetch words (T1 base, T5 mip base, T2 size,
+  T3 format/swizzle/exp), **re-read at every SetTexture**, not on the header pointer.
+  resource.common (header+0) is just type/refcount — not needed for identity.
+- Content: these are CPU-written each frame; re-upload (or hash-check) on every bind, or at
+  least after each 82BD5AA8 unlock for that obj. A first-bind cache will show frame-1 data
+  forever -> skinned characters degrading over time fits, but confirm with the header compare.
+- Same class of object also backs resolve targets (Addendum 3: GPU-written via Resolve).
+  For those, invalidate the host copy on Resolve instead of re-uploading from guest memory.
+- Likely relevant to the menu "lighting/blur" corruption as well: any render-texture obj
+  sampled after a flip/resolve with a cached first-bind copy will show stale content.
+  Not verified.
+
+# Addendum 12 (2026-09-30): who writes animation / VB / IB data, and what changes over time
+
+Static analysis of TU23 generated code + rexglue-sdk sources. Not verified at runtime;
+logs/images named in the question were not inspected here.
+
+## 1. Paths that write guest data (and whether the write watch sees them)
+a) **Guest CPU stores** (recompiled code, incl. guest memcpy 0x83E45090 / memmove 0x83E467E0):
+   ordinary host stores into the protected views -> the access-violation path fires the
+   invalidation. This is how the vertex-texture data (Addendum 11, 82BD6088 lock -> CPU write
+   -> 82BD5AA8 commit) and dynamic VB/IB contents are written. No DMA, no export for these.
+b) **The game's pool changes page protection itself**: `82B55588` -> `82BD2808` ->
+   `83F9F4D8` -> `MmSetAddressProtect` (the only game caller besides 82AB2800).
+   Protect value 4 (READWRITE) or 0x404 (READWRITE|WRITECOMBINE), on block lock/unlock and
+   during relocation. Callers of 82B55588: 82B556D8, 82B56480 (unlock path), 82B57880,
+   82B57B00, 82B57E88, 82B583C0 (relocation), 82B590E8 (lock path).
+   SDK: `MmSetAddressProtect_entry` -> `PhysicalHeap::Protect` (xmemory.cpp:2112) calls
+   TriggerCallbacks for the range **when the new protection is writable**, then
+   `BaseHeap::Protect` rewrites host page protection for that view. So the SDK notifies —
+   but it also **overwrites your read-only watch protection** on that view. If your cache
+   re-arms by setting protection itself and assumes it stays until the next fault, a later
+   MmSetAddressProtect silently resets it; the SDK then only tracks via its own watch flags.
+   Make sure your cache relies on the SDK's watch (EnablePhysicalMemoryAccessCallbacks),
+   not on its own VirtualProtect, and re-arms after each callback.
+c) **Host code using TranslatePhysical** (bypasses the A/C/E views entirely):
+   in the SDK only `audio/xma_context.cpp` (XMA input/output, not geometry) and
+   `system/xfile.cpp` (you said it notifies). In **native** code: `gpu_native/hooks_buffer.cpp`,
+   `textures.cpp`, `memory_watch.h` — i.e. your own resolve writeback / memexport-CPU
+   memmove / uploads. Any native write into guest memory must invalidate your own cache;
+   the watch will not see it.
+d) **Ordering race (most likely cause for a single bad animation line):** re-arming the
+   watch AFTER copying guest data to the host loses any CPU write that lands in between
+   (the game's worker threads write skinning/instance data concurrently with the render
+   thread). xenia's order is: enable watch (protect) FIRST, then read the data; a fault during
+   the copy just marks it dirty again. Check that your cache does it in that order, per page,
+   and that a callback arriving during the copy isn't cleared by the "upload done" step.
+
+## 2. Title screen after ~2 minutes (also with the cache off)
+The pool's relocation/compaction runs over time: `82B592A8` calls `82B57E88` and `82B583C0`.
+Each relocation:
+- moves the block **either by CPU (memcpy 83E45090 / memmove 83E467E0) or by GPU
+  memmove 82BD2AA0** (Addendum 8) — both paths exist in these two functions;
+- changes page protection (82B55588, point 1b);
+- calls **virtual callbacks on the owner (bctrl)** — the owner then rewrites its resource
+  headers with the new address (for textures: 82BD58D0/83F9B5D8 as in Addendum 11; for
+  buffers the same XG header setters 83F9AFD0/83F9B640 as in Addendum 8).
+So after a relocation the **same header pointer holds a new base address**, and the old
+address may be reused for something else. With the cache off, geometry still breaks if
+any native object keeps the base address it read at creation/first bind (VB/IB registry,
+vertex-declaration streams, adopted textures). Idle on the title screen is exactly when the
+pool has time to compact.
+Fix direction: for VB/IB/textures, re-read the header's base (VB/IB: the 32-byte XG buffer
+header; textures: T1/T5) at every SetStreamSource/SetIndices/SetTexture and key host
+resources on (base, size, format), not on the header pointer.
+Quick confirmation: log calls to 82B57E88 / 82B583C0 (src, dst, size) and see whether the
+title breakage starts right after the first one.
+I found no separate "title idle" render pass; the attract/idle logic was not traced.
+
+# Addendum 13 (2026-09-30): relocation callbacks and 83F9B640 (XGOffsetResourceAddress)
+
+Static analysis of TU23 generated code. Not verified at runtime.
+
+## Yes — 83F9B640 is called from the pool's relocation callbacks
+83F9B640(header r3, delta r4) itself calls **GetResourceType 83FC49E8** and dispatches:
+- type 1 -> header+24 += delta, **low 2 bits of +24 kept** (rlwimi 30..31)
+- type 2, 7, 8 -> header+24 += delta (plain add)
+- type 6 -> header+32 += delta
+- type 3 and 17..20 (textures) -> 83F9B5D8(header, delta, mip_delta if header+48 has a mip address)
+- anything else -> no-op
+So a wrong GetResourceType answer for a borrowed VB (6 instead of 1) makes every relocation add
+the delta to header+32 and leave the real base at +24 stale -> exactly "same header, old address"
+(Addendum 12). Your fix (override only for native-owned simplified headers, borrowed ->
+original leaf) matches this contract.
+
+## How callbacks are registered
+`82B56500(pool, handle, callback, userdata)`: stores callback at +40 and userdata at +44 of the
+handle's 52-byte descriptor (max 9216 handles, under the lock at pool+944). The relocation code
+(82B57E88 / 82B583C0, via bctrl) calls `callback(phase r3, old r4, size r5, new r6, userdata r7)`:
+- phase 0 = before the move: switch buffer / BlockUntilNotBusy on the resource(s)
+- phase 1 = cache flush (dcbf) over the old range (only 82B7A500 implements it)
+- phase 2 = after the move: 83F9B640(header, new - old)
+
+## Owners that go through this chain (found by who takes the callback's address)
+| Callback | Registered by | Owner / headers relocated |
+|---|---|---|
+| **82CB9120** | **82CBCBE8** | double-buffered **vertex buffer** owner: two XG VB headers built with 83F9AFD0 at owner+24+32*idx; idx = s16 at owner+4; current address kept at owner+88+4*idx. Phase 0 flips idx and waits (83FBF900), phase 2 offsets header[idx] and updates the stored address |
+| **82CB9070** | **82CBC9E0** | same pattern for **index buffers** (headers built with 83F9B068, idx at owner+12) |
+| **82B7A500** | **82B7D0D0** | placement-shader groups (Addendum 6): walks a list at [userdata+12] (next at +60) and offsets the VS at +104 and the PS at +108 — **shaders move too**, so the shader backing pointer at obj+0x20/+0x18 changes |
+| 82CE86D0 | not found by address scan (registered some other way) | a single header; offsets it only if header byte +3 is 6 or 7 |
+Both buffer owners allocate their blocks with 82CEA680 + 82CBC020, i.e. the same 64 KB pool.
+
+## Implications for native
+- Borrowed VB/IB/shader/texture headers can change base at any phase-2 callback; re-read the
+  base per bind (Addendum 12) — and your shader-by-backing lookup (Addendum 6) needs the same
+  treatment: the physical section moves, and its content is copied by the relocation.
+- 83F9B640 must see the true guest type, so GetResourceType must stay original for anything
+  not created by native.
+
+# Addendum 14 (2026-09-30): InsertCallback, CPU waits, and leads for hands / portal / blur
+
+**Everything below is STATIC analysis of TU23 generated code. Nothing here was verified in
+game.** The session-20260930 artifacts named in the question were not inspected.
+
+## 1. InsertCallback 83FC0F38 — what the callbacks do
+
+### 82BCDE68 and 82BCDF30 = GPU profiler timestamps (static)
+- Both take the context in r3 as `(group << 16) | marker`, index a table through the pointer
+  at global **0x849677B8** (`[0x849677B8] + 4*(1504 + 1122*group + marker)` = per-marker
+  counter, capped at 32), and call **83F9EA20 = `mftb` -> store u64 timebase**.
+  So each callback records "the CPU time at which the GPU reached this point".
+- 82BCDE68 = begin marker (group 0 also clears the counters for all groups first).
+- 82BCDF30 = end marker; for group 0 it then sums (end - begin) timestamp differences into
+  per-group totals (fields around +6200/+6328/+6396 and +128/+320 of the table).
+- They touch nothing else: no resource refcounts, no frees, no pool, no animation data.
+- Other functions also read through the pointer at 0x849677B8 (82B3B278, 82B3EC68,
+  82B79938, 82B79D00, 82B7A420, 82B7A488, 82B7ABF0, 82B9B3F8). I did not check whether any
+  of them read the *timing* fields (e.g. for adaptive quality) or unrelated fields of the
+  same block. If none reads the timings, these callbacks are profiler-only.
+
+### Semantics to emulate
+XDK InsertCallback = "call this when the GPU reaches this point in the command stream".
+It never makes the CPU wait. The wrapper 83FC0F38 only kicks the command buffer if it's full
+(83FC0C10) and records the callback (83FC02C0). Your current "wait for all host fences, then
+call synchronously" turns every marker into a full GPU flush, which is the 29–32 ms.
+Safe options, cheapest first:
+a) For 82BCDE68 / 82BCDF30: call them immediately at insert time (CPU timestamp instead of
+   GPU timestamp). Timings become meaningless but nothing depends on them — **conditional on
+   the readers above not using the timing fields**. Log that one first.
+b) General case: queue (callback, context) with the host fence of the current submission and
+   run it when that fence completes (poll at the next submit/Present, or on a worker thread
+   if the callback is thread-safe). Keep the order of callbacks.
+c) Never block the game thread in InsertCallback itself.
+
+### Other InsertCallback users (static)
+- **83FBD2D8 (inside EndTiling 83FBD098)**: callback **83FCAA80**, context = physical address
+  of the device buffer at device+13760. 83FCAA80 pushes the context into a per-core ring and
+  signals it through import 0x841132FC — this is the D3D worker-queue path (next to
+  StartWorkerQueue 83FCAB10). Native hooks EndTiling, so this is not reached; do not emulate.
+- **83EB711C (inside 83EB6D68)**: after a 6-vertex TRIANGLELIST draw, InsertCallback with
+  flags from a stack table indexed by the current hardware thread (83F9EEC8 = `lbz 268(r13)`),
+  callback **83EB4480** = `this->vtable[23]()` (offset 92), context = the 83EB6D68 object.
+  83EB6D68 is also one of the SetTexture callers (Addendum 11 list), so it's a helper renderer
+  (likely UI/movie/XUI) whose virtual 23 runs when the GPU is done with its draw. I didn't
+  resolve what vtable[23] does; to be safe, run it at the matching host fence (option b), not
+  earlier.
+
+## 4. BlockUntilNotBusy 83FBF900 (static)
+`83FBF900(resource)` -> `83FBF7D0(device, fence = resource+8, reason 5, resource)`.
+BlockOnFence 83FBF8A0 is the same with reason 6 and an explicit fence.
+There is **no CPU-worker contract** in it: it waits purely on the GPU fence stored in
+resource+8. D3D writes that field when a resource gets referenced by queued GPU work (e.g. the
+end of the original SetTexture stores the current fence into the previously bound texture's
++8, device+11036/11040 path).
+So for native:
+- a resource whose data native already copied into an immutable host snapshot is "not busy"
+  for the guest, and waiting can return immediately — **unless** some native path still reads
+  guest memory for it later (lazy texture upload, resolve destination, memexport destination);
+- for those, wait only for the host fence of the last submission that used that resource,
+  not for all fences.
+- 82BD6088 (Addendum 11) and phase 0 of 82CB9120/82CB9070 (Addendum 13) use it only to make
+  sure the GPU is done with the buffer before the CPU rewrites it or the pool moves it.
+  With snapshot semantics, that's satisfied once your snapshot exists.
+Your measurement (removing these waits barely helped) fits: the cost is in InsertCallback.
+
+## 2. Vortech's disappearing hands — where to look (static leads, not analysed in depth)
+Per-part data sources that can differ between body and hands:
+- vertex textures (VS slots 16..19, Addendum 11): written after lock 82BD6088, committed by 82BD5AA8,
+  double-buffered (header index flips each lock);
+- double-buffered dynamic VBs/IBs owned through 82CBCBE8 / 82CBC9E0 (Addendum 13): two
+  headers, index flips in relocation phase 0 as well;
+- GPU memmove relocations 82BD2AA0 (Addendum 8) and CPU relocations 82B57E88 / 82B583C0;
+- placement shaders relocate too (82B7A500): a hands-only shader whose backing moved would
+  fail the lookup or use stale code.
+Suggested runtime check: for the draws of the hands, log per draw the VS hash, all stream
+header addresses + their bases, vertex texture header addresses + T1 base, and compare the
+same draw between a good and a bad frame (and with stock xenia if possible).
+I did not trace the game's skinning code itself.
+
+## 3. Portal layer, black/transparent objects, missing HUD icons, blur (static leads only)
+- **Alpha test.** Xenos alpha test lives in RB_COLORCONTROL (function + reference in
+  RB_ALPHA_REF), not in the shader. D3D12 has no fixed-function alpha test, so it has to be
+  emitted in the PS (xenia does this with a system constant). HUD hearts/coins and portal
+  energy layers are typical alpha-tested/alpha-blended draws. Check whether native applies
+  it and the alpha-to-coverage bit.
+- **Blend state.** RB_BLENDCONTROL0..3 per RT, plus RB_COLOR_MASK (device+10460, Addendum 4).
+  A missing blend or a zero write mask gives "invisible" or "black" objects.
+- **Blur / DoF:** the shipped xenia-based build **turns DoF off by shader hash
+  CFEAC7ADB912F8A9** (release 0.1.17). The released game never showed that DoF pass, so
+  "sharp in Xenia" is partly because of that. Compare native against a xenia run with the
+  DoF cvar disabled before treating the blur as a native bug. Also check the exp bias
+  (Addendum 3/4): a wrong ×32 or /32 scale in the half/HDR chain changes the CoC input.
+- Tonemap PS 3A47E5DDE66B42C6, c12 and slot 3: not analysed (needs the microcode
+  disassembly). Values to compare with xenia at the same frame: c12, the fetch constant of
+  slot 3 (6 dwords at device+0x480+24*3), and the size/format of the texture bound there.
+
+# Addendum 15 (2026-09-30): alpha-test leaves (correction) and the tonemap mix
+
+**All of this is STATIC: raw TU23 words from `xexdump/dump/default.bin` plus the translated HLSL
+`out/native-gpu/session-20260930/bank118-candidate/hlsl/ps-3A47E5DDE66B42C6.hlsl`. None of it was
+run in game.** I also agree that alpha test should not be named as the cause of the portal/hands bug yet.
+
+## 1. Correction: two of the three setters are not alpha test
+The device shadows the register block starting at 0x2200 in order, so the offsets map like this:
+`+10548` RB_DEPTHCONTROL (0x2200), `+10552` RB_BLENDCONTROL0 (0x2201), **`+10556` RB_COLORCONTROL
+(0x2202)**, `+10560` RB_HIZCONTROL, `+10564` PA_CL_CLIP_CNTL, **`+10568` PA_SU_SC_MODE_CNTL (0x2205)**.
+- `83FB7DA0` = `lwz r11,+10568; rlwimi r4,r11,0,0,28; stw r4,+10568; dirty[16] |= 0x40`
+  -> PA_SU_SC_MODE_CNTL bits 0..2 = cull_front / cull_back / face. This is **CullMode**
+  (d3d-hook-map.md already has this). Getter 83FB7DC0 = `&7`.
+- `83FB7DD0` = `rlwimi r11,r4,3,21,28` -> bits 3..10 = poly_mode + front/back ptype. This is **FillMode**,
+  not the alpha reference. Getter 83FB7DF0 = `(x>>3)&0xFF`.
+Don't wire either of them into alpha test.
+
+## 2. The real alpha-test leaves (RB_COLORCONTROL at +10556, dirty[16] |= 0x200)
+| Setter / getter | Word | Field |
+|---|---|---|
+| **83FB82C0 / 83FB82E0** | `rlwimi r4,r11,0,0,28` -> bits 0..2 = r4 | **ALPHA_FUNC** |
+| 83FB7E00 / 83FB7E28 | bit 3 = r4&1 (also dirty[16] \|= 0x40000) | **ALPHA_TEST_ENABLE** |
+| 83FB93B8 / 83FB93D8 | bit 4 = r4&1 | ALPHA_TO_MASK_ENABLE |
+| 83FB93E8 / (none found) | bits 24..31 = r4 | ALPHA_TO_MASK_OFFSET0..3 (2 bits each) |
+| **83FB92B8 / 83FB92E0** | `stw r4,tmp; lfs f0,tmp; stfs f0,+10620`; dirty[24] \|= 1<<48 | **ALPHA_REF (float)** |
+(The neighbouring pair 83FB9280/83FB92A8 writes the float at +10616 with dirty bit 47. It's a
+separate float state, which I did not identify.)
+
+Leaf contract:
+- The function value goes into bits 0..2 **raw**, so the enum is the hardware CompareFunction:
+  0 NEVER, 1 LESS, 2 EQUAL, 3 LEQUAL, 4 GREATER, 5 NOTEQUAL, 6 GEQUAL, 7 ALWAYS
+  (360 D3DCMP_* use these values, not PC's 1..8). Emit all 8, not only `>=`.
+  Xenos semantics: the pixel passes if `alpha FUNC ref` holds, so GEQUAL keeps `oC0.w >= ref` (clip when
+  `oC0.w < ref`, which matches your current code). NEVER/ALWAYS = discard all / no-op.
+- The ref leaf does **no conversion**: it copies r4's 32 bits straight into a float slot.
+  So what reaches the leaf is already float bits, and +10620 is the RB_ALPHA_REF shadow value itself.
+  Normalisation (if any) happens in the caller, not here.
+  **Unleashed's value/256 does not apply to this leaf.** Interpreting an integer 0..255 as float
+  bits would give a denormal ~0, i.e. "alpha test never discards".
+  Runtime check to settle it: log r4 in 83FB92B8. Plausible floats (0..1, e.g. 0x3F000000) confirm this reading.
+  Small ints mean a caller path I missed.
+- Compare in the shader against the value **before** the EDRAM/exp scale (`g_ColorOutputScale`).
+  That's where your clip already sits. Xenos tests the shader's oC0.w against RB_ALPHA_REF before the RB format conversion.
+- The alpha-to-mask bits are a separate feature (the A2C spec bit). If bit 4 is set and native ignores it,
+  foliage/hair-style cutouts render as solid or vanish depending on blend.
+
+Defaults: I did not read them out of 83FC9450's descriptor table. D3D/Xenos reset values are
+enable=0, func=ALWAYS(7), ref=0.0, alpha-to-mask=0. Because native skips the init, store these in
+your shadow at CreateDevice. They are harmless: enable=0 means nothing is discarded until the game sets it.
+
+## 3. Tonemap PS 3A47E5DDE66B42C6: how sharp, "mip" and slot 3 combine
+Slots from the descriptor table: 0 `fullColor_tex`, **3 `blur_tex`**, 4 `mipColor1_tex`, 6 `noiseTex`,
+7 `cubeTex` (3D LUT).
+```
+uv    = TEXCOORD0.xy + noise-distortion * TEXCOORD4.x        // heat-haze style offset
+full  = fullColor(uv).rgb          // sharp scene
+mip   = mipColor1(uv)              // downsampled scene; mip.a = per-pixel CoC/blur factor
+t     = 2 * (mip.a * c12.x + c12.y)
+w     = saturate(t - 1)
+col   = lerp(full, mip.rgb, w)     // <- the DoF mix: sharp vs the mipColor1 downsample
+col  += blur(uv).rgb^2             // <- slot 3 is ADDED (squared), bloom-like, not a DoF lerp
+col   = filmic curve(col * c11.y exposure, consts 0.5/5/0.4/0.12/0.03/0.004)
+oC0.rgb = cubeTex LUT(saturate(col))
+oC0.a = (saturate(t) > c12.z) ? saturate(t) : luma(col) - 0.0892
+```
+- **Sharp output: w = 0 <=> mip.a*c12.x + c12.y <= 0.5.** c12 = (0, 0, *, *) or anything with
+  c12.x = 0 and c12.y <= 0.5 gives a pure `full` image. c12.w is unused.
+- So the blur source is **mipColor1 (slot 4) plus its alpha**, not slot 3. Slot 3 only adds
+  glow: if it's garbage or unresolved you get a haze/wash, not DoF. It should be black or dim where nothing glows.
+- Consequence of filtering CFEAC7ADB912F8A9: whatever normally writes mipColor1's alpha (or the blur chain)
+  no longer runs. In xenia that texture keeps its last or cleared contents. In native, check what
+  mip.a actually holds. **Stale or uninitialised alpha with c12.x != 0 produces exactly "blurry
+  everywhere".** Compare at the same frame: c12, c11.y, and the resource plus alpha content bound at slots 3 and 4, native vs xenia.
+- ABI624 / the explicit-LOD752 candidate: this shader samples everything with plain tfetch2D and no
+  LOD, so the candidate doesn't matter for this pass (static, from the HLSL above).
+
+# Addendum 16 (2026-10-01): Vortech's arms, the TT Games logo, draw entry points, stencil
+
+**Everything here is STATIC: the TU23 recompiled sources (`rexlego/generated/default`, asm comments),
+GAME.DAT entries extracted with `modcli extract`, and container hashes computed the same way as
+`HashShaderContainer` (XXH3_64 over `virtual_size + [4] + physical_size`). The game was not run.
+Nothing below names alpha or stencil as the cause.** The hash rule is confirmed: several of the hashes
+below appear as `adopted vertex shader ... hash=` in your own session logs.
+
+## 1. Vortech's arms / hands are separate models
+The large Vortech isn't one mesh. GAME.DAT has separate characters:
+`chars\misc\vortech_body\vortech_body_nxg.ghg`, **`chars\misc\vortech_arms\vortech_armleft_nxg.ghg`**,
+**`vortech_armright_nxg.ghg`**, plus `vortech_hand` (a boss character, `Addon "BossWWCVortechHandAddOn"`,
+the same shader bank as the arms), `vortech_head`, `vortech_eagle`, `vortech_fort`. So the "missing arms"
+are most likely a **separate object with its own instance/skinning constants and its own VB/IB**, drawn by
+a separate draw call. (Static inference: I don't know which model your screenshot shows.)
+
+Both arm banks contain the same 4 VS/PS pairs (order differs):
+
+| VS | PS | VS constant groups (float regs) | PS |
+|---|---|---|---|
+| `35DB03916F21B74F` | `A71CC74B4EE251E5` | Camera c0-8, Fog c43-47, Instance c48-51, Instance2 c52-54, Material c16-28, MiscGroup1 c72-73, **Skinning c92-247** | Material c4-18, MiscGroup c40-45, s5/s6/s7 (layer0..2) |
+| `A9FDB0A3AC4591FE` | `B2998D20AB498293` | Camera c0-7, Instance c48-51, Instance2 c52-54, Material c16-28, **Skinning c92-247** | same as above |
+| `0406577EF00099BD` | `7CA67FB21C799FE4` | Camera, Fog, Instance, Instance2, MiscGroup1, **Skinning c92-247** | Lights c32-36, Material c4-15, MiscGroup c40-56, s3 = sceneEnvmap cube |
+| `5EFE4DB4D125C51C` | `2E461045C523CE73` | Camera c0-7, Instance, Instance2, MiscGroup1, **Skinning c92-247** | Lights c32-36, Material c4-15, MiscGroup c40-45, s3 cube |
+
+- **All four are skinned through constants:** `g_SkinningVS.vs_skinMatrix` = **c92..c247 (156 regs = 52 bones x 4x3)**.
+  **None of them declares a vertex sampler**, so the arms do **not** use vertex texture fetch.
+  (`vs_vtf_kOffset_kHeight` / `vs_vtf_direction` are just float members of g_MaterialVS.) Rule VS slots 16..19 out for the arms.
+- `vortech_body` uses the **same four pairs plus 14 others** (it adds F8F4E648..., 386847F3..., C9DB3F6A..., E634DE29...,
+  6E25E88B..., 5D70ED05..., 07ADC2A5...). If the body renders correctly with 35DB/A9FD/0406/5EFE, the arms'
+  shaders aren't the problem in themselves. The difference has to be per-draw data:
+  c48-54 (instance world / worldView), c92-247 (bones), the stream/IB, or culling.
+- Fog vs no-fog and with/without g_MaterialVS are separate pairs. A9FD/B299 and 5EFE/2E46 (the no-fog
+  pairs) don't appear in any of the 31 session logs, while 35DB and 0406 do. That may only be your log cap.
+  It's cheap to check whether those two were ever adopted, and whether the arms are the draws using them.
+- Cheapest runtime check: log per draw VS hash in {35DB, A9FD, 0406, 5EFE} -> c48..c54, c92..c95, vertex count,
+  the first stream base and the cull/stencil/blend key. Compare body draws (same hashes) with the arm draws.
+
+## 2. TT Games opening logo
+`commonobjects\cut_openinglogos_nxg.gsc` (+ `.360_shaders`, `.nxg_textures` 4.5 MB). Its bank has only 2 pairs:
+
+| VS | PS | VS groups | PS groups |
+|---|---|---|---|
+| `91007ACB3E640E3D` | `EEE573BE160E037D` | Camera c0-8, Fog c43-47, Instance c48-51, Material c16-24, MiscGroup1 c72-73 | Material c4, MiscGroup c40-56, s5 layer0 |
+| `C3958E2D1B795ED9` | `0C1840BF35E84F2F` | Camera c0-3, Instance c48-51, Material c16-24 | Material c4, MiscGroup c40-45, s5 layer0 |
+
+These are static (unskinned), single-texture shaders with no vertex samplers. 91007ACB/EEE573BE appear in 31 logs.
+**C3958E2D/0C1840BF don't appear in any log, but they are in the microcode index.** If the logo is the part that's
+missing, check whether this pair is ever adopted. (Another identical container elsewhere would produce the same
+hash, so a hash hit doesn't prove it's the logo.) The loading-screen logo is a separate asset: `commonobjects\cut_loadingscreen_logo_nxg.*`.
+
+## 3. Draw entry points (statically complete for the D3D range)
+- Game meshes go through the **central dispatcher `82B74FB8`**. It calls `82BCF958` -> **`82BCE1F8` (the only
+  caller of DrawIndexedVertices 83FC6A58)** or `82BCFA70` (10 calls to DrawVertices 83FC6640, one per primitive type).
+  82B74FB8 is called from 82B3BFB8, 82B40890 (x5), 82B411D8 (x2, skinned path), 82B75468, 82B93070, 82BC8F10,
+  82BD6C68/6C80/6CE8/6D00/6D28, 82BDA388 (x4). Native hooks both Draw functions, so every mesh draw is covered.
+- PM4 draw packets emitted *outside* those two (DRAW_INDX 0x22 / DRAW_INDX_2 0x36):
+  `83FC4EF0`/`83FC5048` (via `83FC53D0`/`83FC54C8`) <- **`82BD2868`, the game's GPU memmove (Addendum 8)**, which also
+  calls DrawVertices; `83FC1B28` <- `83FC2C80`/`83FC3438` <- `83FC4338` <- **Swap 83FB33E0** (present blit);
+  `83FB7310` <- `83FCA0A0` <- **CreateDevice 83FAF9A0**; ResolveEx 83FBDF80; `83FC38D0` (only reached through a function
+  pointer, not identified). **None of these is a character or logo mesh draw.**
+- No DrawVerticesUP / DrawIndexedVerticesUP entry (any D3D function emitting draw packets) was found besides the ones listed.
+  **No hardware instancing:** Xenos has none. Here "instance" means the constant groups g_InstanceVS c48-51 /
+  g_Instance2VS c52-54, written per draw.
+- I did not trace the stream setup (SetStreamSource/vertex declaration) for the arms specifically.
+  Addendum 13's double-buffered VB/IB owners (82CBCBE8 / 82CBC9E0) are still the candidates for skinned/dynamic geometry. Unverified.
+
+## 4. Stencil: every setter, decoded (RB_DEPTHCONTROL = device+10548, RB_STENCILREFMASK = +10496, _BF = +10492)
+Shadow block: +10444 = reg 0x2100, so +10492 = **0x210C RB_STENCILREFMASK_BF** and +10496 = **0x210D RB_STENCILREFMASK**.
+(This is consistent with +10460 = RB_COLOR_MASK 0x2104 and the blend-factor floats at +10464..10476 written by 83FB82F0.)
+All values are inserted **raw** into Xenos fields: CompareFunction 0 NEVER..7 ALWAYS; StencilOp 0 KEEP, 1 ZERO, 2 REPLACE,
+3 INCR_SAT, 4 DECR_SAT, 5 INVERT, 6 INCR_WRAP, 7 DECR_WRAP. Your op table in draw.cpp:378 matches.
+
+| Setter (getter) | Field (LSB bits of RB_DEPTHCONTROL) | Note |
+|---|---|---|
+| **83FB8570** (83FB85A8) | bit0 STENCIL_ENABLE | requested value kept at +12312; the HW bit = value AND (depth surface +12832 != 0) |
+| 83FB84C8 (83FB8500) | bit1 Z_ENABLE | same gating, request at +12308 |
+| 83FB8508 (83FB8528) | bit2 Z_WRITE_ENABLE | |
+| 83FB8538 (83FB8560) | bits4-6 ZFUNC | |
+| **83FB85B0** (83FB85D8) | bit7 BACKFACE_ENABLE (two-sided) | not a ref/mask |
+| 83FB85E8 (83FB8608) | bits8-10 STENCILFUNC | |
+| 83FB8618 (83FB8640) | bits11-13 STENCILFAIL | |
+| 83FB8688 (83FB86A8) | bits14-16 STENCILZPASS | |
+| **83FB8650** (83FB8678) | bits17-19 STENCILZFAIL | not ref |
+| 83FB86B8 (83FB86D8) | bits20-22 STENCILFUNC_BF | |
+| 83FB86E8 (83FB8710) | bits23-25 STENCILFAIL_BF | |
+| 83FB8758 (83FB8778) | bits26-28 STENCILZPASS_BF | |
+| 83FB8720 (83FB8748) | bits29-31 STENCILZFAIL_BF | |
+| 83FB8788 / 87A8 / 87C8 | byte +10499 REF, +10498 MASK (read), +10497 WRITEMASK | dirty[16] bit 28 |
+| 83FB87E8 / 8808 / 8828 | same for _BF (+10495 / +10494 / +10493) | dirty[16] bit 29 |
+| 83FB9560 / 9590 / 95C0 / 95F0 | +10560 bits 3 / 2 / 5 and byte +10562 | Hi-Z / Hi-Stencil control (performance, not a correctness state; static guess) |
+| 83FAF5D8 | emits event 0x46 + reg 0x5C8 = 0x20000 | HiZ/HiStencil flush |
+
+SetDepthStencilSurface (**83FBAE38**) re-derives Z_ENABLE/STENCIL_ENABLE from +12308/+12312 whenever the DS changes.
+So with no DS bound, both are forced off in the shadow, and they come back on rebind. Native should do the same
+(or just read the HW bits from +10548, as it does now).
+
+**Ref/mask facts:** only 83FB8788 (front REF) has callers, and every call passes **0**. **No code writes the read mask,
+the write mask, or any _BF byte** (whole-word writes to +10492/+10496 in the D3D range: none). Those values come only from
+device init / the 83FC9450 state replay, which native skips. **If native's shadow holds 0 there, write mask 0 = stencil never
+written, and read mask 0 = NOTEQUAL-0 always fails / EQUAL-0 always passes.** Seed both bytes with 0xFF at CreateDevice
+(D3D default). I did not read the actual default out of 83FC9450, so treat 0xFF as the D3D default, not a TU23 fact.
+
+### The game's stencil modes: `82BCEAC8(state, mode 0..11)`
+State object pointer at global 0x849677B8. Only 4 functions call the leaf setters at all: 82BCEAC8 (almost everything),
+82B91C88, 82B69FB0, 82B685A8.
+
+| mode | 2-sided | func F / BF | ops | HiZ |
+|---|---|---|---|---|
+| 0 | - | stencil OFF | - | off |
+| 1 | no | **NOTEQUAL** ref 0 | keep all | 9560=1, flush |
+| 2 | no | **EQUAL** ref 0 | keep all | 0 |
+| 3, 9, 11 | yes | ALWAYS (9: NOTEQUAL) | 3/9: zpass INCR_WRAP both faces; 11: zfail INCR_WRAP both | 0 |
+| 4, 10 | yes | ALWAYS (10: NOTEQUAL) | zfail DECR_WRAP front / INCR_WRAP back (z-fail volume, "Carmack's reverse") | 0 |
+| 5 | yes | ALWAYS | zpass INCR_WRAP both | 0 |
+| 6 | no | ALWAYS ref 0 | zpass ZERO (clears the stencil where drawn) | flush |
+| 7 | yes | ALWAYS | zpass ZERO both | 0 |
+| 8 | yes | ALWAYS | zfail INCR_WRAP front / DECR_SAT back | 0 |
+
+Callers with constant modes: 82B97B68 / 82B8D648 / 82B8E508 (**4, then 1, then 0** = mark a volume with z-fail, then draw
+where != 0), 82B94BC0 (8, 7, 1, 0), 82BCC270 (7, 3), 82B91C88 (5, 1), 82B95278 / 82BA75B0 (1, 0), 82B726C8 / 82BAF2B8 (6, 0).
+**Data-driven:** `82BC3D00` takes the mode from the **material: (mat[+948] >> 11) & 7** (modes 1/2 kept; otherwise 3 or 7
+from mat[+952] bit 12), and `82B3EC68` takes it from a byte at obj+116 minus 1, with a push/pop sentinel of 12.
+**I couldn't link any of these callers to the portal statically.** The 4->1 and 5->1 sequences are the "mask then fill
+inside" pattern a portal or volume effect would use, but that's an inference.
+Runtime check: log the 82BCEAC8 mode per draw (or the +10548 / +10496 values your stencil candidate log already prints)
+across a portal frame. Note that in draw.cpp native stencil is opt-in (`LEGO_NATIVE_STENCIL`). Without it, mode-1/2
+consumers draw everywhere or nowhere, depending on the mode.
+
+# Addendum 17 (2026-10-01): shader float constants — any path that bypasses device+0x780 / +0x1780?
+
+**STATIC, TU23 only.** Short answer: **no game path writes VS/PS float constants straight into the command stream.**
+One D3D path loads constants without going through the shadow: shader-embedded literals. XenosRecomp already bakes those.
+
+1. **SetVertexShaderConstantF = `83FB6480`**: copies to device + (120+start)*16 = **+0x780 + 16*start**, then
+   `dirty[+0] |= mask`. **SetPixelShaderConstantF = `83FB6558`**: **+0x1780** (376*16), `dirty[+8] |= mask`.
+   Their only callers: **`82B7AB20`** (game constant upload, 85 call sites, args: stage r5 0=VS, start r7, count r8;
+   it skips the call when the source pointer is unchanged and the param's dirty byte is 0) and **`82B7ABF0`** (the
+   per-draw reflection loop over the shader's used groups, called from the draw dispatcher 82B74FB8), plus D3D-internal
+   `83FD3A70/3AC0/3B50` (VS) and `83FD3D90/3E20/3EA8` (PS). Every material, skinning (c92-247), instance and camera
+   group of the arms and the logo goes through these, so it lands in the shadow.
+2. **Shadow -> GPU** happens inside the draw flush: DrawVertices / DrawIndexedVertices / 83FC6180 / 83FBDD20 call
+   **`83FD1320`** with (dirty mask, reg base **0x4000** VS / **0x4400** PS, src +0x780 / +0x1780). It walks the 64-bit
+   mask (1 bit = 4 float4) and emits **type-0 register writes**. That's consistent with the shadow.
+3. **LOAD_ALU_CONSTANT (0x2F): exactly one emitter, `83FD1EC8`**, called 3x from the draw-state flush **`83FD20B8`**:
+   from the VS object (device+13068: table at +40, base +24) and the PS object (device+13072: table at +872, base +32).
+   Each entry is {u16 start, u16 count, u32 offset}, and the source is **the shader's own physical memory** (the
+   definition table literals), not the shadow. So this does bypass device+0x780/+0x1780. But XenosRecomp materializes
+   definition-table constants as locals (shader_recompiler.cpp:1609-1650), so native needs nothing extra **provided the
+   AOT build saw the same container**. This applies to every shader, not specifically to the arms, the logo or portal particles.
+4. **No SET_CONSTANT (0x2D) emitter** anywhere in 83FA0000-83FF0000 (I searched every `lis 0xC00x / ori` PM4 header).
+   The 0x55 headers there (83FB7AB8, 83FBDF80, 83FCBF00, 83FD0BF8, 83FD20B8) write **registers** (0x2080, 0x2180,
+   0x2001, 0x2208...), not ALU constants. **No GpuBeginShaderConstantF4 / GpuOwn-style function exists**: no D3D function
+   other than 83FD1320 builds 0x4000/0x4400 headers. INDIRECT_BUFFER (0x3F) only appears in ring/segment management
+   (83FBFCA0, 83FC72F8), reached through function pointers. I found no game-built command buffers, but this isn't proven.
+5. Portal particles: I couldn't identify their shaders statically. Nothing found suggests a special constant path for them.
+   They would use 82B7AB20/82B7ABF0 like everything else.
+
+Caveat: native copies the whole 0x780/0x1780 banks every draw (draw.cpp:209), so even direct game stores into the shadow
+without dirty bits would be seen. The bypass risk is limited to (3).
+
+# Addendum 18 (2026-10-01): answers to the Oct 1 follow-up + VTE/clip finding
+
+> **From the user: do NOT launch the game again until everything below (and the earlier open items) is
+> implemented and passes offline tests.** Make one complete fix batch first. Then do a single test run, and
+> only when the user asks for it.
+
+All of this is STATIC (generated TU23 sources, GAME.DAT entries, the translated HLSL in `out/native-gpu/hlsl-depth`).
+
+## 0. Corrections accepted
+- Alpha ref default = 1.0f (descriptor 59 in table 0x847F9B18). That's your reading of the real table. It replaces my
+  "D3D default 0.0" in Addendum 15.
+- Stencil masks were already seeded from the TU23 descriptors (00FFFF00), so drop my Addendum 16 0xFF advice.
+
+## 1. Do the traced 0406/35DB ranges belong to the arms? No, at least not to the arm files
+GAME.DAT decompressed sizes: `vortech_armleft_nxg.ghg` = **59,200 bytes**, `vortech_armright_nxg.ghg` = **61,522 bytes**,
+`vortech_body_nxg.ghg` = 527,760 bytes. Your samples index up to start 43644 + 1440 = **45,084 indices** (90 KB as u16)
+in one IB, with several VBs (bases 0/610/1220/1243/1535). **That IB can't come from either arm file.** Together the two are
+smaller than 90 KB of indices alone. The ranges fit the body (or another large model).
+(Assumption: the IB is the model's own buffer from its GHG, not a shared pool. Your "VB length 36864" points to per-model buffers.)
+**So in mesh1-3 no traced draw is provably an arm draw. The arm draws may not reach Draw at all**, which would point
+to game-side visibility/animation/LOD, not the renderer. Combine that with the user's note that the torso and legs are
+also damaged, and with rebind1-vortech.jpg, where almost the whole big Vortech is absent: the problem covers more than the arms.
+I didn't parse GHG submesh tables (no parser for that format here). A cheaper identity test that needs no game run:
+compare your traced first-index bytes and vertex bytes against the raw bytes inside `vortech_body_nxg.ghg`
+(`modcli extract GAME.DAT "chars\misc\vortech_body\vortech_body_nxg.ghg" out.ghg`, then search for the 24/36-byte vertex
+records you logged). A byte match identifies the asset with certainty.
+
+## 2. Opening logo: which quad is TT
+`cut_openinglogos_nxg.gsc` names 4 objects **WBLOGO, LEGOLOGO, PRESENTS, TTGAMESPRODUCTION** and 5 materials
+`_TTShaderMaterial1..5`. `cut_openinglogos_nxg.nxg_textures` holds 5 textures, in this order:
+`wblogo.nut, legologo.nut, presents.nut, ttgamesproduction.nut, ttlogo.nut` (plus 3 engine dummy lightmaps).
+Your 5 quads (index starts 0/6/12/18/24) match 5 textures. **Most likely order: 0 = WB, 6 = LEGO, 12 = PRESENTS,
+18 = TT GAMES PRODUCTION text, 24 = TT logo.** That's an inference from the string order, not from a parsed material table.
+Confirm it by logging the texture fetch-constant base for each quad and matching it against the 5 DDS headers in the nxg_textures file.
+The starts 18/24 that fade to c73.w = 0.50 / 0.99 are therefore probably the TT text and TT logo, fading in as expected.
+If they reach ~1.0 and still don't show, look at the texture/format of those two entries (the first three DDS entries
+are flagged `OPAQ`, the later ones aren't, so expect alpha-carrying formats there) and at blend. Opacity is not the issue.
+C3958E2D/0C1840BF is the **no-fog variant** of the same material set (no g_FogVS, Camera c0-3 only). The game picks the
+variant from the scene fog state. Only 9100/EEE5 being observed means fog-on is selected here, so C395/0C18 being absent is expected.
+
+## 3. Shader audit of VS 0406577EF00099BD (HLSL `hlsl-depth/smaug_firework_nxg.6.hlsl`, same hash)
+```
+r8      = POSITION.xyz, w=1
+r9      = NORMAL (decoded) * c255.x + c255.w        // c255 = (2.00787, 0, 0.001, -1): ubyte -> [-1,1]
+r4      = BLENDINDICES * c254.x                     // c254.x = 3.0 (definition-table literal) -> index*3
+for each of 4 influences: a0 = round(r4.i); rows c[92+a0], c[93+a0], c[94+a0] (4x3)
+pos     = sum(weights * bone rows)                  // weights: all 4 (x,y explicit, z = 1-x-y style via c254.y = 1)
+world   = c48..c51 (g_InstanceVS), normal via c52..c54 (g_Instance2VS)
+clip    = c0..c3 (g_CameraVS viewProj)
+```
+- The math is as expected: the literals c254/c255 come from the definition table and are baked in, the bone stride is 3,
+  and the relative load is c92+a0. So **if c92+ (bones), c48-51 and c0-3 are correct, this shader places the vertex
+  correctly.** It does not read normal.w or material opacity for position. Alpha only goes through COLOR0, as you found for the logo.
+- Check in the trace: **BLENDINDICES must reach the shader as integers 0..255** (`uint4` input). With UNORM the index
+  becomes 0..1, so every vertex uses bone 0. That collapses the mesh to one bone but doesn't make it disappear.
+  **The weights must be UNORM.** Your host bytes (00000100 indices / 000001FE weights) pair up correctly only if both use the same WZYX swap. They do.
+- **The thing to compare at runtime is not the shader:** c48..c51 of the Vortech draws. A zero/garbage instance matrix or
+  bone palette makes the whole model disappear or collapse. That matches "torso, legs, arms all missing".
+
+## 4. Clip / viewport-transform state: a real gap in native
+| Writer | What it writes |
+|---|---|
+| **83FB8FD0** `(dev, enable)` | **PA_CL_VTE_CNTL (+10572) = enable ? 0x43F : 0x400**, and **PA_CL_CLIP_CNTL (+10564) bit16 CLIP_DISABLE = !enable**; dirty \|= 0x20, 0x80 |
+| 83FB8848 | PA_CL_CLIP_CNTL bits0-5 = user clip plane enables (D3DRS_CLIPPLANEENABLE); also +10420 = 0x1000 if any; no direct callers |
+| 83FCA320 (device init) | PA_CL_CLIP_CNTL \|= 0x80000 (bit19 DX_CLIP_SPACE_DEF: z in [0,w]) |
+
+Callers of 83FB8FD0(0): **83EB6D68** (the InsertCallback/virtual23 helper) and 83FB2898 (Swap's present blit). In 0x400
+mode the viewport scale/offset is OFF: the VS outputs **window-space** coordinates, and clipping is disabled. Native has no
+handling of +10572/+10564 at all (grep finds nothing in src/gpu_native). **Any 83EB6D68 draw then goes through the viewport
+transform a second time and lands off-screen or tiny.** Fix: when VTE == 0x400, set the host viewport to identity for
+window coordinates (or convert in the VS epilogue, as xenia does), and turn depth clip off when bit16 is set. Bit19 means
+D3D-style z, which is what D3D12 already does. **No evidence ties the world meshes to this.** They never call 83FB8FD0, so leave
+their VTE alone.
+
+## 5. 83EB6D68 / 83EB4480 virtual23 (open question 3)
+What 83EB6D68 does, statically: SetTexture slots **0, 1, 2** (83FB58A8 x3, i.e. three planes), sampler states for those 3 stages
+(83FB9640 / 83FB97E8), VS/PS (83FB6A30 / 83FB6828), Z off (83FB84C8(0)), cull off (83FB7DA0(0)), **VTE off
+(83FB8FD0(0))**, 83FBA160(0), then **DrawVertices(prim 4 = TRIANGLELIST, 6 verts)**, then InsertCallback 83FC0F38 ->
+83EB4480 -> `this->vtable[23]`. It is only reached through a function pointer, so there's no static caller.
+Three planes + screen space + a callback after the draw is the shape of a **video (YUV) frame presenter**: the callback releases
+the frame's surfaces back to the decoder. The executable contains `.bik` / `.wmv` / `bink` strings. **No .bik/.wmv/.xmv exists
+in GAME*.DAT or on the disc image**, so I don't expect this path to be active in the intro. The TT logo is the GSC scene
+from section 2, not a video. If it does run, section 4 is why it would be invisible. I haven't resolved the vtable[23] target (it needs
+the object's vtable from a runtime pointer), so lifetime and safety on the guest hook thread are still open.
+
+## 6. Dynamic edits to shader fetch/definition tables
+Not re-traced. From Addendum 17: the only consumer is 83FD20B8 -> 83FD1EC8, which reads the VS/PS object's tables at
++40/+872 each draw. I didn't search for writers to those offsets after creation. Next static step if needed: grep for stores at
++40..+60 / +872..+904 on objects stored to device+13068/+13072.
+
+# Addendum 19 (2026-10-01): the big Vortech is a KRAWLIE swarm; corrections; TT depth
+
+STATIC only. **Still: do not launch the game until the fix batch is complete; one run only when the user asks.**
+Corrections accepted: my "arm files together < 90 KB" in Addendum 18 was wrong arithmetic (120,722 B). Your byte
+matching supersedes it. Thanks for the exact offsets: all 8 skinned samples are vortech_body.
+
+## 1. Why the arm meshes never reach Draw: they are emitters, not drawn meshes (data proof, code partly traced)
+`cut\story\0001_prologue\0001_prologue.txt` (GAME1.DAT), cutscene **0001_PrologueC**:
+```
+// KRAWLIES!!!
+draw_krawlies
+addon WWCCutsceneKnight (Config='WWCCutsceneKnight', CutsceneChar='VORTECH_BODY',     NumberOfKrawlies=12000)
+addon WWCCutsceneKnight (Config='WWCCutsceneKnight', CutsceneChar='VORTECH_ARMRIGHT', NumberOfKrawlies=1200)
+addon WWCCutsceneKnight (Config='WWCCutsceneKnight', CutsceneChar='VORTECH_ARMLEFT',  NumberOfKrawlies=1200)
+```
+So the giant Vortech in this cutscene is **14,400 "krawlie" bricks** placed on the animated body/arm characters.
+The character GHGs (body, armleft, armright) are the **skeleton/surface the bricks follow**. The visible arms are
+**bricks**, not the arm mesh. **An arm mesh that never reaches Draw is therefore consistent with normal behaviour,
+not a CPU gate you need to find.** The renderer-side question becomes: where do the krawlie bricks go?
+- Brick asset: `commonobjects\krawlie_bricks_vortechblue_nxg.gsc` + `.360_shaders`. Its pairs:
+  VS `5FE844BD78B0368C`/PS `DAB518E550952312`, VS `C0C579A9C7D3ACD9`/PS `BC67385D86BABED8`,
+  VS `FFC80E0F8B28B228`/PS `1998CB74A7EDAFE1` and PS `336657153E47E0C3`, VS `362C416499E90A3B`/PS `0705E91ECC4BA3BB`.
+  **None of these VS have skinning or vertex samplers.** Their groups are only Camera, Instance c48-51, Instance2 c52-54,
+  Material c16-24, Lights c36-42, Fog, MiscGroup1 c72-73. So each brick is either one draw with its own c48-54, or
+  (more plausible for 14,400) **a CPU-built batch in a dynamic VB with world-space vertices** and identity-like c48-51.
+  Not proven which. (These hashes also occur in other banks, so log hits don't identify the swarm.)
+- Code anchors (string xrefs, lis/addi pairs): `"WWCCutsceneKnight"` 0x820D6850 is referenced by **82AF3448**
+  (a 1,352-instruction update function, probably the addon's per-frame update) and 84080A50. `"NumberOfKrawlies"`
+  0x820453F0 is referenced by **82722AA8, 82743D88, 83C0D9F0**. `"KrawlieRenderCharInstAddOn"` 0x821F3E5C and
+  `"draw_krawlies"` 0x823543C8 have no direct lis/addi xref (probably table-driven). I did not finish tracing from 82AF3448 to
+  the draw. Next static step: follow 82AF3448's callees for the dynamic-VB lock (Addendum 12/13 owners
+  82CBCBE8 / 82CBC9E0 / lock 82BD6088) or for 82B74FB8.
+- **Prime suspect class for native:** the dynamic, double-buffered VB path. The CPU rewrites the brick vertices every frame
+  and the buffer flips (Addendum 13). A stale, never-re-uploaded or wrong-parity copy gives you "missing or frozen bricks" while
+  static meshes look fine. That's the same class as Addendum 11's VTF re-upload rule, but for vertex buffers.
+  Check offline in your existing traces: do draws with VS in {5FE8, C0C5, FFC8, 362C} exist in the PrologueC window of mesh1-3,
+  and is their VB a dynamic/double-buffered owner?
+- Other krawlie uses exist (`chars\cutscene_krawlies\cut_crowd\*.krw`, `chars\krawlies\...\*.krw`: baked krawlie animations),
+  so the same fix would also cover crowds and swarms elsewhere.
+
+## 2. Body GHG bone/submesh tables
+Not parsed. No GHG parser exists here, and the format isn't documented in our research folders. With section 1 the
+priority drops: the body/arm meshes are the krawlie surfaces. Whether the body mesh itself is also meant to be visible
+(your 8 submitted body draws) or only as a hidden emitter (with the bricks the real visual) is decided by the addon,
+which I haven't traced. Torso/legs "missing" fits "bricks missing" as well.
+
+## 3. TT quads 18/24 vs the galaxy under ZFUNC GREATEREQUAL
+I can't derive per-quad depth/UV statically without parsing the GSC instance table. One concrete thing to check offline:
+**ZFUNC GEQUAL means the game expects a depth buffer cleared to 0 (or reversed-Z).** Check that native's Clear
+(83FBC9D8) honours the Z value and flags the game passes, and that a depth clear isn't skipped or replaced by 1.0
+before the logo pass. If the clear is 1.0, every GEQUAL test fails except at exactly 1.0, and a quad "behind the galaxy"
+is the visible symptom. The fact that your logo samples submit with alpha test off fits this. Opacity reaches 0.99, so it's not the cause.
+
+## 4. Stage ownership: corrected
+Confirmed from code: SetVertexShader 83FB6A30 stores to **device+13072**, SetPixelShader 83FB6828 to **device+13068**.
+In 83FD20B8, r30 = +13072 = **VS object, literal table at +872** (with +896 / +904 sub-offsets), and r29 = +13068 = **PS
+object, literal table at +40**, base at +24. Addendum 17 had VS/PS swapped. The mechanism is unchanged. When you trace
+post-creation writes, look at **VS+872.. and PS+40..**.
diff --git a/tools/gpu-aot/tiling-reference-20260929.md b/tools/gpu-aot/tiling-reference-20260929.md
new file mode 100644
index 0000000..ad0e469
--- /dev/null
+++ b/tools/gpu-aot/tiling-reference-20260929.md
@@ -0,0 +1,106 @@
+# LEGO TU23: measured Xenos tiling reference, 2026-09-29
+
+## Scope and capture
+
+Read-only rendering diagnostics; no native rendering change in this investigation.
+Xenos test PID14684 ran 21:39:52–21:40:30 on monitor2, sound ON, QuickStartup
+only, SDL_WINDOW_ACTIVATE_WHEN_SHOWN=0. Assistant requested normal close at
+21:40:29; not a crash. No keys, mouse inputs or focus activation.
+Computer Use capture showed a correctly coloured WB Games logo over the galaxy
+intro. This is not visual verification of the later story scene or hub.
+
+Trace: `rexlego/out/native-gpu/xenos-tiling-20260929-2140.log`.
+4662 events, partial frame0 then complete frames1/2, overflow=0.
+Opt-in environment variable LEGO_XENOS_TILING_TRACE; trace automatically arms
+at the first 1280-wide surface with a nonzero window offset. Draw packets are
+logged BEFORE bin predication, and actual draw / decoded resolve details after
+the applicable execution paths. No render registers or predication changed.
+Trace stops after three swaps or 20000 events. It covers early intro, NOT all
+worlds, arbitrary BeginTiling flag combinations or all geometry draw patterns.
+
+Only the separate win-amd64-release-xenos plugin was replaced. Previous DLL
+preserved beside it as rexgpu-xenos.pre-tiling-trace.dll. Diagnostic DLL SHA256:
+6510FD6FE2198B426B591B0FBD23C22E67002092536D6DA5633E4A5739F5D2F5.
+Xenos config unchanged before/after (SHA256
+029E0D611203EA9C957912CB1D78544E4052851C2945CDCE5A3FC5D2FB131CD6).
+Monitor2 was a command-line override. Native exe/config unchanged.
+
+## Observed tile pair
+
+Corresponding geometry draw in frame1: packet events1511/1678, draw
+events1512/1679. VS microcode hash166D424A038E594A,
+PS microcode hash9171C4BB046A5B2E, indices4680. These are Xenos microcode
+hashes, not necessarily the native archive's container hashes.
+
+- Tile0 window=(0,0), window scissor=(0,0)..(1280,512).
+- Tile1 window=(0,-512), window scissor=(0,512)..(1280,720).
+- Screen scissor remains (0,0)..(8192,8192) in BOTH, unlike the generic
+  suggestion that PA_SC_SCREEN_SCISSOR necessarily changes per tile.
+- Same viewport words: 44200000,44200000,C3B40000,43B40000,3F800000,00000000
+  (X scale/offset640/640, Y scale/offset-360/360, Z1/0).
+- Same VS constant-bank hash997FECFBBABA9FC1, PS158A1AC01E0263DC,
+  fetch99F3D8E5B7316F19 for both corresponding draws.
+- Tile0 maskFFFFFFFF/select80000003; tile1 mask8000000F/selectC.
+  Predicated resolve commands for the other tile are observed and skipped.
+- Frame2 repeats the same geometry pair (events3185/3352). Its banks differ
+  from frame1 but match each other within that frame: VS45C987895919889D,
+  PS9E20609B2AAD4F5F, fetchB75BF464743E03EA.
+
+Thus these captured draws do not show a tile-dependent shader constant or
+viewport adjustment. Equality of hashes is evidence for these banks/frames,
+not proof for every draw or world.
+
+## Resolve layout (frame1)
+
+| Source | Format | Tile0 destination base | Tile1 destination base |
+| --- | --- | --- | --- |
+| RT0 | 6 / 8_8_8_8 | 1E140000 | 1E3C0000 |
+| RT1 | 6 / 8_8_8_8 | 1DDA8000 | 1E028000 |
+| RT2 | 6 / 8_8_8_8 | 1DA10000 | 1DC90000 |
+| Depth (src4) | 22 / 24_8 | 1E4D8000 | 1E758000 |
+
+Each second base is first+0x280000 (=1280*512*4). Destination pitch register
+02D00500, decoded source rect tile0=(0,0,1280,512), tile1=(0,0,1280,208).
+Decoded destination offset is0,0 in both; placement is encoded in the changed
+base, not a nonzero decoded destination offset. Guest resolve vertices still
+refer to Y511.5..719.5 for tile1; window offset maps source back to eDRAM top.
+These are physical Xenos addresses and must not be hardcoded into native code.
+Colour tile resolves have clear_color=1, clear_depth=0. Depth resolves have
+both clear flags0. This does NOT mean depth is never cleared: clear draws are
+also in the command stream, and their complete clear-state semantics were not
+decoded by this trace.
+
+Later in the SAME frame: full1280x720 HDR resolve (src0, format26, exp11,
+number2) at event2269; full1280x720 depth resolve to1E4D8000 at event2271.
+Do not treat all subsequent resolves as duplicate tile replays and discard them.
+
+## Native gaps / review question
+
+Native draw.cpp currently creates a PSO with renderTargetCount0/1 and uses
+RT0. Xenos demonstrably resolves RT0,RT1,RT2 and depth in this pass. Bound RT
+registers alone do not establish each pixel shader's output mask; this trace
+does not log RB_COLOR_MASK or shader export masks. MRT support/output mapping
+must be checked before claiming a full-size depth change fixes the scene.
+
+BeginTilingHook currently logs rectangles and invokes ClearHook unless flags&1;
+it does NOT promote surfaces or reproduce the guest tiling state. Its old
+"using full host target" log is not evidence of an implemented full-size target.
+EndTilingHook directly resolves the selected native surface. Original TU23
+sub_83FBCD28 writes count+13124, rects+13128, aligned origins+13368,
+extent+13556/+13560 and flags+13564, then enters original clear/worker/command
+machinery. Native's deliberately minimal D3DDevice does not have that machinery;
+calling the original wholesale is unsafe (same reason for SetPredicationHook).
+
+Question for the peer reviewing TU23 (via user):
+Can we isolate a CPU-only BeginTiling/EndTiling state contract from
+sub_83FBCD28/sub_83FBD098, while retaining direct native draws and resolves?
+Which fields must be set/reset, and which unhooked consumers would interpret
+a nonzero +13124/+13564 as permission to access the absent Xbox command queues?
+Need concrete consumer/function addresses and end-of-tiling reset semantics,
+not a recommendation to call the full original or merely write count/extent.
+Host-side full-size RT0–RT2/DS can then be implemented without accidentally
+reactivating the old worker path. Do not implement speculative metadata writes.
+
+Build verification: rexgpu-xenos target compiled successfully, bounded capture
+completed normally, scoped SDK git diff --check passed. Native unchanged and
+not re-tested in this investigation. No new performance benchmark claim.
diff --git a/tools/gpu-aot/verify_alpha_archives.py b/tools/gpu-aot/verify_alpha_archives.py
new file mode 100644
index 0000000..b125328
--- /dev/null
+++ b/tools/gpu-aot/verify_alpha_archives.py
@@ -0,0 +1,44 @@
+"""Verify complete alpha candidate coverage; optionally install backed-up banks.
+
+Run installation only while the native game is closed. Never launches a game.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+import shutil
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('candidate', type=Path)
+p.add_argument('backup', type=Path)
+p.add_argument('--install', action='store_true')
+a = p.parse_args()
+root = Path(__file__).resolve().parents[2]
+native = root / 'rexlego/out/native-gpu'
+
+def entries(p):
+    # hash, DXIL offset/length, SPIRV offset/length, specialization mask.
+    return {(m[0], int(m[1])) for m in re.findall(
+        r'\{ 0x([A-Fa-f0-9]+), \d+, \d+, \d+, \d+, (\d+) \}', p.read_text())}
+
+report = {}
+for name, old_name, new_name, n in [('main', 'lego-fixed16-cache.cpp', 'lego-cache.cpp', 9290),
+                                    ('runtime', 'runtime-cache.cpp', 'runtime-cache.cpp', 118)]:
+    old, new = a.backup/old_name, a.candidate/new_name
+    prev, updated = entries(old), entries(new)
+    assert len(prev) == len(updated) == n and prev == updated, (name, len(prev), len(updated))
+    report[name] = {'containers': n, 'hash_and_specialization_masks_unchanged': True,
+                    'sha256': hashlib.sha256(new.read_bytes()).hexdigest()}
+main = (a.candidate/'main-compiler-color0.log').read_text()
+run = (a.candidate/'runtime-compiler-color0.log').read_text()
+assert 'Recompile failures: 1 of 9291' in main
+assert re.findall(r'hash=0x([A-Fa-f0-9]+) reason=', main) == ['A58EBA123EFB541C']
+assert 'AOT coverage: 9290 of 9291' in main and 'AOT coverage: 118 of 118' in run
+report.update({'known_excluded_memexport': 'A58EBA123EFB541C', 'shared_bytes': 624,
+               'alpha_function_offset': 600, 'game_visual_validation': False})
+(a.candidate/'coverage-proof.json').write_text(json.dumps(report, indent=2))
+if a.install:
+    shutil.copyfile(a.candidate/'lego-cache.cpp', native/'lego-fixed16-cache.cpp')
+    shutil.copyfile(a.candidate/'runtime-cache.cpp', native/'runtime-cache.cpp')
+print(json.dumps(report, indent=2))
diff --git a/tools/gpu-aot/verify_fps_build.py b/tools/gpu-aot/verify_fps_build.py
new file mode 100644
index 0000000..bb4bc38
--- /dev/null
+++ b/tools/gpu-aot/verify_fps_build.py
@@ -0,0 +1,58 @@
+"""Write a separate current-build proof without overwriting older probe proofs."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+root=Path(__file__).resolve().parents[2];build=root/'rexlego/out/build/win-amd64-release'
+fingerprint=re.search(r'kNativeGpuBuildFingerprint\[\] = "([0-9a-f]+)"',
+ (build/'generated/native_gpu_build_info.h').read_text())[1]
+exe=(build/'legodimensions.exe').read_bytes();assert fingerprint.encode() in exe
+for literal in [b'placement buffer relocation type',b'cpu_resource_wait_skips',b'Native synchronization timing',
+ b'Native callback queue', b'deferred callback=', b'asynchronous readback generations',
+ b'disabled DoF sharp/mip tonemap mix', b'render default setter=',
+ b'frame-ring GPU wait failed']:
+ assert literal in exe
+config=(build/'legodimensions.toml').read_bytes();sha=lambda b:hashlib.sha256(b).hexdigest()
+assert sha(config).upper()=='AC5DA9E5E080B4E999F28194424F3E11BEEFAC09232550047FE86BF6F75DDFA5'
+native=root/'rexlego/out/native-gpu';session=native/'session-20260930'
+assert len(re.findall(r'\{ 0x[A-Fa-f0-9]+, \d+, \d+, \d+, \d+, \d+ \}',(native/'runtime-cache.cpp').read_text()))==118
+assert 'g_shaderMicrocodeEntryCount = 18722;' in (native/'lego-microcode-index.cpp').read_text()
+tests={}
+for name in ['placement-resource-type-tests','inline-hash-sdk-tests','cpu-resource-wait-tests','completion-callback-tests',
+ 'buffer-variant-tests','draw-upload-tests','query-retirement-tests','resize-fence-tests','alpha-tests']:
+ proof=json.loads((session/name/'verification.json').read_text())
+ assert proof.get('passed',proof.get('identical',False));tests[name]=proof
+prelinked={}
+for name,raw,linked in [('main','lego-fixed16-cache.cpp','lego-linked-dxil-cache.cpp'),
+                         ('runtime','runtime-cache.cpp','runtime-linked-dxil-cache.cpp')]:
+ expected=set()
+ for shader,mask in re.findall(r'\{ 0x([A-Fa-f0-9]+), \d+, \d+, \d+, \d+, (\d+) \}',(native/raw).read_text()):
+  mask=int(mask)
+  if mask:
+   expected.update((int(shader,16),value) for value in range(mask+1) if not value&~mask)
+ actual=set()
+ with (native/linked).open() as stream:
+  for line in stream:
+   if line.strip()=='};':break
+   m=re.search(r'\{ 0x([A-Fa-f0-9]+), 0x([A-Fa-f0-9]+), (\d+), (\d+) \}',line)
+   if m:
+    assert int(m[4])>0
+    actual.add((int(m[1],16),int(m[2],16)))
+ assert expected==actual,(name,len(expected),len(actual))
+ prelinked[name]={'variants':len(actual),'all_specialization_subsets_present':True}
+report={'fingerprint':fingerprint,'exe_sha256':sha(exe),'config_sha256':sha(config),
+ 'runtime_containers':118,'index_entries':18722,'shared_constants_bytes':624,
+ 'cpu_resource_wait_optimization':'opt-in, default full waits',
+ 'insert_callback':'nonblocking enqueue; FIFO execution after containing submission completes',
+ 'profiler_marker_timing':'coarse completion-poll timing, not exact GPU timestamps',
+ 'query_lifetime':'fresh readback generation per BEGIN; retired at every live host fence; GetData remains fence-gated',
+ 'disabled_dof':'tonemap 3A47E5DDE66B42C6 host-only c12.xy=0 when depth_of_field=false; exposure/bloom/LUT unchanged',
+ 'alpha_test':'all 8 Xenos comparisons using original shader alpha and raw float reference; no test for depth-only shaders',
+ 'render_defaults':'15 audited TU23 leaf alpha/blend/color-mask setters, descriptor values and order; alpha reference default 1.0',
+ 'resize':'wait all submitted GPU fences before destruction; event wakeups gated by actual fence values; retry failed/minimized resize',
+ 'prelinked_shader_coverage':prelinked,
+ 'full_content_hash_validation_preserved':True,'tests':tests,'stable_30_fps_verified':False,
+ 'full_cutscene_verified':False,'settings_and_hub_verified':False}
+a.output.write_text(json.dumps(report,indent=2));print(json.dumps({k:v for k,v in report.items() if k!='tests'},indent=2))
diff --git a/tools/gpu-aot/watch_native_probe.ps1 b/tools/gpu-aot/watch_native_probe.ps1
new file mode 100644
index 0000000..f07c567
--- /dev/null
+++ b/tools/gpu-aot/watch_native_probe.ps1
@@ -0,0 +1,70 @@
+param([Parameter(Mandatory)][ValidatePattern('^[a-z0-9-]+$')][string]$Label)
+$ErrorActionPreference='Stop'
+$projectRoot=(Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
+$runRoot=Join-Path $projectRoot 'rexlego/out/native-gpu/session-20260930'
+$record=Get-Content -LiteralPath (Join-Path $runRoot "$Label-process.json") | ConvertFrom-Json
+$traceRoot=Join-Path $runRoot "trace-$Label"
+$logPath=Join-Path $runRoot "$Label-game.log"
+$captureRoot=Join-Path $runRoot "gpu-stages-$Label"
+New-Item -ItemType Directory -Path $captureRoot -Force | Out-Null
+$offset=0L; $pending=''; $lastGrowth=[datetime]::UtcNow; $stalled=$false
+$menuStart=$null; $idleComplete=$false
+$monitorLog=Join-Path $traceRoot 'monitor.jsonl'
+function Write-MonitorEvent([string]$kind,$detail) {
+    [ordered]@{utc=[datetime]::UtcNow.ToString('o');event=$kind;detail=$detail} |
+        ConvertTo-Json -Compress -Depth 5 | Add-Content -LiteralPath $monitorLog
+}
+Write-MonitorEvent 'watch_started' @{gamePid=$record.id;log=$logPath}
+while ($true) {
+    $game=Get-Process -Id $record.id -ErrorAction SilentlyContinue
+    if (!$game) { Write-MonitorEvent 'game_exited' @{}; break }
+    if ($game.Path -ne $record.path -or
+        $game.StartTime.ToUniversalTime().Ticks -ne ([datetime]$record.startedAt).ToUniversalTime().Ticks) {
+        Write-MonitorEvent 'identity_changed_stopping_watch' @{}; break
+    }
+    if (Test-Path -LiteralPath $logPath) {
+        $stream=[IO.File]::Open($logPath,[IO.FileMode]::Open,[IO.FileAccess]::Read,[IO.FileShare]::ReadWrite)
+        try {
+            if ($stream.Length -lt $offset) { $offset=0; $pending='' }
+            $length=$stream.Length-$offset
+            if ($length -gt 0) {
+                $lastGrowth=[datetime]::UtcNow; $stalled=$false
+                $stream.Position=$offset
+                $bytes=New-Object byte[] ([int][Math]::Min($length,4MB))
+                $read=$stream.Read($bytes,0,$bytes.Length); $offset+=$read
+                $text=$pending+[Text.Encoding]::UTF8.GetString($bytes,0,$read)
+                $lines=$text -split "`n"; $pending=$lines[-1]
+                for ($index=0;$index -lt $lines.Length-1;$index++) {
+                    $line=$lines[$index].TrimEnd("`r")
+                    if ($line -match '\[(error|critical)\]|Native GPU:.*(failed|mismatch|unsupported|unrecognized|exhausted)|Native D3D12') {
+                        Write-MonitorEvent 'log_anomaly_candidate' @{line=$line;offset=$offset}
+                        Set-Content -LiteralPath (Join-Path $captureRoot 'capture-next-frame') -Value 'log anomaly candidate'
+                    }
+                }
+            }
+        } finally { $stream.Dispose() }
+    }
+    if (!$stalled -and ([datetime]::UtcNow-$lastGrowth).TotalSeconds -gt 30) {
+        $stalled=$true
+        Write-MonitorEvent 'log_silent_30s_candidate' @{responding=$game.Responding}
+        Set-Content -LiteralPath (Join-Path $captureRoot 'capture-next-frame') -Value 'log silent candidate'
+    }
+    $marker=Join-Path $traceRoot 'menu-start.json'
+    if (!$menuStart -and (Test-Path -LiteralPath $marker)) {
+        $menuStart=[datetime](Get-Content -LiteralPath $marker | ConvertFrom-Json).utc
+        Write-MonitorEvent 'menu_idle_started' @{utc=$menuStart.ToUniversalTime().ToString('o')}
+        Set-Content -LiteralPath (Join-Path $captureRoot 'capture-next-frame') -Value 'menu idle baseline'
+    }
+    $idleSeconds=if ($menuStart) { ([datetime]::UtcNow-$menuStart.ToUniversalTime()).TotalSeconds } else { $null }
+    if ($menuStart -and !$idleComplete -and $idleSeconds -ge 600) {
+        $idleComplete=$true
+        Write-MonitorEvent 'menu_idle_600s_elapsed' @{idleSeconds=$idleSeconds}
+        Set-Content -LiteralPath (Join-Path $captureRoot 'capture-next-frame') -Value 'menu idle 600 seconds'
+    }
+    [ordered]@{utc=[datetime]::UtcNow.ToString('o');gamePid=$game.Id;workingSet=$game.WorkingSet64;
+        privateBytes=$game.PrivateMemorySize64;cpuSeconds=$game.CPU;responding=$game.Responding;
+        logBytesRead=$offset;secondsSinceLogGrowth=([datetime]::UtcNow-$lastGrowth).TotalSeconds;
+        menuIdleSeconds=$idleSeconds;menuIdle600sElapsed=$idleComplete} |
+        ConvertTo-Json | Set-Content -LiteralPath (Join-Path $traceRoot 'heartbeat.json')
+    Start-Sleep -Seconds 2
+}

From c0172a94474a87547f1bfab61d43f40921c4258b Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 20:36:33 +0300
Subject: [PATCH 17/44] Checkpoint integrated native D3D12 renderer and SDK
 fallback

---
 rexglue-sdk                                   |    2 +-
 rexlego/CMakeLists.txt                        |  104 +-
 rexlego/src/cheat_menu.cpp                    |    2 -
 rexlego/src/cheat_menu.h                      |    2 +
 rexlego/src/gpu_native/alpha_test.h           |   16 +
 rexlego/src/gpu_native/blend_state.h          |   24 +
 rexlego/src/gpu_native/buffer_header.h        |   21 +
 rexlego/src/gpu_native/buffer_window.h        |   19 +
 rexlego/src/gpu_native/buffers.cpp            |  255 +++-
 rexlego/src/gpu_native/buffers.h              |   31 +-
 rexlego/src/gpu_native/completion_queue.h     |   30 +
 rexlego/src/gpu_native/d3d.h                  |    9 +-
 rexlego/src/gpu_native/depth_alias.h          |   23 +
 rexlego/src/gpu_native/depth_state.h          |   29 +
 .../src/gpu_native/descriptor_retirement.h    |   31 +
 rexlego/src/gpu_native/device.cpp             |  859 ++++++++++-
 rexlego/src/gpu_native/device.h               |   58 +
 rexlego/src/gpu_native/draw.cpp               | 1128 ++++++++++++--
 rexlego/src/gpu_native/draw.h                 |   11 +
 rexlego/src/gpu_native/format.cpp             |   16 +
 rexlego/src/gpu_native/hooks_buffer.cpp       |  151 +-
 rexlego/src/gpu_native/hooks_device.cpp       |  219 ++-
 rexlego/src/gpu_native/hooks_shader.cpp       |   14 +-
 rexlego/src/gpu_native/hooks_state.cpp        |   71 +-
 rexlego/src/gpu_native/hooks_texture.cpp      |    2 +-
 rexlego/src/gpu_native/linked_shader_cache.h  |    6 +
 rexlego/src/gpu_native/long_probe.h           |   89 ++
 rexlego/src/gpu_native/memory_watch.cpp       |  129 ++
 rexlego/src/gpu_native/memory_watch.h         |   24 +
 rexlego/src/gpu_native/pixel_shader_filter.h  |   26 +
 rexlego/src/gpu_native/pool_copy.h            |   37 +
 rexlego/src/gpu_native/queries.cpp            |  204 +++
 rexlego/src/gpu_native/queries.h              |   19 +
 rexlego/src/gpu_native/renderdoc_probe.h      |   54 +
 rexlego/src/gpu_native/renderer_mode.cpp      |    7 +
 rexlego/src/gpu_native/renderer_mode.h        |   25 +
 rexlego/src/gpu_native/renderer_route.h       |   29 +
 rexlego/src/gpu_native/resolve_region.h       |   28 +
 .../runtime_linked_shader_cache.cpp           |    6 +
 .../src/gpu_native/runtime_shader_cache.cpp   |    9 +
 rexlego/src/gpu_native/sampler_state.h        |  102 ++
 rexlego/src/gpu_native/shader_archive.cpp     |  150 +-
 rexlego/src/gpu_native/shader_archive.h       |   32 +-
 rexlego/src/gpu_native/shader_texture_usage.h |   47 +
 rexlego/src/gpu_native/shaders.cpp            |  318 +++-
 rexlego/src/gpu_native/shaders.h              |    9 +
 .../src/gpu_native/shaders/depth_pack_ps.hlsl |   12 +
 .../gpu_native/shaders/depth_restore_ps.hlsl  |   13 +
 .../gpu_native/shaders/resolve_color_ps.hlsl  |   12 +
 rexlego/src/gpu_native/state.cpp              |   18 +
 rexlego/src/gpu_native/state.h                |    9 +-
 rexlego/src/gpu_native/stencil_state.h        |   30 +
 rexlego/src/gpu_native/texture_number.h       |   22 +
 rexlego/src/gpu_native/texture_upload_cache.h |   29 +
 .../src/gpu_native/texture_volume_upload.h    |   41 +
 rexlego/src/gpu_native/textures.cpp           | 1309 ++++++++++++++++-
 rexlego/src/gpu_native/textures.h             |   40 +-
 rexlego/src/gpu_native/tile_extent.h          |   34 +
 rexlego/src/gpu_native/tonemap_dof.h          |   21 +
 rexlego/src/gpu_native/vertex_byte_order.h    |   45 +
 .../src/gpu_native/vertex_declarations.cpp    |   91 +-
 rexlego/src/gpu_native/vertex_declarations.h  |    2 +
 rexlego/src/legodimensions_app.h              |    9 +-
 thirdparty/plume                              |    2 +-
 64 files changed, 5921 insertions(+), 295 deletions(-)
 create mode 100644 rexlego/src/gpu_native/alpha_test.h
 create mode 100644 rexlego/src/gpu_native/blend_state.h
 create mode 100644 rexlego/src/gpu_native/buffer_header.h
 create mode 100644 rexlego/src/gpu_native/buffer_window.h
 create mode 100644 rexlego/src/gpu_native/completion_queue.h
 create mode 100644 rexlego/src/gpu_native/depth_alias.h
 create mode 100644 rexlego/src/gpu_native/depth_state.h
 create mode 100644 rexlego/src/gpu_native/descriptor_retirement.h
 create mode 100644 rexlego/src/gpu_native/long_probe.h
 create mode 100644 rexlego/src/gpu_native/memory_watch.cpp
 create mode 100644 rexlego/src/gpu_native/memory_watch.h
 create mode 100644 rexlego/src/gpu_native/pixel_shader_filter.h
 create mode 100644 rexlego/src/gpu_native/pool_copy.h
 create mode 100644 rexlego/src/gpu_native/queries.cpp
 create mode 100644 rexlego/src/gpu_native/queries.h
 create mode 100644 rexlego/src/gpu_native/renderdoc_probe.h
 create mode 100644 rexlego/src/gpu_native/renderer_mode.cpp
 create mode 100644 rexlego/src/gpu_native/renderer_mode.h
 create mode 100644 rexlego/src/gpu_native/renderer_route.h
 create mode 100644 rexlego/src/gpu_native/resolve_region.h
 create mode 100644 rexlego/src/gpu_native/runtime_linked_shader_cache.cpp
 create mode 100644 rexlego/src/gpu_native/runtime_shader_cache.cpp
 create mode 100644 rexlego/src/gpu_native/sampler_state.h
 create mode 100644 rexlego/src/gpu_native/shader_texture_usage.h
 create mode 100644 rexlego/src/gpu_native/shaders/depth_pack_ps.hlsl
 create mode 100644 rexlego/src/gpu_native/shaders/depth_restore_ps.hlsl
 create mode 100644 rexlego/src/gpu_native/shaders/resolve_color_ps.hlsl
 create mode 100644 rexlego/src/gpu_native/stencil_state.h
 create mode 100644 rexlego/src/gpu_native/texture_number.h
 create mode 100644 rexlego/src/gpu_native/texture_upload_cache.h
 create mode 100644 rexlego/src/gpu_native/texture_volume_upload.h
 create mode 100644 rexlego/src/gpu_native/tile_extent.h
 create mode 100644 rexlego/src/gpu_native/tonemap_dof.h
 create mode 100644 rexlego/src/gpu_native/vertex_byte_order.h

diff --git a/rexglue-sdk b/rexglue-sdk
index d846a43..17e3675 160000
--- a/rexglue-sdk
+++ b/rexglue-sdk
@@ -1 +1 @@
-Subproject commit d846a4310369323e054f683141145ee96cc6f982
+Subproject commit 17e3675767067339581018c7e722e1d8d65de201
diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index fdc542d..991c37c 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -56,6 +56,9 @@ if(LEGODIMENSIONS_NATIVE_GPU)
 
     list(APPEND LEGODIMENSIONS_SOURCES
         src/gpu_native/buffers.cpp
+        src/gpu_native/renderer_mode.cpp
+        src/gpu_native/memory_watch.cpp
+        src/gpu_native/queries.cpp
         src/gpu_native/device.cpp
         src/gpu_native/draw.cpp
         src/gpu_native/format.cpp
@@ -69,6 +72,8 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         src/gpu_native/state.cpp
         src/gpu_native/textures.cpp
         src/gpu_native/vertex_declarations.cpp
+        "${CMAKE_CURRENT_SOURCE_DIR}/../rexglue-sdk/src/graphics/pipeline/texture/util.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../rexglue-sdk/src/graphics/pipeline/texture/info.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c"
     )
@@ -107,7 +112,8 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     # multi-megabyte generated source file while still making an enabled build
     # fail loudly if the AOT stage has not been run.
     set(LEGO_NATIVE_SHADER_CACHE
-        "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/lego-dxil-cache.cpp")
+        "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/lego-dxil-cache.cpp"
+        CACHE FILEPATH "AOT DXIL cache to embed (keep older caches for rollback)")
     if(NOT EXISTS "${LEGO_NATIVE_SHADER_CACHE}")
         message(FATAL_ERROR
             "Native GPU needs ${LEGO_NATIVE_SHADER_CACHE}; run tools/gpu-aot first")
@@ -115,6 +121,24 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     target_sources(legodimensions PRIVATE "${LEGO_NATIVE_SHADER_CACHE}")
     set_source_files_properties("${LEGO_NATIVE_SHADER_CACHE}"
         PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+    set(LEGO_NATIVE_MICROCODE_INDEX
+        "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/lego-microcode-index.cpp")
+    if(NOT EXISTS "${LEGO_NATIVE_MICROCODE_INDEX}")
+        message(FATAL_ERROR
+            "Native GPU needs ${LEGO_NATIVE_MICROCODE_INDEX}; run lego_gpu_microcode_index first")
+    endif()
+    target_sources(legodimensions PRIVATE "${LEGO_NATIVE_MICROCODE_INDEX}")
+    set_source_files_properties("${LEGO_NATIVE_MICROCODE_INDEX}"
+        PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+    set(LEGO_NATIVE_RUNTIME_SHADER_CACHE
+        "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/runtime-cache.cpp")
+    if(NOT EXISTS "${LEGO_NATIVE_RUNTIME_SHADER_CACHE}")
+        message(FATAL_ERROR
+            "Native GPU needs ${LEGO_NATIVE_RUNTIME_SHADER_CACHE}; capture and compile runtime shaders first")
+    endif()
+    target_sources(legodimensions PRIVATE
+        src/gpu_native/runtime_shader_cache.cpp
+        src/gpu_native/runtime_linked_shader_cache.cpp)
 
     # XenosRecomp emits a DXIL library whenever a shader reads one of its two
     # specialization bits. Link every used value at build time; gameplay then
@@ -135,7 +159,7 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_BINARY_DIR}/generated/gpu_native/shaders")
     set(LEGO_NATIVE_DXC
         "${LEGO_DXC_ROOT}/bin/${LEGO_DXC_ARCH}/dxc.exe")
-    foreach(LEGO_HOST_SHADER IN ITEMS copy_vs copy_color_ps)
+    foreach(LEGO_HOST_SHADER IN ITEMS copy_vs copy_color_ps resolve_color_ps depth_pack_ps depth_restore_ps)
         if(LEGO_HOST_SHADER STREQUAL "copy_vs")
             set(LEGO_HOST_SHADER_PROFILE vs_6_6)
         else()
@@ -164,6 +188,10 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     target_include_directories(legodimensions PRIVATE
         "${CMAKE_CURRENT_BINARY_DIR}/generated")
 
+    add_executable(lego_gpu_microcode_index EXCLUDE_FROM_ALL
+        "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/generate_microcode_index.cpp")
+    target_link_libraries(lego_gpu_microcode_index PRIVATE rex::xxhash)
+
     add_executable(lego_gpu_prelink EXCLUDE_FROM_ALL
         "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/prelink_shader_cache.cpp"
         src/gpu_native/dxc_link.cpp
@@ -184,6 +212,26 @@ if(LEGODIMENSIONS_NATIVE_GPU)
             "$<TARGET_FILE_DIR:lego_gpu_prelink>"
         VERBATIM)
 
+    add_executable(lego_gpu_prelink_runtime EXCLUDE_FROM_ALL
+        "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/prelink_shader_cache.cpp"
+        src/gpu_native/dxc_link.cpp
+        "${LEGO_NATIVE_RUNTIME_SHADER_CACHE}"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz/miniz.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd/zstddeclib.c")
+    target_include_directories(lego_gpu_prelink_runtime PRIVATE
+        src
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz"
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd"
+        "${LEGO_DXC_ROOT}/inc")
+    target_link_libraries(lego_gpu_prelink_runtime PRIVATE
+        "${LEGO_DXC_ROOT}/lib/${LEGO_DXC_ARCH}/dxcompiler.lib")
+    add_custom_command(TARGET lego_gpu_prelink_runtime POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "${LEGO_DXC_ROOT}/bin/${LEGO_DXC_ARCH}/dxcompiler.dll"
+            "${LEGO_DXC_ROOT}/bin/${LEGO_DXC_ARCH}/dxil.dll"
+            "$<TARGET_FILE_DIR:lego_gpu_prelink_runtime>"
+        VERBATIM)
+
     add_custom_command(
         OUTPUT "${LEGO_NATIVE_LINKED_SHADER_CACHE}"
         COMMAND "$<TARGET_FILE:lego_gpu_prelink>"
@@ -194,19 +242,63 @@ if(LEGODIMENSIONS_NATIVE_GPU)
     target_sources(legodimensions PRIVATE "${LEGO_NATIVE_LINKED_SHADER_CACHE}")
     set_source_files_properties("${LEGO_NATIVE_LINKED_SHADER_CACHE}"
         PROPERTIES SKIP_PRECOMPILE_HEADERS ON)
+    set(LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE
+        "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/runtime-linked-dxil-cache.cpp")
+    add_custom_command(
+        OUTPUT "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}"
+        COMMAND "$<TARGET_FILE:lego_gpu_prelink_runtime>"
+                "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}"
+        DEPENDS lego_gpu_prelink_runtime
+        COMMENT "Pre-linking LEGO runtime shader variants"
+        VERBATIM)
+    add_custom_target(lego_runtime_linked_cache
+        DEPENDS "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}")
+    add_dependencies(legodimensions lego_runtime_linked_cache)
+    set_source_files_properties(
+        src/gpu_native/runtime_linked_shader_cache.cpp
+        PROPERTIES OBJECT_DEPENDS "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}")
 
     # Fingerprint the exact native renderer inputs. CMAKE_CONFIGURE_DEPENDS
     # makes a header-only edit trigger reconfigure as well as recompilation,
     # and the fingerprint printed at startup proves which executable is live.
     set(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS
+        "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/plume/plume_d3d12.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderer_route.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderer_mode.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderer_mode.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderdoc_probe.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/legodimensions_app.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/long_probe.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/memory_watch.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/memory_watch.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/descriptor_retirement.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/resolve_region.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/pixel_shader_filter.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/pool_copy.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/sampler_state.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/cheat_menu.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/draw.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/draw.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/alpha_test.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/tonemap_dof.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_common.hlsli"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_vs.hlsl"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/copy_color_ps.hlsl"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/resolve_color_ps.hlsl"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/depth_pack_ps.hlsl"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders/depth_restore_ps.hlsl"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/depth_alias.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/d3d.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/depth_state.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/blend_state.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/texture_number.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/texture_volume_upload.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/tile_extent.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shader_texture_usage.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/texture_upload_cache.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/vertex_byte_order.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/format.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/format.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/textures.cpp"
@@ -219,6 +311,9 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/vertex_declarations.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffers.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/queries.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/queries.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffer_header.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_buffer.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_device.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_shader.cpp"
@@ -226,8 +321,11 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shader_archive.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shader_archive.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/runtime_shader_cache.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/runtime_linked_shader_cache.cpp"
         "${LEGO_NATIVE_SHADER_CACHE}"
-        "${LEGO_NATIVE_LINKED_SHADER_CACHE}")
+        "${LEGO_NATIVE_LINKED_SHADER_CACHE}"
+        "${LEGO_NATIVE_RUNTIME_SHADER_CACHE}")
     set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
         ${LEGO_NATIVE_GPU_FINGERPRINT_INPUTS})
     set(LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL "")
diff --git a/rexlego/src/cheat_menu.cpp b/rexlego/src/cheat_menu.cpp
index 5da1fca..5dac773 100644
--- a/rexlego/src/cheat_menu.cpp
+++ b/rexlego/src/cheat_menu.cpp
@@ -273,8 +273,6 @@ void Shutdown() {
 // menu gets a plain on/off instead of a hex string.
 namespace {
 
-constexpr std::string_view kDofPixelShaderHash = "CFEAC7ADB912F8A9";
-
 void ApplyDepthOfField(bool enabled) {
   const std::string current = rex::cvar::GetFlagByName("skip_pixel_shaders");
   std::string next;
diff --git a/rexlego/src/cheat_menu.h b/rexlego/src/cheat_menu.h
index f66832c..9457b19 100644
--- a/rexlego/src/cheat_menu.h
+++ b/rexlego/src/cheat_menu.h
@@ -6,6 +6,7 @@
 #pragma once
 
 #include <memory>
+#include <string_view>
 
 namespace rex::ui {
 class ImGuiDialog;
@@ -13,6 +14,7 @@ class ImGuiDrawer;
 }  // namespace rex::ui
 
 namespace legodimensions::cheats {
+inline constexpr std::string_view kDofPixelShaderHash = "CFEAC7ADB912F8A9";
 
 // Creates the DEL dialog. The caller owns it; destroying it closes the menu,
 // the same way the mod menu and the SDK overlays are toggled. The scanner
diff --git a/rexlego/src/gpu_native/alpha_test.h b/rexlego/src/gpu_native/alpha_test.h
new file mode 100644
index 0000000..bb5a42b
--- /dev/null
+++ b/rexlego/src/gpu_native/alpha_test.h
@@ -0,0 +1,16 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct NativeAlphaState {
+  bool enabled;
+  std::uint32_t function;
+  float reference;
+};
+inline NativeAlphaState DecodeNativeAlphaState(std::uint32_t color_control,
+                                               float reference) {
+  // TU23 83FB7E00 / 83FB82C0: enable bit 3, raw Xenos compare bits 0..2.
+  // 83FB92B8 stores FLOAT BITS unchanged; no integer /255 or /256 mapping.
+  return {(color_control & 8u) != 0, color_control & 7u, reference};
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/blend_state.h b/rexlego/src/gpu_native/blend_state.h
new file mode 100644
index 0000000..d6dc28d
--- /dev/null
+++ b/rexlego/src/gpu_native/blend_state.h
@@ -0,0 +1,24 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct ColorBlendState {
+  uint32_t source, destination, operation;
+  uint32_t alpha_source, alpha_destination, alpha_operation;
+  bool enabled, supported;
+};
+constexpr bool SupportedBlendFactor(uint32_t value) {
+  // Constant alpha for RGB needs a distinct host representation, not silently
+  // substituting constant RGB. Keep that case explicitly unsupported for now.
+  return value <= 1 || (value >= 4 && value <= 13) || value == 16;
+}
+constexpr ColorBlendState DecodeColorBlend(uint32_t control) {
+  ColorBlendState state{control & 31, (control >> 8) & 31, (control >> 5) & 7,
+      (control >> 16) & 31, (control >> 24) & 31, (control >> 21) & 7};
+  state.enabled = (control & 0x1FFF1FFFu) != 0x00010001u;
+  state.supported = state.operation <= 4 && state.alpha_operation <= 4 &&
+      SupportedBlendFactor(state.source) && SupportedBlendFactor(state.destination) &&
+      SupportedBlendFactor(state.alpha_source) && SupportedBlendFactor(state.alpha_destination);
+  return state;
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/buffer_header.h b/rexlego/src/gpu_native/buffer_header.h
new file mode 100644
index 0000000..2b1471f
--- /dev/null
+++ b/rexlego/src/gpu_native/buffer_header.h
@@ -0,0 +1,21 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct PlacementBufferInfo {
+  uint32_t address = 0;
+  uint32_t size = 0;
+  uint32_t index_format = 0;  // Native convention: 1 = 16-bit, 2 = 32-bit.
+};
+
+// TU23 asset headers observed in game.log; matches re:Blue's physical-buffer
+// adoption. These are initialized fetch headers, not XGSet*BufferHeader inputs.
+constexpr PlacementBufferInfo DecodePlacementBuffer(uint32_t common,
+    uint32_t fetch_lo, uint32_t fetch_hi, bool index) {
+  if ((common & 31u) != (index ? 2u : 1u)) return {};
+  const uint32_t address = fetch_lo & ~3u;
+  const uint32_t size = fetch_hi & 0x03FFFFFCu;
+  if (!address || !size || uint64_t(address) + size > 0x100000000ull) return {};
+  return {address, size, index ? ((common & 0x80000000u) ? 2u : 1u) : 0u};
+}
+}
diff --git a/rexlego/src/gpu_native/buffer_window.h b/rexlego/src/gpu_native/buffer_window.h
new file mode 100644
index 0000000..75ab2f7
--- /dev/null
+++ b/rexlego/src/gpu_native/buffer_window.h
@@ -0,0 +1,19 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct VertexBufferWindow { uint32_t offset = 0, length = 0; };
+// The rebased index is index-min_index; the upload starts at
+// stream_offset+(min_index+original_base_vertex)*stride.
+constexpr VertexBufferWindow DrawVertexWindow(uint32_t min_index, uint32_t max_index,
+    int32_t base_vertex, uint32_t stride, uint32_t stream_offset, uint32_t buffer_length) {
+  const int64_t first = int64_t(min_index) + base_vertex;
+  const int64_t last = int64_t(max_index) + base_vertex;
+  if (!stride || first < 0 || last < first) return {};
+  const uint64_t offset = uint64_t(stream_offset) + uint64_t(first) * stride;
+  const uint64_t length = uint64_t(last - first + 1) * stride;
+  if (offset > buffer_length || length > buffer_length - offset ||
+      offset % 4 || length % 4) return {};
+  return {uint32_t(offset), uint32_t(length)};
+}
+}
diff --git a/rexlego/src/gpu_native/buffers.cpp b/rexlego/src/gpu_native/buffers.cpp
index 294d279..c10ba71 100644
--- a/rexlego/src/gpu_native/buffers.cpp
+++ b/rexlego/src/gpu_native/buffers.cpp
@@ -1,6 +1,11 @@
 #include "gpu_native/buffers.h"
+#include "gpu_native/buffer_header.h"
+#include "gpu_native/memory_watch.h"
 
 #include <cstring>
+#include <algorithm>
+#include <atomic>
+#include <chrono>
 #include <memory>
 #include <mutex>
 #include <unordered_map>
@@ -8,12 +13,17 @@
 #include <vector>
 
 #include <plume_render_interface.h>
+// Use the game's configured ISA for bulk hashing (AVX2 on x86-64-v3),
+// retaining XXH3's exact hash and every-use content validation.
+#define XXH_INLINE_ALL
+#include <xxhash.h>
 #include <rex/logging.h>
 #include <rex/system/kernel_state.h>
 #include <rex/system/xmemory.h>
 
 #include "gpu_native/d3d.h"
 #include "gpu_native/device.h"
+#include "gpu_native/textures.h"
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -24,16 +34,27 @@ struct BufferResource {
   u32 length = 0;
   u32 format = 0;
   BufferKind kind = BufferKind::kVertex;
+  bool owns_guest_memory = true;
   std::mutex upload_mutex;
-  std::unique_ptr<plume::RenderBuffer> buffer;
+  std::shared_ptr<plume::RenderBuffer> buffer;
+  u64 content_hash = 0;
+  u64 byte_order_hash = 0;
+  CpuMemoryStamp cpu_stamp;
+  // Immutable conversions of the SAME contents for alternating declarations.
+  std::unordered_map<u64, std::shared_ptr<plume::RenderBuffer>> variants;
+  std::unordered_map<u64, std::shared_ptr<BufferResource>> windows;
+  u64 window_cache_bytes = 0;
+  u64 window_use_serial = 0;
+  u64 last_window_use = 0;
 };
 
 std::mutex g_buffers_mutex;
 std::unordered_map<u32, std::shared_ptr<BufferResource>> g_buffers;
+BufferUploadTiming g_buffer_timing;  // Serialized by LockRecording.
 
 D3DBuffer* GuestBuffer(u32 guest_address) {
   auto* memory = REX_KERNEL_MEMORY();
-  return reinterpret_cast<D3DBuffer*>(memory->virtual_membase() + guest_address);
+  return memory->TranslateVirtual<D3DBuffer*>(guest_address);
 }
 
 std::shared_ptr<BufferResource> FindBuffer(u32 guest_address) {
@@ -42,6 +63,39 @@ std::shared_ptr<BufferResource> FindBuffer(u32 guest_address) {
   return it == g_buffers.end() ? nullptr : it->second;
 }
 
+std::shared_ptr<BufferResource> AdoptBuffer(u32 address, BufferKind kind) {
+  if (!address || !HostDevice::IsReady()) return nullptr;
+  auto existing = FindBuffer(address);
+  if (existing && existing->owns_guest_memory) return existing;
+  const auto* header = GuestBuffer(address);
+  const auto info = DecodePlacementBuffer(header->resource.common,
+      header->fetch_lo, header->fetch_hi, kind == BufferKind::kIndex);
+  if (!info.address) return nullptr;
+  if (existing && existing->kind == kind && existing->mirror_address == info.address &&
+      existing->length == info.size && existing->format == info.index_format)
+    return existing;
+  auto resource = std::make_shared<BufferResource>();
+  resource->guest_address = address;
+  resource->mirror_address = info.address;
+  resource->length = info.size;
+  resource->format = info.index_format;
+  resource->kind = kind;
+  resource->owns_guest_memory = false;
+  {
+    std::lock_guard lock(g_buffers_mutex);
+    auto& entry = g_buffers[address];
+    if (entry != existing) return entry;  // Another caller adopted/replaced it.
+    entry = resource;
+  }
+  if (existing) HostDevice::RetireResource(std::move(existing));
+  static std::atomic<u32> adopted_logs{0};
+  if (adopted_logs.fetch_add(1) < 32)
+    REXLOG_INFO("Native GPU: adopted {} buffer 0x{:08X} data=0x{:08X} bytes={} index_format={}",
+        kind == BufferKind::kIndex ? "index" : "vertex", address,
+        info.address, info.size, info.index_format);
+  return resource;
+}
+
 void ByteSwapElements(void* destination, const void* source, size_t size,
                       u32 element_size) {
   auto* dst = static_cast<u8*>(destination);
@@ -67,6 +121,36 @@ void ByteSwapElements(void* destination, const void* source, size_t size,
 
 }  // namespace
 
+BufferUploadTiming ConsumeBufferUploadTiming() {
+  auto recording = HostDevice::LockRecording();
+  return std::exchange(g_buffer_timing, {});
+}
+
+u32 InvalidatePoolCopyBuffers(PhysicalCopyRange destination) {
+  auto recording = HostDevice::LockRecording();
+  std::vector<std::shared_ptr<BufferResource>> retired;
+  {
+    std::lock_guard lock(g_buffers_mutex);
+    for (auto it = g_buffers.begin(); it != g_buffers.end();) {
+      const auto& resource = it->second;
+      if (!resource->owns_guest_memory &&
+          (GuestRangeOverlapsPoolCopy(resource->mirror_address, resource->length, destination) ||
+           GuestRangeOverlapsPoolCopy(resource->guest_address, sizeof(D3DBuffer), destination))) {
+        static std::atomic<u32> logs{0};
+        if (logs.fetch_add(1, std::memory_order_relaxed) < 32)
+          REXLOG_INFO("Native GPU: pool copy invalidates {} buffer {:08X} data={:08X} bytes={}",
+              resource->kind == BufferKind::kIndex ? "index" : "vertex",
+              resource->guest_address, resource->mirror_address, resource->length);
+        retired.push_back(std::move(it->second));
+        it = g_buffers.erase(it);
+      } else ++it;
+    }
+  }
+  const u32 count = u32(retired.size());
+  for (auto& resource : retired) HostDevice::RetireResource(std::move(resource));
+  return count;
+}
+
 u32 CreateBufferResource(u32 length, u32 /*usage*/, u32 format, u32 /*pool*/,
                          BufferKind kind) {
   if (!length || !HostDevice::IsReady()) {
@@ -106,7 +190,7 @@ u32 CreateBufferResource(u32 length, u32 /*usage*/, u32 format, u32 /*pool*/,
 
 u32 LockBufferResource(u32 guest_address, u32 offset, u32 size, u32 /*flags*/,
                        BufferKind kind) {
-  const auto resource = FindBuffer(guest_address);
+  const auto resource = AdoptBuffer(guest_address, kind);
   if (!resource || resource->kind != kind || offset > resource->length ||
       (size != 0 && size > resource->length - offset)) {
     return 0;
@@ -114,13 +198,50 @@ u32 LockBufferResource(u32 guest_address, u32 offset, u32 size, u32 /*flags*/,
   return resource->mirror_address + offset;
 }
 
-plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind) {
-  const auto resource = FindBuffer(guest_address);
+plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr<BufferResource>& resource,
+    u32 guest_address, BufferKind kind, const VertexByteOrder& byte_order) {
   if (!resource || resource->kind != kind) {
+    static std::atomic<u32> missing_logs{0};
+    if (guest_address && missing_logs.fetch_add(1) < 8) {
+      const auto* words = REX_KERNEL_MEMORY()->TranslateVirtual<const be_u32*>(guest_address);
+      REXLOG_WARN("Native GPU: missing {} buffer 0x{:08X} header={:08X} {:08X} {:08X} {:08X} {:08X} {:08X} {:08X} {:08X}",
+          static_cast<u32>(kind), guest_address, u32(words[0]), u32(words[1]), u32(words[2]),
+          u32(words[3]), u32(words[4]), u32(words[5]), u32(words[6]), u32(words[7]));
+    }
     return nullptr;
   }
   std::lock_guard upload_lock(resource->upload_mutex);
-  if (!resource->buffer) {
+  if (!byte_order.reversed_elements.empty() &&
+      (byte_order.stream_offset > resource->length || byte_order.stride < 4)) return nullptr;
+  const auto conversion_order = CanonicalVertexByteOrder(byte_order);
+  const u64 byte_order_hash = conversion_order.reversed_elements.empty() ? 0 :
+      XXH3_64bits_withSeed(conversion_order.reversed_elements.data(),
+          conversion_order.reversed_elements.size_bytes(),
+          (u64(conversion_order.stream_offset) << 32) | conversion_order.stride);
+  const auto* source =
+      REX_KERNEL_MEMORY()->TranslateVirtual<const u8*>(resource->mirror_address);
+  const bool timing = NativeTextureTimingEnabled();
+  const auto hash_start = timing ? std::chrono::steady_clock::now()
+                                : std::chrono::steady_clock::time_point{};
+  const u64 content_hash = XXH3_64bits(source, resource->length);
+  if (timing) {
+    ++g_buffer_timing.calls;
+    g_buffer_timing.hashed_bytes += resource->length;
+    g_buffer_timing.hash_ms += std::chrono::duration<double, std::milli>(
+        std::chrono::steady_clock::now() - hash_start).count();
+  }
+  if (resource->content_hash != content_hash) {
+    for (auto& [key, version] : resource->variants) HostDevice::RetireResource(std::move(version));
+    resource->variants.clear();
+    resource->buffer.reset();
+  } else if (const auto it = resource->variants.find(byte_order_hash); it != resource->variants.end()) {
+    resource->buffer = it->second;
+    return it->second.get();
+  }
+  // Never overwrite an UPLOAD buffer referenced by already recorded draws.
+  // CPU changes get a new version; the previous one survives its frame fence.
+  std::shared_ptr<plume::RenderBuffer> next_buffer;
+  {
     auto* device = HostDevice::Device();
     if (!device) {
       return nullptr;
@@ -130,38 +251,123 @@ plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind) {
                                 resource->length, plume::RenderHeapType::UPLOAD)
                           : plume::RenderBufferDesc::VertexBuffer(
                                 resource->length, plume::RenderHeapType::UPLOAD);
-    resource->buffer = device->createBuffer(desc);
-    if (!resource->buffer) {
+    next_buffer = device->createBuffer(desc);
+    if (!next_buffer) {
       REXLOG_ERROR("Native GPU: failed to allocate {} buffer ({} bytes)",
                    kind == BufferKind::kIndex ? "index" : "vertex",
                    resource->length);
       return nullptr;
     }
-    resource->buffer->setName(kind == BufferKind::kIndex ? "LEGO guest index buffer"
+    next_buffer->setName(kind == BufferKind::kIndex ? "LEGO guest index buffer"
                                                           : "LEGO guest vertex buffer");
   }
 
-  void* mapped = resource->buffer->map();
+  void* mapped = next_buffer->map();
   if (!mapped) {
     return nullptr;
   }
-  const auto* source =
-      REX_KERNEL_MEMORY()->virtual_membase() + resource->mirror_address;
   const u32 element_size =
       kind == BufferKind::kIndex && resource->format == 1 ? 2 : 4;
-  ByteSwapElements(mapped, source, resource->length, element_size);
-  resource->buffer->unmap();
+  // UPLOAD heaps can be write-combined. Read/modify/write there (the packed
+  // vertex swizzle below) is extremely slow. Transform in cacheable CPU RAM,
+  // then perform a sequential write-only copy into the new GPU version.
+  std::vector<u8> converted;
+  void* conversion_target = mapped;
+  if (kind == BufferKind::kVertex && !byte_order.reversed_elements.empty()) {
+    converted.resize(resource->length);
+    conversion_target = converted.data();
+  }
+  ByteSwapElements(conversion_target, source, resource->length, element_size);
+  if (timing) g_buffer_timing.converted_bytes += resource->length;
+  if (kind == BufferKind::kVertex &&
+      !ApplyVertexByteOrder(static_cast<u8*>(conversion_target), resource->length, conversion_order)) {
+    next_buffer->unmap();
+    REXLOG_WARN("Native GPU: rejected invalid packed vertex layout stride={}",
+                byte_order.stride);
+    return nullptr;
+  }
+  if (!converted.empty()) std::memcpy(mapped, converted.data(), converted.size());
+  next_buffer->unmap();
+  if (resource->variants.size() >= 8) {
+    auto old = resource->variants.begin();
+    HostDevice::RetireResource(std::move(old->second));
+    resource->variants.erase(old);
+  }
+  resource->variants.emplace(byte_order_hash, next_buffer);
+  resource->buffer = std::move(next_buffer);
+  resource->content_hash = content_hash;
+  resource->byte_order_hash = byte_order_hash;
   return resource->buffer.get();
 }
 
+plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind,
+                                         const VertexByteOrder& byte_order) {
+  return ResolveBufferContents(AdoptBuffer(guest_address, kind), guest_address, kind, byte_order);
+}
+
+BufferResourceView InspectBufferResource(u32 guest_address, BufferKind kind) {
+  const auto resource = AdoptBuffer(guest_address, kind);
+  if (!resource || resource->kind != kind) return {};
+  return {nullptr, resource->length, resource->format, resource->mirror_address};
+}
+
+BufferResourceView ResolveBufferResourceWindow(u32 guest_address, BufferKind kind,
+    u32 offset, u32 length, const VertexByteOrder& byte_order) {
+  const auto parent = AdoptBuffer(guest_address, kind);
+  if (!parent || parent->kind != kind || !length || offset > parent->length ||
+      length > parent->length - offset) return {};
+  // Keep DWORD endian conversion aligned with the original whole buffer.
+  const u32 alignment = kind == BufferKind::kIndex && parent->format == 1 ? 2 : 4;
+  if (offset % alignment || length % alignment) return {};
+  const u64 key = (u64(offset) << 32) | length;
+  std::shared_ptr<BufferResource> resource;
+  {
+    std::lock_guard lock(parent->upload_mutex);
+    auto it = parent->windows.find(key);
+    if (it != parent->windows.end()) resource = it->second;
+    else {
+      resource = std::make_shared<BufferResource>();
+      resource->guest_address = guest_address;
+      resource->mirror_address = parent->mirror_address + offset;
+      resource->length = length;
+      resource->format = parent->format;
+      resource->kind = kind;
+      resource->owns_guest_memory = false;
+      // A shared mesh pool has far more than 64 draw ranges. Evicting begin()
+      // repeatedly discarded newly inserted ranges and allocated GPU buffers
+      // for them again on every frame. Bound bytes as well as entry count and
+      // retire the least recently used range instead.
+      const u64 budget = std::clamp<u64>(parent->length, 256 * 1024, 16 * 1024 * 1024);
+      while (!parent->windows.empty() && (parent->windows.size() >= 4096 ||
+             parent->window_cache_bytes + length > budget)) {
+        auto old = std::min_element(parent->windows.begin(), parent->windows.end(),
+            [](const auto& a, const auto& b) {
+              return a.second->last_window_use < b.second->last_window_use;
+            });
+        parent->window_cache_bytes -= old->second->length;
+        HostDevice::RetireResource(std::move(old->second));
+        parent->windows.erase(old);
+      }
+      parent->window_cache_bytes += length;
+      parent->windows.emplace(key, resource);
+    }
+    resource->last_window_use = ++parent->window_use_serial;
+  }
+  auto* buffer = ResolveBufferContents(resource, guest_address, kind, byte_order);
+  return {buffer, resource->length, resource->format, resource->mirror_address,
+          resource->content_hash, resource->byte_order_hash};
+}
+
 BufferResourceView ResolveBufferResourceView(u32 guest_address,
-                                             BufferKind kind) {
+                                             BufferKind kind,
+                                             const VertexByteOrder& byte_order) {
+  auto* buffer = ResolveBufferResource(guest_address, kind, byte_order);
   const auto resource = FindBuffer(guest_address);
   if (!resource || resource->kind != kind) {
     return {};
   }
-  auto* buffer = ResolveBufferResource(guest_address, kind);
-  return {buffer, resource->length, resource->format};
+  return {buffer, resource->length, resource->format, resource->mirror_address,
+          resource->content_hash, resource->byte_order_hash};
 }
 
 bool IsNativeBuffer(u32 guest_address) {
@@ -171,7 +377,10 @@ bool IsNativeBuffer(u32 guest_address) {
 
 u32 NativeBufferType(u32 guest_address) {
   const auto resource = FindBuffer(guest_address);
-  return resource ? static_cast<u32>(resource->kind) : 0;
+  // Placement headers keep the TU23 types 1/2. In particular XGOffsetResource
+  // uses type 1 to update +24, whereas type 6 would update +32 instead.
+  // Only our simplified, owned headers need a registry type override.
+  return resource && resource->owns_guest_memory ? static_cast<u32>(resource->kind) : 0;
 }
 
 u32 AddRefNativeBuffer(u32 guest_address) {
@@ -186,6 +395,7 @@ u32 AddRefNativeBuffer(u32 guest_address) {
 }
 
 u32 ReleaseNativeBuffer(u32 guest_address) {
+  auto recording = HostDevice::LockRecording();
   std::shared_ptr<BufferResource> released;
   {
     std::lock_guard lock(g_buffers_mutex);
@@ -205,8 +415,10 @@ u32 ReleaseNativeBuffer(u32 guest_address) {
   }
   auto* memory = REX_KERNEL_MEMORY();
   HostDevice::RetireResource(released);
-  memory->SystemHeapFree(released->mirror_address);
-  memory->SystemHeapFree(guest_address);
+  if (released->owns_guest_memory) {
+    memory->SystemHeapFree(released->mirror_address);
+    memory->SystemHeapFree(guest_address);
+  }
   return 0;
 }
 
@@ -216,7 +428,8 @@ void ResetBufferResources() {
     std::lock_guard lock(g_buffers_mutex);
     allocations.reserve(g_buffers.size());
     for (const auto& [guest_address, resource] : g_buffers) {
-      allocations.emplace_back(guest_address, resource->mirror_address);
+      if (resource->owns_guest_memory)
+        allocations.emplace_back(guest_address, resource->mirror_address);
     }
     g_buffers.clear();
   }
diff --git a/rexlego/src/gpu_native/buffers.h b/rexlego/src/gpu_native/buffers.h
index 8afc52c..cf5d2bb 100644
--- a/rexlego/src/gpu_native/buffers.h
+++ b/rexlego/src/gpu_native/buffers.h
@@ -2,6 +2,8 @@
 #pragma once
 
 #include <rex/types.h>
+#include "gpu_native/vertex_byte_order.h"
+#include "gpu_native/pool_copy.h"
 
 namespace plume {
 struct RenderBuffer;
@@ -14,28 +16,47 @@ enum class BufferKind : u32 {
   kIndex = 7,
 };
 
+struct BufferUploadTiming {
+  u64 calls = 0;
+  u64 hashed_bytes = 0;
+  u64 converted_bytes = 0;
+  double hash_ms = 0;
+};
+BufferUploadTiming ConsumeBufferUploadTiming();
+
 u32 CreateBufferResource(u32 length, u32 usage, u32 format, u32 pool,
                          BufferKind kind);
 u32 LockBufferResource(u32 guest_address, u32 offset, u32 size, u32 flags,
                        BufferKind kind);
 
-// Uploads the current guest mirror on every call. This deliberately favors
-// correctness over dirty-range guesses: CPU writes made without Lock/Unlock
-// and later CPU readback remain visible.
-plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind);
+// Checks current guest contents on every call, including placement buffers.
+// Changed contents get a new upload buffer retained through the frame fence,
+// so later CPU writes cannot overwrite geometry in earlier recorded draws.
+plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind,
+                                         const VertexByteOrder& byte_order = {});
 
 struct BufferResourceView {
   plume::RenderBuffer* buffer = nullptr;
   u32 length = 0;
   u32 guest_format = 0;
+  u32 mirror_address = 0;
+  u64 content_hash = 0;
+  u64 byte_order_hash = 0;
 };
 BufferResourceView ResolveBufferResourceView(u32 guest_address,
-                                             BufferKind kind);
+                                             BufferKind kind,
+                                             const VertexByteOrder& byte_order = {});
+// Bounded immutable uploads for the bytes fetched by a draw. Metadata reads
+// revalidate borrowed headers; window contents are still hashed every use.
+BufferResourceView InspectBufferResource(u32 guest_address, BufferKind kind);
+BufferResourceView ResolveBufferResourceWindow(u32 guest_address, BufferKind kind,
+    u32 offset, u32 length, const VertexByteOrder& byte_order = {});
 
 bool IsNativeBuffer(u32 guest_address);
 u32 NativeBufferType(u32 guest_address);
 u32 AddRefNativeBuffer(u32 guest_address);
 u32 ReleaseNativeBuffer(u32 guest_address);
 void ResetBufferResources();
+u32 InvalidatePoolCopyBuffers(PhysicalCopyRange destination);
 
 }  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/completion_queue.h b/rexlego/src/gpu_native/completion_queue.h
new file mode 100644
index 0000000..dc825a1
--- /dev/null
+++ b/rexlego/src/gpu_native/completion_queue.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <utility>
+
+namespace legodimensions::gpu_native {
+
+// One graphics queue has a monotonically ordered submission timeline. The
+// caller serializes access; popping never invokes guest code under its mutex.
+class CompletionQueue {
+ public:
+  void Push(std::uint64_t submission, std::function<void()> callback) {
+    entries_.push_back({submission, std::move(callback)});
+  }
+  std::function<void()> PopReady(std::uint64_t completed) {
+    if (entries_.empty() || entries_.front().submission > completed) return {};
+    auto callback = std::move(entries_.front().callback);
+    entries_.pop_front();
+    return callback;
+  }
+  std::size_t size() const { return entries_.size(); }
+
+ private:
+  struct Entry { std::uint64_t submission; std::function<void()> callback; };
+  std::deque<Entry> entries_;
+};
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h
index 5855523..d2dcf36 100644
--- a/rexlego/src/gpu_native/d3d.h
+++ b/rexlego/src/gpu_native/d3d.h
@@ -124,15 +124,16 @@ struct D3DDevice {
   be_u32 get_sampler_state[kSamplerStateCount];
   u8 unknown_0408[0x480 - 0x408];
   FetchConstant fetch_constants[kFetchConstantCount];
-  u8 unknown_06F0[0x3208 - 0x6F0];
+  u8 unknown_06F0[0x2FD0 - 0x6F0];
   be_u32 vertex_declaration;
+  u8 unknown_2FD4[0x3208 - 0x2FD4];
+  be_u32 fvf;
   be_u32 index_buffer;
   be_u32 render_targets[kNativeRenderTargets];
   be_u32 depth_stencil;
   be_u32 vertex_stream_buffers[18];
   u8 unknown_326C[0x3278 - 0x326C];
   be_u32 textures[kNativeTextureSlots];
-  u8 unknown_32B8[0x32E0 - 0x32B8];
   D3DViewport9 viewport;
   be_u32 viewport_reserved;
   D3DRect scissor;
@@ -149,7 +150,7 @@ static_assert(offsetof(D3DDevice, get_sampler_state) == 0x3B8);
 static_assert(offsetof(D3DDevice, fetch_constants) == 0x480);
 static_assert(offsetof(D3DDevice, viewport) == 0x32E0);
 static_assert(offsetof(D3DDevice, index_buffer) == 0x320C);
-static_assert(offsetof(D3DDevice, vertex_declaration) == 0x3208);
+static_assert(offsetof(D3DDevice, vertex_declaration) == 0x2FD0);
 static_assert(offsetof(D3DDevice, render_targets) == 0x3210);
 static_assert(offsetof(D3DDevice, depth_stencil) == 0x3220);
 static_assert(offsetof(D3DDevice, vertex_stream_buffers) == 0x3224);
@@ -158,4 +159,6 @@ static_assert(offsetof(D3DDevice, scissor) == 0x32FC);
 static_assert(offsetof(D3DDevice, pixel_shader) == 0x330C);
 static_assert(offsetof(D3DDevice, vertex_shader) == 0x3310);
 
+void SetNativeTexture(D3DDevice* device, u32 sampler, u32 texture, u64 dirty);
+
 }  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/depth_alias.h b/rexlego/src/gpu_native/depth_alias.h
new file mode 100644
index 0000000..7b37976
--- /dev/null
+++ b/rexlego/src/gpu_native/depth_alias.h
@@ -0,0 +1,23 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+// Xenos 32bpp color and depth use opposite 40-sample halves of each 80x16
+// EDRAM tile. This path covers equal-pitch, equal-base, single-sample views.
+constexpr uint32_t DepthColorAliasX(uint32_t x) {
+  return x % 80 < 40 ? x + 40 : x - 40;
+}
+constexpr float UnpackDepth24(uint32_t depth) {
+  // Same UNORM24 conversion as the SDK render-target cache (within half a
+  // 24-bit unit, monotonic, with exact 0/1 endpoints).
+  return float(depth + (depth >> 23)) * (1.0f / 16777216.0f);
+}
+struct DepthAliasLayout {
+  uint32_t physical_base, width, height, pitch, tiled, endian, levels, dimension;
+  bool operator==(const DepthAliasLayout&) const = default;
+};
+constexpr bool CompatibleDepthAlias(DepthAliasLayout a, DepthAliasLayout b) {
+  return a == b && a.physical_base && a.width && a.height &&
+      a.levels == 1 && a.dimension == 1;
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/depth_state.h b/rexlego/src/gpu_native/depth_state.h
new file mode 100644
index 0000000..1dd322f
--- /dev/null
+++ b/rexlego/src/gpu_native/depth_state.h
@@ -0,0 +1,29 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct DepthState {
+  bool enabled;
+  bool write;
+  uint32_t comparison;  // Xenos: NEVER=0 ... ALWAYS=7.
+};
+// TU23 SetDepthStencilSurface re-applies the requested enable bits on every
+// bind. A request made while no depth surface is bound must survive rebinding.
+constexpr uint32_t RebindDepthStencilControl(uint32_t control, uint32_t depth_request,
+                                             uint32_t stencil_request, bool bound) {
+  return (control & ~3u) | (bound ? ((depth_request & 1u) << 1) |
+                                          (stencil_request & 1u) : 0u);
+}
+constexpr DepthState DecodeDepthState(uint32_t control, bool has_depth) {
+  const bool enabled = has_depth && (control & 2u);
+  return {enabled, enabled && (control & 4u), enabled ? ((control >> 4) & 7u) : 7u};
+}
+static_assert(!DecodeDepthState(0, true).enabled);
+static_assert(!DecodeDepthState(0x76, false).enabled);
+static_assert(!DecodeDepthState(0x74, true).write);
+static_assert(DecodeDepthState(0x32, true).comparison == 3);
+static_assert(!DecodeDepthState(0x32, true).write);
+static_assert(DecodeDepthState(0x46, true).write);
+static_assert(DecodeDepthState(0x46, true).comparison == 4);
+static_assert(DecodeDepthState(0xFFFFFF89, true).comparison == 7);
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/descriptor_retirement.h b/rexlego/src/gpu_native/descriptor_retirement.h
new file mode 100644
index 0000000..c197ce7
--- /dev/null
+++ b/rexlego/src/gpu_native/descriptor_retirement.h
@@ -0,0 +1,31 @@
+#pragma once
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace legodimensions::gpu_native {
+// A descriptor may be referenced by several submitted lists and by the list
+// still being recorded. Each ticket must finish before its slot is reusable.
+class DescriptorRetirement {
+ public:
+  explicit DescriptorRetirement(std::size_t count = 0) : pending_(count, 0) {}
+  bool Retire(uint32_t index, uint32_t live_frames) {
+    if (pending_[index]) return false;
+    for (uint32_t frame = 0; frame < tickets_.size(); ++frame) {
+      if (!(live_frames & (1u << frame))) continue;
+      tickets_[frame].push_back(index);
+      ++pending_[index];
+    }
+    return pending_[index] == 0;
+  }
+  template<class Release> void CompleteFrame(uint32_t frame, Release release) {
+    for (uint32_t index : tickets_[frame])
+      if (--pending_[index] == 0) release(index);
+    tickets_[frame].clear();
+  }
+ private:
+  std::vector<uint8_t> pending_;
+  std::array<std::vector<uint32_t>, 3> tickets_;
+};
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 5e083c8..909b6fa 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -1,6 +1,9 @@
 #include "gpu_native/device.h"
+#include "gpu_native/descriptor_retirement.h"
+#include "gpu_native/completion_queue.h"
 
 #include "gpu_native/buffers.h"
+#include "gpu_native/queries.h"
 #include "gpu_native/draw.h"
 #include "gpu_native/shaders.h"
 #include "gpu_native/textures.h"
@@ -8,17 +11,30 @@
 #include "gpu_native/vertex_declarations.h"
 #include "gpu_native/shaders/copy_color_ps.h"
 #include "gpu_native/shaders/copy_vs.h"
+#include "gpu_native/shaders/resolve_color_ps.h"
+#include "gpu_native/shaders/depth_pack_ps.h"
+#include "gpu_native/shaders/depth_restore_ps.h"
 #include "native_gpu_build_info.h"
+#include "gpu_native/long_probe.h"
+#include "gpu_native/memory_watch.h"
+#include "gpu_native/renderdoc_probe.h"
+#include <cmath>
 
 #include <array>
 #include <chrono>
 #include <cstring>
+#include <filesystem>
+#include <fstream>
 #include <mutex>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include <plume_render_interface.h>
 #include <plume_render_interface_builders.h>
+#include <plume_d3d12.h>
+#include <d3d12sdklayers.h>
+#include <cstdlib>
 #include <rex/logging.h>
 #include <rex/ui/window.h>
 
@@ -29,6 +45,90 @@ std::unique_ptr<RenderInterface> CreateD3D12Interface();
 namespace legodimensions::gpu_native {
 namespace {
 
+struct GpuSnapshot {
+  std::unique_ptr<plume::RenderBuffer> readback;
+  D3D12_PLACED_SUBRESOURCE_FOOTPRINT footprint{};
+  u32 row_bytes = 0;
+  u32 dxgi_format = 0;
+  u32 capture_frame = 0;
+  std::filesystem::path path;
+};
+
+struct DrawUploadPage {
+  std::unique_ptr<plume::RenderBuffer> buffer;
+  u8* mapped = nullptr;
+  u64 size = 0;
+  ~DrawUploadPage() { if (mapped) buffer->unmap(); }
+};
+
+const std::filesystem::path& SnapshotDirectory() {
+  static const std::filesystem::path path = [] {
+    char* value = nullptr;
+    size_t length = 0;
+    _dupenv_s(&value, &length, "LEGO_GPU_SNAPSHOT_DIR");
+    std::filesystem::path result(value ? value : "");
+    std::free(value);
+    return result;
+  }();
+  return path;
+}
+
+void SaveSnapshots(std::vector<GpuSnapshot>& snapshots) {
+  for (auto& sample : snapshots) {
+    const auto* data = static_cast<const u8*>(sample.readback->map());
+    if (!data) continue;
+    const auto& fp = sample.footprint.Footprint;
+    u32 header[37]{};  // DDS + DX10 extension, one 2D base level.
+    header[0] = 0x20534444; header[1] = 124; header[2] = 0x100F;
+    header[3] = fp.Height; header[4] = fp.Width; header[5] = sample.row_bytes;
+    header[19] = 32; header[20] = 4; header[21] = 0x30315844;
+    header[27] = 0x1000; header[32] = sample.dxgi_format;
+    header[33] = 3; header[35] = 1;
+    std::ofstream file(sample.path, std::ios::binary);
+    file.write(reinterpret_cast<const char*>(header), sizeof(header));
+    for (u32 y = 0; y < fp.Height; ++y)
+      file.write(reinterpret_cast<const char*>(data + sample.footprint.Offset +
+                                               u64(y) * fp.RowPitch), sample.row_bytes);
+    if (LongProbeEnabled() && sample.path.filename().string().ends_with("-present.dds") &&
+        (sample.dxgi_format==28 || sample.dxgi_format==87)) {
+      double sum=0,edge=0; u64 black=0,white=0,count=0,edges=0;
+      const auto luminance=[&](u32 x,u32 y) {
+        const auto* p=data+sample.footprint.Offset+u64(y)*fp.RowPitch+4*x;
+        return (double(p[0])+p[1]+p[2])/(3*255.0);
+      };
+      for(u32 y=0;y<fp.Height;y+=4) for(u32 x=0;x<fp.Width;x+=4) {
+        const double l=luminance(x,y); sum+=l; ++count;
+        black+=l<0.005; white+=l>0.995;
+        if(x+4<fp.Width) { edge+=std::abs(l-luminance(x+4,y)); ++edges; }
+      }
+      const double black_fraction=double(black)/count, white_fraction=double(white)/count;
+      LongProbeEvent("present_pixels", black_fraction>0.995 || white_fraction>0.995,
+          "captured_frame=", sample.capture_frame, "file=", sample.path.filename().string(),
+          "mean=", sum/count, "edge=", edge/std::max<u64>(1,edges),
+          "black_fraction=", black_fraction, "white_fraction=", white_fraction);
+    }
+    if (LongProbeEnabled()) LongProbeEvent("snapshot_saved", false,
+        "captured_frame=", sample.capture_frame, "file=", sample.path.filename().string(),
+        "width=", fp.Width, "height=", fp.Height, "dxgi=", sample.dxgi_format);
+    sample.readback->unmap();
+    if (file) REXLOG_INFO("Native GPU: saved snapshot {}", sample.path.string());
+  }
+  snapshots.clear();
+}
+
+struct SamplerKeyHash {
+  size_t operator()(const NativeSamplerKey& key) const {
+    size_t value = 0;
+    for (u32 word : key.words) value = (value * 16777619u) ^ word;
+    return value;
+  }
+};
+
+struct CachedNativeSampler {
+  u32 descriptor = 0;
+  std::unique_ptr<plume::RenderSampler> sampler;
+};
+
 struct State {
   static constexpr u32 kFramesInFlight = 3;
 
@@ -41,6 +141,8 @@ struct State {
   std::unique_ptr<plume::RenderDescriptorSet> texture_descriptors;
   std::unique_ptr<plume::RenderDescriptorSet> sampler_descriptors;
   std::unique_ptr<plume::RenderSampler> default_sampler;
+  std::unordered_map<NativeSamplerKey, CachedNativeSampler, SamplerKeyHash> samplers;
+  u32 sampler_error_logs = 0;
   std::unique_ptr<plume::RenderBuffer> null_vertex_buffer;
   std::array<std::unique_ptr<plume::RenderTexture>, 3> null_textures;
   std::array<std::unique_ptr<plume::RenderTextureView>, 3> null_texture_views;
@@ -49,6 +151,10 @@ struct State {
   std::unique_ptr<plume::RenderShader> copy_vertex_shader;
   std::unique_ptr<plume::RenderShader> copy_pixel_shader;
   std::unique_ptr<plume::RenderPipeline> copy_pipeline;
+  std::unique_ptr<plume::RenderShader> resolve_pixel_shader;
+  std::array<std::unique_ptr<plume::RenderPipeline>, 4> resolve_pipelines;
+  std::array<std::unique_ptr<plume::RenderShader>, 2> depth_alias_shaders;
+  std::array<std::unique_ptr<plume::RenderPipeline>, 2> depth_alias_pipelines;
   std::vector<std::unique_ptr<plume::RenderFramebuffer>> framebuffers;
   std::array<std::unique_ptr<plume::RenderCommandList>, kFramesInFlight>
       command_lists;
@@ -59,17 +165,48 @@ struct State {
   std::vector<std::unique_ptr<plume::RenderCommandSemaphore>>
       render_semaphores;
   std::array<bool, kFramesInFlight> frame_submitted{};
+  std::array<u64, kFramesInFlight> slot_submission{};
+  u64 last_submission = 0, completed_submission = 0;
+  CompletionQueue completion_callbacks;
+  u64 callbacks_enqueued = 0, callbacks_executed = 0;
+  std::array<std::vector<std::unique_ptr<DrawUploadPage>>, kFramesInFlight> draw_uploads;
+  u32 draw_upload_page = 0;
+  u64 draw_upload_offset = 0;
   std::array<std::vector<std::shared_ptr<void>>, kFramesInFlight>
       retired_resources;
   u32 frame_slot = 0;
+  u32 present_number = 0;
+  u32 snapshot_number = 0;
+  u32 snapshot_draw_number = 0;
+  u32 snapshot_start_frame = ~0u;
+  std::chrono::steady_clock::time_point snapshot_start_time;
+  u32 snapshot_time_window = 0;
+  u32 snapshot_selected_frame = ~0u;
+  std::chrono::steady_clock::time_point snapshot_trigger_check;
+  std::array<std::vector<GpuSnapshot>, kFramesInFlight> snapshots;
   bool command_list_open = false;
   u32 long_wait_log_count = 0;
+  u32 timing_log_count = 0;
+  u64 sync_calls = 0, cpu_resource_wait_skips = 0;
+  double sync_ms = 0;
+  std::array<u64, static_cast<u32>(SyncReason::kCount)> sync_reason_calls{};
+  std::array<double, static_cast<u32>(SyncReason::kCount)> sync_reason_ms{};
+  std::chrono::steady_clock::time_point probe_next_snapshot{};
+  std::chrono::steady_clock::time_point probe_next_present{};
+  std::chrono::steady_clock::time_point probe_last_anomaly_snapshot{};
+  u64 probe_snapshot_bytes = 0;
+  bool probe_budget_reported = false;
+  std::chrono::steady_clock::time_point last_present_time{};
   std::vector<bool> texture_slots;
+  DescriptorRetirement retired_descriptors{65536};
   Backend backend = Backend::kD3D12;
 };
 
 constexpr u32 kBindlessTextureCount = 65536;
-constexpr u32 kBindlessSamplerCount = 2048;
+// Plume's D3D12 sampler heap is 1024 entries. Requesting 2048 makes its
+// allocator return INVALID_OFFSET; a subsequent Release-build setSampler then
+// writes an invalid CPU descriptor handle inside the display driver.
+constexpr u32 kBindlessSamplerCount = 1024;
 constexpr u32 kFirstTextureSlot = 3;
 
 std::mutex g_mutex;
@@ -151,6 +288,7 @@ bool CreatePipelineLayout(State& state) {
 }
 
 bool CreatePresentPipeline(State& state) {
+  REXLOG_INFO("Native GPU init: creating present shaders");
   state.copy_vertex_shader = state.device->createShader(
       g_copy_vs_dxil, sizeof(g_copy_vs_dxil), "main",
       plume::RenderShaderFormat::DXIL);
@@ -174,11 +312,15 @@ bool CreatePresentPipeline(State& state) {
   desc.renderTargetCount = 1;
   desc.renderTargetFormat[0] = plume::RenderFormat::B8G8R8A8_UNORM;
   desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy();
+  REXLOG_INFO("Native GPU init: creating present pipeline");
   state.copy_pipeline = state.device->createGraphicsPipeline(desc);
+  REXLOG_INFO("Native GPU init: present pipeline {}",
+              state.copy_pipeline ? "ready" : "failed");
   return state.copy_pipeline != nullptr;
 }
 
 bool CreateFrameRing(State& state) {
+  REXLOG_INFO("Native GPU init: creating frame ring");
   const u32 texture_count = state.swap_chain->getTextureCount();
   state.framebuffers.clear();
   state.render_semaphores.clear();
@@ -201,6 +343,8 @@ bool CreateFrameRing(State& state) {
     state.acquire_semaphores[i] = state.device->createCommandSemaphore();
     state.frame_submitted[i] = false;
     state.retired_resources[i].clear();
+    state.retired_descriptors.CompleteFrame(i,
+        [&](u32 index) { state.texture_slots[index] = false; });
     if (!state.command_lists[i] || !state.frame_fences[i] ||
         !state.acquire_semaphores[i]) {
       return false;
@@ -211,6 +355,7 @@ bool CreateFrameRing(State& state) {
 }
 
 bool CreateNullVertexBuffer(State& state) {
+  REXLOG_INFO("Native GPU init: creating null vertex buffer");
   state.null_vertex_buffer = state.device->createBuffer(
       plume::RenderBufferDesc::VertexBuffer(256,
                                              plume::RenderHeapType::UPLOAD));
@@ -227,6 +372,7 @@ bool CreateNullVertexBuffer(State& state) {
 }
 
 bool CreateNullTextures(State& state) {
+  REXLOG_INFO("Native GPU init: creating null textures");
   const plume::RenderTextureDimension dimensions[] = {
       plume::RenderTextureDimension::TEXTURE_2D,
       plume::RenderTextureDimension::TEXTURE_3D,
@@ -236,6 +382,7 @@ bool CreateNullTextures(State& state) {
       plume::RenderTextureViewDimension::TEXTURE_3D,
       plume::RenderTextureViewDimension::TEXTURE_CUBE};
   for (u32 i = 0; i < 3; ++i) {
+    REXLOG_INFO("Native GPU init: null texture {} resource", i);
     plume::RenderTextureDesc desc;
     desc.dimension = dimensions[i];
     desc.width = desc.height = desc.depth = desc.mipLevels = 1;
@@ -257,6 +404,7 @@ bool CreateNullTextures(State& state) {
     if (!state.null_texture_views[i]) {
       return false;
     }
+    REXLOG_INFO("Native GPU init: null texture {} descriptor", i);
     state.texture_descriptors->setTexture(
         i, state.null_textures[i].get(),
         plume::RenderTextureLayout::SHADER_READ,
@@ -319,8 +467,57 @@ void InitializeNullTextures(State& state, plume::RenderCommandList* commands) {
   state.null_textures_initialized = true;
 }
 
+void RefreshCompletedSubmissionsLocked(State& state) {
+  if (state.backend != Backend::kD3D12) return;
+  for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) {
+    if (!state.frame_submitted[slot]) continue;
+    auto* fence = static_cast<plume::D3D12CommandFence*>(state.frame_fences[slot].get());
+    const u64 completed = fence->d3d->GetCompletedValue();
+    // Plume signals fenceValue, then increments it. UINT64_MAX is removal,
+    // never successful completion. Do not consume its auto-reset wait event.
+    if (completed != UINT64_MAX && fence->fenceValue > 1 &&
+        completed >= fence->fenceValue - 1) {
+      state.completed_submission = std::max(state.completed_submission,
+                                           state.slot_submission[slot]);
+    }
+  }
+}
+
+void MarkSubmissionLocked(State& state, u32 slot) {
+  state.frame_submitted[slot] = true;
+  state.slot_submission[slot] = ++state.last_submission;
+}
+
+bool WaitForSubmittedFramesLocked(State& state) {
+  // DXGI's frame-latency object is a present throttle, not a GPU completion
+  // fence. Rebuilding/releasing the ring requires all submitted GPU users.
+  for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) {
+    if (!state.frame_submitted[slot]) continue;
+    if (state.backend == Backend::kD3D12) {
+      auto* fence = static_cast<plume::D3D12CommandFence*>(state.frame_fences[slot].get());
+      if (fence->d3d->GetCompletedValue() == UINT64_MAX) {
+        REXLOG_ERROR("Native GPU: cannot rebuild frame ring after device removal");
+        return false;
+      }
+    }
+    state.queue->waitForCommandFence(state.frame_fences[slot].get());
+    if (state.backend == Backend::kD3D12) {
+      auto* fence = static_cast<plume::D3D12CommandFence*>(state.frame_fences[slot].get());
+      const u64 completed = fence->d3d->GetCompletedValue();
+      if (completed == UINT64_MAX || completed < fence->fenceValue - 1) {
+        REXLOG_ERROR("Native GPU: frame-ring GPU wait failed");
+        return false;
+      }
+    }
+    state.completed_submission = std::max(state.completed_submission, state.slot_submission[slot]);
+    state.frame_submitted[slot] = false;
+  }
+  RefreshCompletedSubmissionsLocked(state);
+  return true;
+}
+
 bool RebuildSwapChain(State& state) {
-  state.swap_chain->wait();
+  if (state.command_list_open || !WaitForSubmittedFramesLocked(state)) return false;
   state.framebuffers.clear();
   state.render_semaphores.clear();
   state.frame_submitted.fill(false);
@@ -334,7 +531,8 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) {
   if (state.command_list_open) {
     return state.command_lists[state.frame_slot].get();
   }
-  if (state.swap_chain->needsResize() && !RebuildSwapChain(state)) {
+  if ((state.framebuffers.empty() || state.swap_chain->needsResize()) &&
+      !RebuildSwapChain(state)) {
     return nullptr;
   }
 
@@ -342,6 +540,9 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) {
   if (state.frame_submitted[slot]) {
     const auto wait_start = std::chrono::steady_clock::now();
     state.queue->waitForCommandFence(state.frame_fences[slot].get());
+    RefreshCompletedSubmissionsLocked(state);
+    if (state.backend != Backend::kD3D12)
+      state.completed_submission = std::max(state.completed_submission, state.slot_submission[slot]);
     const auto wait_ms = std::chrono::duration<double, std::milli>(
                              std::chrono::steady_clock::now() - wait_start)
                              .count();
@@ -350,8 +551,13 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) {
                   wait_ms);
     }
     state.frame_submitted[slot] = false;
+    SaveSnapshots(state.snapshots[slot]);
   }
   state.retired_resources[slot].clear();
+  state.draw_upload_page = 0;
+  state.draw_upload_offset = 0;
+  state.retired_descriptors.CompleteFrame(slot,
+      [&](u32 index) { state.texture_slots[index] = false; });
   auto* commands = state.command_lists[slot].get();
   commands->begin();
   InitializeNullTextures(state, commands);
@@ -361,7 +567,13 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) {
 
 }  // namespace
 
+std::unique_lock<std::recursive_mutex> HostDevice::LockRecording() {
+  static std::recursive_mutex recording_mutex;
+  return std::unique_lock(recording_mutex);
+}
+
 bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
+  auto recording = LockRecording();
   std::lock_guard lock(g_mutex);
   if (g_state) {
     return true;
@@ -372,7 +584,21 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
   }
 
   auto state = std::make_unique<State>();
+  REXLOG_INFO("Native GPU init: starting {}", NameOf(backend));
   state->backend = backend;
+  InitializeLogoCapture();
+  if (std::getenv("LEGO_NATIVE_GPU_DEBUG")) {
+    ID3D12Debug* debug = nullptr;
+    const HRESULT debug_result = D3D12GetDebugInterface(IID_PPV_ARGS(&debug));
+    if (SUCCEEDED(debug_result)) {
+      debug->EnableDebugLayer();
+      debug->Release();
+      REXLOG_INFO("Native GPU: D3D12 debug layer enabled");
+    } else {
+      REXLOG_WARN("Native GPU: D3D12 debug layer unavailable HRESULT=0x{:08X}",
+                  static_cast<u32>(debug_result));
+    }
+  }
   state->render_interface = CreateInterface(backend);
   if (!state->render_interface) {
     REXLOG_ERROR("Native GPU: failed to create {} interface", NameOf(backend));
@@ -384,10 +610,12 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
     REXLOG_ERROR("Native GPU: failed to create {} device", NameOf(backend));
     return false;
   }
+  REXLOG_INFO("Native GPU init: device ready");
   if (!CreatePipelineLayout(*state)) {
     REXLOG_ERROR("Native GPU: failed to create bindless pipeline layout");
     return false;
   }
+  REXLOG_INFO("Native GPU init: pipeline layout ready");
   state->queue = state->device->createCommandQueue(plume::RenderCommandListType::DIRECT);
   state->idle_fence = state->device->createCommandFence();
   if (!state->queue || !state->idle_fence) {
@@ -402,6 +630,7 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
     REXLOG_ERROR("Native GPU: failed to create or size the swap chain");
     return false;
   }
+  REXLOG_INFO("Native GPU init: swap chain ready");
   if (!CreatePresentPipeline(*state) || !CreateNullVertexBuffer(*state) ||
       !CreateNullTextures(*state) || !CreateFrameRing(*state)) {
     REXLOG_ERROR("Native GPU: failed to create present pipeline or frame ring");
@@ -414,24 +643,30 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) {
               state->swap_chain->getWidth(), state->swap_chain->getHeight(),
               state->swap_chain->getTextureCount());
   g_state = std::move(state);
+  REXLOG_INFO("Native pixel texture page cache: {} (vertex textures and buffers remain content-checked)",
+              CpuMemoryWatchEnabled());
   return true;
 }
 
 void HostDevice::Shutdown() {
+  auto recording = LockRecording();
   std::lock_guard lock(g_mutex);
   if (!g_state) {
     return;
   }
-  if (g_state->swap_chain) {
-    g_state->swap_chain->wait();
-  }
+  WaitForSubmittedFramesLocked(*g_state);
+  if (g_state->completion_callbacks.size())
+    REXLOG_INFO("Native GPU: cancelling {} completion callbacks at title shutdown",
+                g_state->completion_callbacks.size());
   // All GPU users must be idle before Plume resources are released.
   ResetDrawResources();
+  ResetQueryResources();
   ResetShaderResources();
   ResetBufferResources();
   ResetTextureResources();
   ResetDrawBindings();
   ResetVertexDeclarations();
+  ShutdownCpuMemoryWatch();
   g_state.reset();
 }
 
@@ -470,6 +705,35 @@ plume::RenderDescriptorSet* HostDevice::SamplerDescriptorSet() {
   return g_state ? g_state->sampler_descriptors.get() : nullptr;
 }
 
+u32 HostDevice::RegisterSampler(const TextureFetchWords& fetch) {
+  std::lock_guard lock(g_mutex);
+  if (!g_state || !g_state->sampler_descriptors) return 0;
+  auto& state = *g_state;
+  const auto desc = DecodeTextureSampler(fetch);
+  const auto key = TextureSamplerKey(desc);
+  if (const auto it = state.samplers.find(key); it != state.samplers.end()) return it->second.descriptor;
+  // Slot 0 remains the copy/resolve fallback. Never overwrite a descriptor
+  // referenced by an open or in-flight command list, and never evict samplers.
+  const u32 slot = u32(state.samplers.size()) + 1;
+  if (slot >= kBindlessSamplerCount) {
+    if (state.sampler_error_logs++ < 4) REXLOG_ERROR("Native GPU: sampler heap exhausted");
+    return 0;
+  }
+  auto sampler = state.device->createSampler(desc);
+  if (!sampler) {
+    if (state.sampler_error_logs++ < 4) REXLOG_ERROR("Native GPU: sampler creation failed");
+    return 0;
+  }
+  state.sampler_descriptors->setSampler(slot, sampler.get());
+  state.samplers.emplace(key, CachedNativeSampler{slot, std::move(sampler)});
+  if (slot <= 48) REXLOG_INFO(
+      "Native sampler {} uvw={},{},{} filter={},{},{} aniso={} bias={} lod={}..{} border={}",
+      slot, u32(desc.addressU), u32(desc.addressV), u32(desc.addressW), u32(desc.minFilter),
+      u32(desc.magFilter), u32(desc.mipmapMode), desc.anisotropyEnabled ? desc.maxAnisotropy : 0,
+      desc.mipLODBias, desc.minLOD, desc.maxLOD, u32(desc.borderColor));
+  return slot;
+}
+
 plume::RenderBuffer* HostDevice::NullVertexBuffer() {
   std::lock_guard lock(g_mutex);
   return g_state ? g_state->null_vertex_buffer.get() : nullptr;
@@ -504,10 +768,19 @@ void HostDevice::UnregisterTexture(u32 descriptor_index) {
       descriptor_index >= g_state->texture_slots.size()) {
     return;
   }
-  g_state->texture_slots[descriptor_index] = false;
+  if (!g_state->texture_slots[descriptor_index]) return;
+  u32 live_frames = 0;
+  for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) {
+    if (g_state->frame_submitted[slot] ||
+        (g_state->command_list_open && slot == g_state->frame_slot))
+      live_frames |= 1u << slot;
+  }
+  if (g_state->retired_descriptors.Retire(descriptor_index, live_frames))
+    g_state->texture_slots[descriptor_index] = false;
 }
 
 plume::RenderCommandList* HostDevice::BeginFrameCommands() {
+  auto recording = LockRecording();
   std::lock_guard lock(g_mutex);
   if (!g_state) {
     return nullptr;
@@ -515,7 +788,158 @@ plume::RenderCommandList* HostDevice::BeginFrameCommands() {
   return BeginFrameCommandsLocked(*g_state);
 }
 
+DrawUploadSlice HostDevice::AllocateDrawUpload(u32 size) {
+  auto recording = LockRecording();
+  std::lock_guard lock(g_mutex);
+  if (!g_state || !size || !BeginFrameCommandsLocked(*g_state)) return {};
+  auto& state = *g_state;
+  auto& pages = state.draw_uploads[state.frame_slot];
+  const u64 reserved = (u64(size) + 255) & ~u64{255};
+  while (true) {
+    if (state.draw_upload_page >= pages.size()) {
+      auto page = std::make_unique<DrawUploadPage>();
+      page->size = std::max(u64{4 * 1024 * 1024}, reserved);
+      page->buffer = state.device->createBuffer(plume::RenderBufferDesc::UploadBuffer(page->size));
+      if (!page->buffer) return {};
+      page->mapped = static_cast<u8*>(page->buffer->map());
+      if (!page->mapped) return {};
+      pages.push_back(std::move(page));
+    }
+    auto& page = *pages[state.draw_upload_page];
+    if (state.draw_upload_offset + reserved <= page.size) {
+      const u64 offset = state.draw_upload_offset;
+      state.draw_upload_offset += reserved;
+      return {page.buffer.get(), page.mapped + offset, offset};
+    }
+    ++state.draw_upload_page;
+    state.draw_upload_offset = 0;
+  }
+}
+
+bool HostDevice::Synchronize(SyncReason reason) {
+  const auto sync_start = std::chrono::steady_clock::now();
+  auto recording = LockRecording();
+  std::lock_guard lock(g_mutex);
+  if (!g_state) {
+    return false;
+  }
+
+  State& state = *g_state;
+  if (state.command_list_open) {
+    const u32 slot = state.frame_slot;
+    auto* commands = state.command_lists[slot].get();
+    commands->end();
+    state.command_list_open = false;
+    state.queue->executeCommandLists(commands, state.frame_fences[slot].get());
+    MarkSubmissionLocked(state, slot);
+  }
+
+  const auto wait_start = std::chrono::steady_clock::now();
+  bool waited = false;
+  for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) {
+    if (!state.frame_submitted[slot]) {
+      continue;
+    }
+    state.queue->waitForCommandFence(state.frame_fences[slot].get());
+    RefreshCompletedSubmissionsLocked(state);
+    if (state.backend != Backend::kD3D12)
+      state.completed_submission = std::max(state.completed_submission, state.slot_submission[slot]);
+    state.frame_submitted[slot] = false;
+    state.retired_resources[slot].clear();
+    state.retired_descriptors.CompleteFrame(slot,
+        [&](u32 index) { state.texture_slots[index] = false; });
+    SaveSnapshots(state.snapshots[slot]);
+    waited = true;
+  }
+  if (waited) {
+    const auto wait_ms = std::chrono::duration<double, std::milli>(
+                             std::chrono::steady_clock::now() - wait_start)
+                             .count();
+    if (wait_ms > 20.0 && state.long_wait_log_count++ < 20) {
+      REXLOG_WARN("Native GPU: completion callback GPU wait took {:.2f} ms",
+                  wait_ms);
+    }
+  }
+  if (NativeTextureTimingEnabled()) {
+    ++state.sync_calls;
+    const double elapsed = std::chrono::duration<double, std::milli>(
+        std::chrono::steady_clock::now() - sync_start).count();
+    state.sync_ms += elapsed;
+    const auto index = static_cast<u32>(reason);
+    if (index < state.sync_reason_calls.size()) {
+      ++state.sync_reason_calls[index];
+      state.sync_reason_ms[index] += elapsed;
+    }
+  }
+  return true;
+}
+
+void HostDevice::RecordCpuResourceWaitSkip() {
+  auto recording = LockRecording();
+  std::lock_guard lock(g_mutex);
+  if (g_state) ++g_state->cpu_resource_wait_skips;
+}
+
+bool HostDevice::EnqueueCompletionCallback(std::function<void()> callback) {
+  auto recording = LockRecording();
+  std::lock_guard lock(g_mutex);
+  if (!g_state || !callback) return false;
+  State& state = *g_state;
+  // An open list's fence will be assigned on submission. Otherwise all work
+  // preceding this insertion is in the latest already submitted list.
+  state.completion_callbacks.Push(state.last_submission + u64(state.command_list_open),
+                                  std::move(callback));
+  ++state.callbacks_enqueued;
+  return true;
+}
+
+void HostDevice::PollCompletionCallbacks() {
+  // A callback may enter another hook that polls. Keep the original FIFO
+  // order without holding a renderer mutex while executing guest code.
+  static thread_local bool executing = false;
+  if (executing) return;
+  static std::mutex execution_mutex;
+  std::unique_lock execution_lock(execution_mutex, std::try_to_lock);
+  if (!execution_lock.owns_lock()) return;
+  struct Guard { bool& flag; Guard(bool& f) : flag(f) { flag = true; }
+                 ~Guard() { flag = false; } } guard(executing);
+  for (;;) {
+    std::function<void()> callback;
+    {
+      auto recording = LockRecording();
+      std::lock_guard lock(g_mutex);
+      if (!g_state) return;
+      RefreshCompletedSubmissionsLocked(*g_state);
+      callback = g_state->completion_callbacks.PopReady(g_state->completed_submission);
+      if (!callback) return;
+      ++g_state->callbacks_executed;
+    }
+    callback();
+  }
+}
+
+bool HostDevice::SubmitRecordedWork() {
+  auto recording = LockRecording();
+  std::lock_guard lock(g_mutex);
+  if (!g_state) {
+    return false;
+  }
+  State& state = *g_state;
+  if (!state.command_list_open) {
+    return true;
+  }
+  const u32 slot = state.frame_slot;
+  auto* commands = state.command_lists[slot].get();
+  commands->end();
+  state.command_list_open = false;
+  state.queue->executeCommandLists(commands, state.frame_fences[slot].get());
+  MarkSubmissionLocked(state, slot);
+  state.frame_slot = (slot + 1) % State::kFramesInFlight;
+  return true;
+}
+
 void HostDevice::RetireResource(std::shared_ptr<void> resource) {
+  auto recording = LockRecording();
   if (!resource) {
     return;
   }
@@ -523,15 +947,181 @@ void HostDevice::RetireResource(std::shared_ptr<void> resource) {
   if (!g_state) {
     return;
   }
-  g_state->retired_resources[g_state->frame_slot].emplace_back(
-      std::move(resource));
+  // The current slot may be unused after SubmitRecordedWork advanced it.
+  // Retiring only there lets BeginFrameCommands destroy resources before an
+  // older submitted list has finished. Keep a reference at EVERY live fence.
+  for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) {
+    if (g_state->frame_submitted[slot] ||
+        (g_state->command_list_open && slot == g_state->frame_slot))
+      g_state->retired_resources[slot].push_back(resource);
+  }
+}
+
+bool HostDevice::ResolveHdrColor(plume::RenderTexture* source, u32 descriptor_index,
+                                 plume::RenderTexture* destination,
+                                 u32 width, u32 height, float scale,
+                                 ColorResolveDestination destination_format, bool source_is_target,
+                                 const ResolveRegion* region) {
+  auto recording = LockRecording();
+  // Acquire the list before g_mutex: BeginFrameCommands owns that mutex too.
+  auto* commands = BeginFrameCommands();
+  if (!commands || !source || !destination || source == destination) return false;
+  const ResolveRegion area = region ? *region : ResolveRegion{0, 0, width, height, 0, 0};
+  const i32 dx = i32(area.left) - i32(area.x), dy = i32(area.top) - i32(area.y);
+  if (dx < -32768 || dx > 32767 || dy < -32768 || dy > 32767) return false;
+  std::shared_ptr<plume::RenderFramebuffer> framebuffer;
+  {
+    std::lock_guard lock(g_mutex);
+    if (!g_state || descriptor_index >= g_state->texture_slots.size() ||
+        !g_state->texture_slots[descriptor_index]) return false;
+    auto& state = *g_state;
+    const size_t format_index = static_cast<size_t>(destination_format);
+    if (format_index >= state.resolve_pipelines.size()) return false;
+    auto& pipeline = state.resolve_pipelines[format_index];
+    if (!state.resolve_pixel_shader) {
+      state.resolve_pixel_shader = state.device->createShader(
+          g_resolve_color_ps_dxil, sizeof(g_resolve_color_ps_dxil), "main",
+          plume::RenderShaderFormat::DXIL);
+      if (!state.resolve_pixel_shader) return false;
+    }
+    if (!pipeline) {
+      plume::RenderGraphicsPipelineDesc desc;
+      desc.pipelineLayout = state.pipeline_layout.get();
+      desc.vertexShader = state.copy_vertex_shader.get();
+      desc.pixelShader = state.resolve_pixel_shader.get();
+      desc.depthFunction = plume::RenderComparisonFunction::ALWAYS;
+      desc.depthEnabled = false;
+      desc.depthWriteEnabled = false;
+      desc.primitiveTopology = plume::RenderPrimitiveTopology::TRIANGLE_LIST;
+      desc.cullMode = plume::RenderCullMode::NONE;
+      desc.fillMode = plume::RenderFillMode::SOLID;
+      desc.renderTargetCount = 1;
+      constexpr plume::RenderFormat formats[] = {
+          plume::RenderFormat::R16G16B16A16_UNORM,
+          plume::RenderFormat::R32G32B32A32_FLOAT,
+          plume::RenderFormat::R16G16_UNORM,
+          plume::RenderFormat::R32_FLOAT};
+      desc.renderTargetFormat[0] = formats[format_index];
+      desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy();
+      pipeline = state.device->createGraphicsPipeline(desc);
+      if (!pipeline || !static_cast<plume::D3D12GraphicsPipeline*>(pipeline.get())->d3d)
+        return false;
+    }
+    plume::RenderFramebufferDesc desc;
+    const plume::RenderTexture* attachments[] = {destination};
+    desc.colorAttachments = attachments;
+    desc.colorAttachmentsCount = 1;
+    framebuffer = state.device->createFramebuffer(desc);
+    if (!framebuffer) return false;
+    const plume::RenderTextureBarrier barriers[] = {
+      {source, plume::RenderTextureLayout::SHADER_READ},
+      {destination, plume::RenderTextureLayout::COLOR_WRITE}};
+    commands->barriers(plume::RenderBarrierStage::GRAPHICS, barriers, 2);
+    commands->setFramebuffer(framebuffer.get());
+    commands->setViewports(plume::RenderViewport(float(area.x), float(area.y),
+        float(area.width), float(area.height)));
+    commands->setScissors(plume::RenderRect(area.x, area.y,
+        area.x + area.width, area.y + area.height));
+    commands->setGraphicsPipelineLayout(state.pipeline_layout.get());
+    for (u32 set = 0; set < 3; ++set)
+      commands->setGraphicsDescriptorSet(state.texture_descriptors.get(), set);
+    commands->setGraphicsDescriptorSet(state.sampler_descriptors.get(), 3);
+    commands->setPipeline(pipeline.get());
+    const struct { u32 index, index2; float scale, unused; } constants{
+        descriptor_index, (u32(dx) & 0xFFFFu) | ((u32(dy) & 0xFFFFu) << 16), scale, 0};
+    commands->setGraphicsPushConstants(0, &constants);
+    commands->drawInstanced(3, 1, 0, 0);
+    commands->setFramebuffer(nullptr);
+    const plume::RenderTextureBarrier after[] = {
+      {source, source_is_target ? (destination_format == ColorResolveDestination::kDepthFloat32
+                                     ? plume::RenderTextureLayout::DEPTH_WRITE
+                                     : plume::RenderTextureLayout::COLOR_WRITE)
+                               : plume::RenderTextureLayout::SHADER_READ},
+      {destination, plume::RenderTextureLayout::SHADER_READ}};
+    commands->barriers(plume::RenderBarrierStage::GRAPHICS, after, 2);
+  }
+  RetireResource(std::move(framebuffer));
+  return true;
+}
+
+bool HostDevice::TransferDepthAlias(plume::RenderTexture* source, u32 descriptor_index,
+                                   plume::RenderTexture* destination,
+                                   u32 width, u32 height, bool restore) {
+  auto recording = LockRecording();
+  auto* commands = BeginFrameCommands();
+  if (!commands || !source || !destination || source == destination || !width || !height)
+    return false;
+  std::shared_ptr<plume::RenderFramebuffer> framebuffer;
+  {
+    std::lock_guard lock(g_mutex);
+    if (!g_state || descriptor_index >= g_state->texture_slots.size() ||
+        !g_state->texture_slots[descriptor_index]) return false;
+    auto& state = *g_state;
+    auto& shader = state.depth_alias_shaders[restore];
+    auto& pipeline = state.depth_alias_pipelines[restore];
+    if (!shader) shader = state.device->createShader(
+        restore ? g_depth_restore_ps_dxil : g_depth_pack_ps_dxil,
+        restore ? sizeof(g_depth_restore_ps_dxil) : sizeof(g_depth_pack_ps_dxil),
+        "main", plume::RenderShaderFormat::DXIL);
+    if (!shader) return false;
+    if (!pipeline) {
+      plume::RenderGraphicsPipelineDesc desc;
+      desc.pipelineLayout = state.pipeline_layout.get();
+      desc.vertexShader = state.copy_vertex_shader.get();
+      desc.pixelShader = shader.get();
+      desc.depthFunction = plume::RenderComparisonFunction::ALWAYS;
+      desc.depthEnabled = restore;
+      desc.depthWriteEnabled = restore;
+      desc.depthTargetFormat = restore ? plume::RenderFormat::D32_FLOAT_S8_UINT
+                                      : plume::RenderFormat::UNKNOWN;
+      desc.primitiveTopology = plume::RenderPrimitiveTopology::TRIANGLE_LIST;
+      desc.cullMode = plume::RenderCullMode::NONE;
+      desc.fillMode = plume::RenderFillMode::SOLID;
+      desc.renderTargetCount = restore ? 0 : 1;
+      desc.renderTargetFormat[0] = plume::RenderFormat::R8G8B8A8_UNORM;
+      desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy();
+      pipeline = state.device->createGraphicsPipeline(desc);
+      if (!pipeline || !static_cast<plume::D3D12GraphicsPipeline*>(pipeline.get())->d3d)
+        return false;
+    }
+    plume::RenderFramebufferDesc desc;
+    const plume::RenderTexture* attachments[] = {destination};
+    if (restore) desc.depthAttachment = destination;
+    else { desc.colorAttachments = attachments; desc.colorAttachmentsCount = 1; }
+    framebuffer = state.device->createFramebuffer(desc);
+    if (!framebuffer) return false;
+    const plume::RenderTextureBarrier barriers[] = {
+        {source, plume::RenderTextureLayout::SHADER_READ},
+        {destination, restore ? plume::RenderTextureLayout::DEPTH_WRITE
+                              : plume::RenderTextureLayout::COLOR_WRITE}};
+    commands->barriers(plume::RenderBarrierStage::GRAPHICS, barriers, 2);
+    commands->setFramebuffer(framebuffer.get());
+    commands->setViewports(plume::RenderViewport(0, 0, float(width), float(height)));
+    commands->setScissors(plume::RenderRect(0, 0, width, height));
+    commands->setGraphicsPipelineLayout(state.pipeline_layout.get());
+    for (u32 set = 0; set < 3; ++set)
+      commands->setGraphicsDescriptorSet(state.texture_descriptors.get(), set);
+    commands->setGraphicsDescriptorSet(state.sampler_descriptors.get(), 3);
+    commands->setPipeline(pipeline.get());
+    const struct { u32 index, index2; float unused1, unused2; } constants{descriptor_index, 0, 0, 0};
+    commands->setGraphicsPushConstants(0, &constants);
+    commands->drawInstanced(3, 1, 0, 0);
+    commands->setFramebuffer(nullptr);
+    if (!restore) commands->barriers(plume::RenderBarrierStage::GRAPHICS,
+        plume::RenderTextureBarrier(destination, plume::RenderTextureLayout::SHADER_READ));
+  }
+  RetireResource(std::move(framebuffer));
+  return true;
 }
 
 bool HostDevice::PresentTexture(plume::RenderTexture* texture,
                                 u32 descriptor_index) {
+  auto recording = LockRecording();
+  const auto present_start = std::chrono::steady_clock::now();
   if (!texture || descriptor_index == ~u32{0}) {
     return false;
   }
+  SnapshotTexture(texture, "present", false);
 
   std::lock_guard lock(g_mutex);
   if (!g_state || !g_state->copy_pipeline ||
@@ -540,18 +1130,19 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
     return false;
   }
   State& state = *g_state;
-  if (!state.command_list_open && state.swap_chain->needsResize()) {
+  if (!state.command_list_open &&
+      (state.framebuffers.empty() || state.swap_chain->needsResize())) {
     if (!RebuildSwapChain(state)) {
       REXLOG_WARN("Native GPU: swap chain resize deferred");
       return false;
     }
   }
 
-  const u32 slot = state.frame_slot;
   plume::RenderCommandList* commands = BeginFrameCommandsLocked(state);
   if (!commands) {
     return false;
   }
+  const u32 slot = state.frame_slot;
 
   u32 image_index = 0;
   if (!state.swap_chain->acquireTexture(
@@ -562,7 +1153,7 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
     commands->end();
     state.command_list_open = false;
     state.queue->executeCommandLists(commands, state.frame_fences[slot].get());
-    state.frame_submitted[slot] = true;
+    MarkSubmissionLocked(state, slot);
     state.frame_slot = (slot + 1) % State::kFramesInFlight;
     return false;
   }
@@ -610,15 +1201,257 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
       state.render_semaphores[image_index].get()};
   state.queue->executeCommandLists(lists, 1, waits, 1, signals, 1,
                                    state.frame_fences[slot].get());
-  state.frame_submitted[slot] = true;
+  MarkSubmissionLocked(state, slot);
   const bool presented = state.swap_chain->present(image_index, signals, 1);
+  ReportLogoCapture();
+  ++state.present_number;
+  g_probe_frame.store(state.present_number);
+  if (NativeTextureTimingEnabled()) {
+    const auto now = std::chrono::steady_clock::now();
+    const auto uploads = ConsumeTextureUploadTiming();
+    const auto draws = ConsumeDrawTiming();
+    const auto buffers = ConsumeBufferUploadTiming();
+    if (state.last_present_time.time_since_epoch().count() != 0) {
+      const double interval_ms = std::chrono::duration<double, std::milli>(
+          now - state.last_present_time).count();
+      static std::ofstream frame_metrics = [] {
+        const auto* path = std::getenv("LEGO_NATIVE_FRAME_METRICS");
+        std::ofstream stream;
+        if (path && *path) {
+          stream.open(path);
+          stream << "frame,interval_ms,texture_ms,draw_calls,constants_ms,vertices_ms,buffer_hash_ms,texture_hash_bytes,buffer_hash_bytes,present_cpu_ms,sync_calls,sync_ms,cpu_resource_wait_skips";
+          for (const char* name : {"other", "idle", "fence", "resource", "callback", "query_begin", "query_release"})
+            stream << ",sync_" << name << "_calls,sync_" << name << "_ms";
+          stream << ",callbacks_enqueued,callbacks_executed,callbacks_pending"
+                 << ",bindings_ms,begin_ms,pipeline_ms,issue_ms,tail_ms,buffer_converted_bytes\n";
+        }
+        return stream;
+      }();
+      if (frame_metrics.is_open()) {
+        frame_metrics << state.present_number << ',' << interval_ms << ',' << uploads.cpu_ms << ','
+            << draws.calls << ',' << draws.stages_ms[3] << ',' << draws.stages_ms[4] << ','
+            << buffers.hash_ms << ',' << uploads.hashed_bytes << ',' << buffers.hashed_bytes << ','
+            << std::chrono::duration<double, std::milli>(now - present_start).count() << ','
+            << state.sync_calls << ',' << state.sync_ms << ',' << state.cpu_resource_wait_skips;
+        for (u32 i = 0; i < state.sync_reason_calls.size(); ++i)
+          frame_metrics << ',' << state.sync_reason_calls[i] << ',' << state.sync_reason_ms[i];
+        frame_metrics << ',' << state.callbacks_enqueued << ',' << state.callbacks_executed
+                      << ',' << state.completion_callbacks.size()
+                      << ',' << draws.stages_ms[0] << ',' << draws.stages_ms[1]
+                      << ',' << draws.stages_ms[2] << ',' << draws.stages_ms[5]
+                      << ',' << draws.stages_ms[6] << ',' << buffers.converted_bytes << '\n';
+        if (state.present_number % 120 == 0) frame_metrics.flush();
+      }
+      if (LongProbeEnabled()) LongProbeEvent("frame", interval_ms > 2000.0,
+          "interval_ms=", interval_ms, "texture_cpu_ms=", uploads.cpu_ms,
+          "texture_calls=", uploads.calls, "source_hits=", uploads.source_hits,
+          "converted_bytes=", uploads.converted_bytes, "hashed_bytes=", uploads.hashed_bytes,
+          "draw_calls=", draws.calls, "bindings_ms=", draws.stages_ms[0],
+          "pipeline_ms=", draws.stages_ms[2], "constants_ms=", draws.stages_ms[3],
+          "vertices_ms=", draws.stages_ms[4], "issue_ms=", draws.stages_ms[5],
+          "buffer_hash_ms=", buffers.hash_ms, "buffer_hashed_bytes=", buffers.hashed_bytes,
+          "presented=", presented);
+      if (interval_ms > 40.0 && (state.timing_log_count++ < 16 ||
+                                state.timing_log_count % 30 == 0)) {
+        REXLOG_INFO("Native timing: frame={} interval={:.2f}ms texture_cpu={:.2f}ms "
+            "calls={} source_hits={} converted={} bytes source={:.2f}ms hash={:.2f}ms hashed={}",
+            state.present_number, interval_ms, uploads.cpu_ms, uploads.calls,
+            uploads.source_hits, uploads.converted_bytes, uploads.source_ms, uploads.hash_ms,
+            uploads.hashed_bytes);
+        REXLOG_INFO("Native draw timing: calls={} bindings={:.2f} begin={:.2f} "
+            "pipeline={:.2f} constants={:.2f} vertices={:.2f} issue={:.2f} tail={:.2f}ms",
+            draws.calls, draws.stages_ms[0], draws.stages_ms[1], draws.stages_ms[2],
+            draws.stages_ms[3], draws.stages_ms[4], draws.stages_ms[5], draws.stages_ms[6]);
+        REXLOG_INFO("Native buffer timing: calls={} hash={:.2f}ms hashed={} converted={}",
+            buffers.calls, buffers.hash_ms, buffers.hashed_bytes, buffers.converted_bytes);
+        REXLOG_INFO("Native synchronization timing: calls={} cpu_resource_skips={} time={:.2f}ms",
+                    state.sync_calls, state.cpu_resource_wait_skips, state.sync_ms);
+        REXLOG_INFO("Native synchronization reasons ms: other={:.2f} idle={:.2f} fence={:.2f} resource={:.2f} callback={:.2f} query_begin={:.2f} query_release={:.2f}",
+            state.sync_reason_ms[0], state.sync_reason_ms[1], state.sync_reason_ms[2],
+            state.sync_reason_ms[3], state.sync_reason_ms[4], state.sync_reason_ms[5], state.sync_reason_ms[6]);
+        REXLOG_INFO("Native callback queue: enqueued={} executed={} pending={}",
+            state.callbacks_enqueued, state.callbacks_executed, state.completion_callbacks.size());
+      }
+    }
+    state.last_present_time = now;
+    state.sync_calls = 0;
+    state.cpu_resource_wait_skips = 0;
+    state.sync_ms = 0;
+    state.sync_reason_calls.fill(0);
+    state.sync_reason_ms.fill(0);
+    state.callbacks_enqueued = state.callbacks_executed = 0;
+  }
+  state.snapshot_number = 0;
+  state.snapshot_draw_number = 0;
   state.frame_slot = (slot + 1) % State::kFramesInFlight;
   if (!presented) {
+    LongProbeEvent("present_failed", true, "frame=", state.present_number);
     REXLOG_ERROR("Native GPU: swap-chain present failed");
   }
+  if (std::getenv("LEGO_NATIVE_GPU_DEBUG")) {
+    auto* native = static_cast<plume::D3D12Device*>(state.device.get());
+    ID3D12InfoQueue* info = nullptr;
+    if (SUCCEEDED(native->d3d->QueryInterface(IID_PPV_ARGS(&info)))) {
+      const auto count = info->GetNumStoredMessagesAllowedByRetrievalFilter();
+      for (UINT64 i = 0; i < count; ++i) {
+        SIZE_T size = 0;
+        info->GetMessage(i, nullptr, &size);
+        std::vector<u8> storage(size);
+        auto* message = reinterpret_cast<D3D12_MESSAGE*>(storage.data());
+        if (SUCCEEDED(info->GetMessage(i, message, &size)) &&
+            message->Severity <= D3D12_MESSAGE_SEVERITY_WARNING) {
+          REXLOG_WARN("Native D3D12 [{}]: {}", static_cast<u32>(message->ID), message->pDescription);
+        }
+      }
+      info->ClearStoredMessages();
+      info->Release();
+    }
+  }
   return presented;
 }
 
+void HostDevice::BeginTiledPassSnapshot() {
+  if (!std::getenv("LEGO_GPU_SNAPSHOT_START_AT_TILING")) return;
+  auto recording = LockRecording();
+  std::lock_guard lock(g_mutex);
+  if (g_state && g_state->snapshot_start_frame == ~0u) {
+    g_state->snapshot_start_frame = g_state->present_number;
+    g_state->snapshot_start_time = std::chrono::steady_clock::now();
+  }
+}
+
+bool HostDevice::LongProbeSnapshotActive() {
+  if (!LongProbeEnabled()) return false;
+  std::lock_guard lock(g_mutex);
+  return g_state && g_state->present_number == g_state->snapshot_selected_frame;
+}
+
+void HostDevice::SnapshotTexture(plume::RenderTexture* texture,
+                                 std::string_view label, bool render_target) {
+  if (!texture || SnapshotDirectory().empty()) return;
+  auto recording = LockRecording();
+  auto* commands = BeginFrameCommands();
+  if (!commands) return;
+  std::lock_guard lock(g_mutex);
+  if (!g_state) return;
+  auto& state = *g_state;
+  if (state.snapshot_start_frame == ~0u &&
+      std::getenv("LEGO_GPU_SNAPSHOT_START_AT_TILING")) return;
+  auto* native_texture = static_cast<plume::D3D12Texture*>(texture)->d3d;
+  const auto desc = native_texture->GetDesc();
+  // Startup can present hundreds of empty loading frames before any draws.
+  // Anchor this cutscene diagnostic to its first actual HDR resolve instead.
+  if (state.snapshot_start_frame == ~0u && render_target &&
+      desc.Format == DXGI_FORMAT_R16G16B16A16_FLOAT) {
+    state.snapshot_start_frame = state.present_number;
+    state.snapshot_start_time = std::chrono::steady_clock::now();
+  }
+  if (state.snapshot_start_frame == ~0u) return;
+  const u32 frame = state.present_number - state.snapshot_start_frame;
+  // Opt-in diagnostic runs can capture later menu/hub frames without restart.
+  // Check at most twice a second; consume the trigger once for a whole frame.
+  const auto now = std::chrono::steady_clock::now();
+  if (now - state.snapshot_trigger_check >= std::chrono::milliseconds(500)) {
+    state.snapshot_trigger_check = now;
+    std::error_code trigger_error;
+    if (std::filesystem::remove(SnapshotDirectory() / "capture-next-frame", trigger_error))
+      // The trigger can be noticed in the middle of a pass. Capture the next
+      // frame from its first snapshot, not just the remainder of this one.
+      state.snapshot_selected_frame = state.present_number + 1;
+  }
+  static const bool time_windows = [] {
+    char* value = nullptr;
+    size_t length = 0;
+    _dupenv_s(&value, &length, "LEGO_GPU_SNAPSHOT_TIME_WINDOWS");
+    const bool enabled = value && length > 1;
+    std::free(value);
+    return enabled;
+  }();
+  if (LongProbeEnabled()) {
+    const bool requested = g_probe_capture_requested.exchange(false);
+    const bool anomaly_due = requested && now-state.probe_last_anomaly_snapshot >= std::chrono::seconds(10);
+    if (now >= state.probe_next_snapshot || anomaly_due) {
+      state.snapshot_selected_frame = state.present_number + 1;
+      state.probe_next_snapshot = now + std::chrono::seconds(30);
+      if (anomaly_due) state.probe_last_anomaly_snapshot = now;
+      LongProbeEvent("snapshot_scheduled", false, "target_frame=", state.snapshot_selected_frame,
+          "reason=", anomaly_due ? "anomaly_candidate" : "periodic");
+    }
+    const bool present_sample = label=="present" && now>=state.probe_next_present;
+    if (present_sample) state.probe_next_present=now+std::chrono::seconds(5);
+    if (state.present_number != state.snapshot_selected_frame && !present_sample) return;
+  } else if (time_windows) {
+    // Capture across scene changes even when fast stars and slow geometry
+    // make fixed frame indices cluster in the first seconds of the cutscene.
+    constexpr double windows[] = {0, 8, 16, 24, 32, 48};
+    const double elapsed = std::chrono::duration<double>(
+        std::chrono::steady_clock::now() - state.snapshot_start_time).count();
+    if (state.snapshot_time_window < std::size(windows) &&
+        elapsed >= windows[state.snapshot_time_window]) {
+      state.snapshot_selected_frame = state.present_number;
+      do { ++state.snapshot_time_window; }
+      while (state.snapshot_time_window < std::size(windows) &&
+             elapsed >= windows[state.snapshot_time_window]);
+    }
+    if (state.present_number != state.snapshot_selected_frame) return;
+  } else if (state.present_number != state.snapshot_selected_frame &&
+             frame != 0 && frame != 2 && frame != 8 && frame != 32 &&
+             frame != 64 && frame != 128 && frame != 256) return;
+  // A busy character pass can issue hundreds of draws. Reserve captures for
+  // its final resolve and postprocessing rather than exhausting them on meshes.
+  if (label.starts_with("draw-") && state.snapshot_draw_number++ >= 16) return;
+  if (state.snapshot_number >= 128) return;
+  if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D ||
+      desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1) return;
+  // Only color formats inspected by this startup diagnostic. Never read the
+  // combined depth/stencil planes as a packed color texture.
+  const u32 format = u32(desc.Format);
+  if (format != 2 && format != 28 && format != 87 && format != 10 && format != 11 && format != 13 &&
+      format != 34 && format != 35 && format != 41) return;
+  std::error_code error;
+  std::filesystem::create_directories(SnapshotDirectory(), error);
+  if (error) return;
+  GpuSnapshot sample;
+  UINT64 total_bytes = 0, row_bytes = 0;
+  UINT rows = 0;
+  auto* native_device = static_cast<plume::D3D12Device*>(state.device.get())->d3d;
+  native_device->GetCopyableFootprints(&desc, 0, 1, 0, &sample.footprint,
+                                       &rows, &row_bytes, &total_bytes);
+  if (rows != desc.Height || total_bytes > 64 * 1024 * 1024) return;
+  if (LongProbeEnabled() && state.probe_snapshot_bytes + total_bytes > 12ull*1024*1024*1024) {
+    if (!state.probe_budget_reported) {
+      state.probe_budget_reported = true;
+      LongProbeEvent("snapshot_budget_reached", false, "limit_bytes=", 12ull*1024*1024*1024,
+          "event_and_timing_logging_continues=1");
+    }
+    return;
+  }
+  sample.readback = state.device->createBuffer(plume::RenderBufferDesc::ReadbackBuffer(total_bytes));
+  if (!sample.readback) return;
+  state.probe_snapshot_bytes += total_bytes;
+  sample.row_bytes = u32(row_bytes);
+  sample.dxgi_format = format;
+  sample.capture_frame = state.present_number;
+  sample.path = SnapshotDirectory() / ("f" + std::to_string(frame) + "-" +
+      std::to_string(state.snapshot_number++) + "-" + std::string(label) + ".dds");
+  commands->setFramebuffer(nullptr);
+  commands->barriers(plume::RenderBarrierStage::COPY,
+      plume::RenderTextureBarrier(texture, plume::RenderTextureLayout::COPY_SOURCE));
+  D3D12_TEXTURE_COPY_LOCATION source{};
+  source.pResource = native_texture;
+  source.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+  D3D12_TEXTURE_COPY_LOCATION target{};
+  target.pResource = static_cast<plume::D3D12Buffer*>(sample.readback.get())->d3d;
+  target.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
+  target.PlacedFootprint = sample.footprint;
+  static_cast<plume::D3D12CommandList*>(commands)->d3d->CopyTextureRegion(
+      &target, 0, 0, 0, &source, nullptr);
+  commands->barriers(plume::RenderBarrierStage::GRAPHICS,
+      plume::RenderTextureBarrier(texture, render_target
+          ? plume::RenderTextureLayout::COLOR_WRITE : plume::RenderTextureLayout::SHADER_READ));
+  state.snapshots[state.frame_slot].push_back(std::move(sample));
+}
+
 std::string_view HostDevice::BackendName() {
   std::lock_guard lock(g_mutex);
   return g_state ? NameOf(g_state->backend) : "none";
diff --git a/rexlego/src/gpu_native/device.h b/rexlego/src/gpu_native/device.h
index b3a79f6..192553b 100644
--- a/rexlego/src/gpu_native/device.h
+++ b/rexlego/src/gpu_native/device.h
@@ -2,9 +2,13 @@
 #pragma once
 
 #include <memory>
+#include <functional>
+#include <mutex>
 #include <string_view>
 
 #include <rex/types.h>
+#include "gpu_native/resolve_region.h"
+#include "gpu_native/sampler_state.h"
 
 namespace plume {
 struct RenderBuffer;
@@ -31,8 +35,25 @@ enum class Backend {
   kVulkan,
 };
 
+// Attribution only: all reasons retain the same completion semantics.
+enum class SyncReason : u32 {
+  kOther, kIdle, kFence, kResource, kCallback, kQueryBegin, kQueryRelease, kCount
+};
+
+enum class ColorResolveDestination { kUnormRGBA16, kFloatRGBA32, kUnormRG16, kDepthFloat32 };
+
+struct DrawUploadSlice {
+  plume::RenderBuffer* buffer = nullptr;
+  void* mapped = nullptr;
+  u64 offset = 0;
+  explicit operator bool() const { return buffer && mapped; }
+};
+
 class HostDevice {
  public:
+  // Hold across the whole CPU recording operation, not only list acquisition.
+  // Lock order: recording -> resource/cache locks -> device state.
+  static std::unique_lock<std::recursive_mutex> LockRecording();
   static bool Create(rex::ui::Window* window, Backend backend = Backend::kD3D12);
   static void Shutdown();
   static bool IsReady();
@@ -43,6 +64,7 @@ class HostDevice {
   static plume::RenderPipelineLayout* PipelineLayout();
   static plume::RenderDescriptorSet* TextureDescriptorSet();
   static plume::RenderDescriptorSet* SamplerDescriptorSet();
+  static u32 RegisterSampler(const TextureFetchWords& fetch);
   static plume::RenderBuffer* NullVertexBuffer();
   static u32 RegisterTexture(plume::RenderTexture* texture,
                              plume::RenderTextureView* view);
@@ -50,6 +72,21 @@ class HostDevice {
   // Returns the current frame's direct command list, opening a new ring slot
   // and waiting only when that slot is being reused.
   static plume::RenderCommandList* BeginFrameCommands();
+  // Append-only upload storage owned by the current command-list fence slot.
+  // It is reset only after that slot's GPU work has completed.
+  static DrawUploadSlice AllocateDrawUpload(u32 size);
+  // Submit all work recorded so far and wait for every in-flight frame. This
+  // is reserved for GPU data dependencies and explicit idle points;
+  // the normal present path remains asynchronous through the frame ring.
+  static bool Synchronize(SyncReason reason = SyncReason::kOther);
+  // Completion is conservative at the end of the containing submission.
+  // Enqueue never submits or waits; poll executes guest code outside g_mutex.
+  static bool EnqueueCompletionCallback(std::function<void()> callback);
+  static void PollCompletionCallbacks();
+  static void RecordCpuResourceWaitSkip();
+  // Submit the currently recorded list without waiting. Used for the guest's
+  // Xenos ring kick-off boundary.
+  static bool SubmitRecordedWork();
   // Keeps a released resource alive through the fence of the command list
   // which may still reference it.
   static void RetireResource(std::shared_ptr<void> resource);
@@ -58,6 +95,27 @@ class HostDevice {
   // in a frame ring so the CPU does not wait for the frame it just submitted.
   static bool PresentTexture(plume::RenderTexture* texture,
                              u32 descriptor_index);
+  // Exact texel color scale into UNORM16 storage or a float32 sampling mirror.
+  // Width/height select the top-left logical extent of a padded source.
+  // Optional validated region supports atlas placement. Depth writes normalized
+  // X into an R32_FLOAT mirror; stencil, MSAA and CPU readback are unsupported.
+  static bool ResolveHdrColor(plume::RenderTexture* source, u32 descriptor_index,
+                              plume::RenderTexture* destination,
+                              u32 width, u32 height, float scale,
+                              ColorResolveDestination destination_format =
+                                  ColorResolveDestination::kUnormRGBA16,
+                              bool source_is_target = true,
+                              const ResolveRegion* region = nullptr);
+  // Single-sample raw D24S8 <-> RGBA8 representations. Pack reads the R32
+  // depth sampling mirror; restore writes depth only and retains stencil.
+  static bool TransferDepthAlias(plume::RenderTexture* source, u32 descriptor_index,
+                                 plume::RenderTexture* destination,
+                                 u32 width, u32 height, bool restore);
+  // Opt-in intermediate GPU image readback; disabled outside diagnostic runs.
+  static void SnapshotTexture(plume::RenderTexture* texture,
+                               std::string_view label, bool render_target);
+  static void BeginTiledPassSnapshot();
+  static bool LongProbeSnapshotActive();
   static std::string_view BackendName();
 
  private:
diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
index 1c3de5f..cca92d2 100644
--- a/rexlego/src/gpu_native/draw.cpp
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -1,34 +1,58 @@
 #include <algorithm>
 #include <array>
+#include <atomic>
+#include <chrono>
 #include <cstring>
+#include <cstdlib>
 #include <memory>
 #include <mutex>
+#include <set>
 #include <unordered_map>
+#include <unordered_set>
+#include <utility>
 #include <vector>
 
 #include <plume_render_interface.h>
+#include <plume_d3d12.h>
 #include <rex/graphics/xenos.h>
-#include <rex/hook.h>
+#include "gpu_native/renderer_route.h"
 #include <rex/logging.h>
+#include <rex/cvar.h>
 #include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
 #include <rex/types.h>
 
 #include "gpu_native/buffers.h"
+#include "gpu_native/buffer_window.h"
+#include "gpu_native/alpha_test.h"
+#include "gpu_native/blend_state.h"
+#include "gpu_native/queries.h"
 #include "gpu_native/d3d.h"
+#include "gpu_native/draw.h"
+#include "gpu_native/depth_state.h"
+#include "gpu_native/stencil_state.h"
 #include "gpu_native/device.h"
 #include "gpu_native/shaders.h"
 #include "gpu_native/state.h"
 #include "gpu_native/textures.h"
+#include "gpu_native/tile_extent.h"
+#include "gpu_native/tonemap_dof.h"
+#include "gpu_native/long_probe.h"
+#include "gpu_native/renderdoc_probe.h"
 #include "gpu_native/vertex_declarations.h"
 
+REXCVAR_DEFINE_BOOL(gpu_native_buffer_windows, false, "GPU",
+    "Upload and retain only the vertex buffer ranges fetched by native draws")
+    .lifecycle(rex::cvar::Lifecycle::kRequiresRestart);
+
 namespace legodimensions::gpu_native {
 namespace {
 
 struct SharedConstants {
-  u32 texture_2d[16]{};
-  u32 texture_3d[16]{};
-  u32 texture_cube[16]{};
-  u32 samplers[16]{};
+  u32 texture_2d[32]{};
+  u32 texture_3d[32]{};
+  u32 texture_cube[32]{};
+  u32 samplers[32]{};
   u32 booleans[8]{};
   u32 swapped_texcoords = 0;
   float half_pixel_x = 0.0f;
@@ -42,18 +66,37 @@ struct SharedConstants {
   u32 sint_texcoords = 0;
   float shadow_pcf_scale = 1.0f;
   u32 padding = 0;
-  float blit_half_pixel_x = 0.0f;
-  float blit_half_pixel_y = 0.0f;
-  u32 padding_2[2]{};
+  float viewport_scale_x = 0.0f;
+  float viewport_scale_y = 0.0f;
+  u32 alpha_function = 7;
+  u32 viewport_mode = 0;
+  float color_output_scale[4]{1, 1, 1, 1};
 };
-static_assert(sizeof(SharedConstants) == 352);
+static_assert(sizeof(SharedConstants) == 624);
+static_assert(offsetof(SharedConstants, booleans) == 512);
+static_assert(offsetof(SharedConstants, color_output_scale) == 608);
+static_assert(offsetof(SharedConstants, alpha_function) == 600);
+static_assert(offsetof(SharedConstants, viewport_scale_x) == 592);
+static_assert(offsetof(SharedConstants, viewport_mode) == 604);
+static_assert(offsetof(SharedConstants, alpha_threshold) == 556);
 
 struct PipelineKey {
   u32 vertex_shader = 0;
   u32 pixel_shader = 0;
   u32 declaration = 0;
-  u32 render_target_format = 0;
+  // Guest allocation addresses may be reused after Release or XGRegister.
+  // Pipeline identity must follow the actual shader and input-layout contents.
+  u64 vertex_shader_hash = 0;
+  u64 pixel_shader_hash = 0;
+  u64 declaration_hash = 0;
+  std::array<u32, kNativeRenderTargets> render_target_formats{};
   u32 depth_format = 0;
+  u32 depth_control = 0;
+  u32 clip_disable = 0;
+  u32 stencil_masks = 0;
+  u32 stencil_face = 0;
+  u32 color_write_mask = 0;
+  std::array<u32, kNativeRenderTargets> blend_controls{0x10001, 0x10001, 0x10001, 0x10001};
   u32 topology = 0;
   u32 spec_constants = 0;
   std::array<u32, kNativeVertexStreams> strides{};
@@ -63,10 +106,15 @@ struct PipelineKey {
 struct PipelineKeyHash {
   size_t operator()(const PipelineKey& key) const {
     size_t hash = 1469598103934665603ull;
-    const auto* bytes = reinterpret_cast<const u8*>(&key);
-    for (size_t i = 0; i < sizeof(key); ++i) {
-      hash = (hash ^ bytes[i]) * 1099511628211ull;
-    }
+    const auto mix = [&](u64 value) { hash = (hash ^ value) * 1099511628211ull; };
+    mix(key.vertex_shader); mix(key.pixel_shader); mix(key.declaration);
+    mix(key.vertex_shader_hash); mix(key.pixel_shader_hash); mix(key.declaration_hash);
+    for (const auto format : key.render_target_formats) mix(format);
+    mix(key.depth_format); mix(key.depth_control); mix(key.color_write_mask); mix(key.clip_disable);
+    mix(key.stencil_masks); mix(key.stencil_face);
+    for (const auto blend : key.blend_controls) mix(blend);
+    mix(key.topology); mix(key.spec_constants);
+    for (const auto stride : key.strides) mix(stride);
     return hash;
   }
 };
@@ -76,6 +124,43 @@ std::unordered_map<PipelineKey, std::unique_ptr<plume::RenderPipeline>,
                    PipelineKeyHash>
     g_pipelines;
 u32 g_unsupported_draw_logs = 0;
+DrawTiming g_draw_timing;
+bool g_seen_tiled_pass = false; // Serialized by LockRecording; diagnostics only.
+
+struct DrawTimer {
+  bool enabled = NativeTextureTimingEnabled();
+  size_t stage = 0;
+  std::chrono::steady_clock::time_point last;
+  std::array<double, 7> elapsed{};
+  DrawTimer() {
+    if (enabled) {
+      ++g_draw_timing.calls;
+      last = std::chrono::steady_clock::now();
+    }
+  }
+  void Next() {
+    if (!enabled) return;
+    const auto now = std::chrono::steady_clock::now();
+    elapsed[stage] += std::chrono::duration<double, std::milli>(now - last).count();
+    last = now;
+    ++stage;
+  }
+  ~DrawTimer() {
+    if (!enabled) return;
+    Next();
+    double total = 0;
+    for (size_t i = 0; i < elapsed.size(); ++i) {
+      g_draw_timing.stages_ms[i] += elapsed[i];
+      total += elapsed[i];
+    }
+    static u32 slow_draws = 0;
+    if (total > 250.0 && slow_draws++ < 24)
+      REXLOG_INFO("Native slow draw: total={:.2f}ms bindings={:.2f} begin={:.2f} "
+          "pipeline={:.2f} constants={:.2f} vertices={:.2f} issue={:.2f} tail={:.2f}",
+          total, elapsed[0], elapsed[1], elapsed[2], elapsed[3], elapsed[4],
+          elapsed[5], elapsed[6]);
+  }
+};
 
 bool MapTopology(u32 primitive_type, plume::RenderPrimitiveTopology& result) {
   using rex::graphics::xenos::PrimitiveType;
@@ -90,6 +175,7 @@ bool MapTopology(u32 primitive_type, plume::RenderPrimitiveTopology& result) {
       result = plume::RenderPrimitiveTopology::LINE_STRIP;
       return true;
     case PrimitiveType::kTriangleList:
+    case PrimitiveType::kQuadList:
       result = plume::RenderPrimitiveTopology::TRIANGLE_LIST;
       return true;
     case PrimitiveType::kTriangleStrip:
@@ -108,35 +194,63 @@ void CopyBigEndianDwords(void* destination, const void* source, u32 size) {
   }
 }
 
-std::shared_ptr<plume::RenderBuffer> BindConstants(
+bool BindConstants(
     plume::RenderCommandList* commands, const D3DDevice* device,
-    const DrawBindings& bindings, const VertexDeclarationView& declaration) {
+    const DrawBindings& bindings, const VertexDeclarationView& declaration, bool capture_constants) {
   constexpr u32 kConstantsSize = 0x1000;
   constexpr u32 kSharedOffset = kConstantsSize * 2;
-  auto* host = HostDevice::Device();
-  if (!host) {
-    return {};
-  }
-  auto upload = std::shared_ptr<plume::RenderBuffer>(
-      host->createBuffer(plume::RenderBufferDesc::UploadBuffer(
-                             kSharedOffset + sizeof(SharedConstants)))
-          .release());
-  if (!upload) {
-    return {};
-  }
-  auto* mapped = static_cast<u8*>(upload->map());
-  if (!mapped) {
-    return {};
-  }
+  DrawUploadSlice upload;
+  std::shared_ptr<plume::RenderBuffer> separate_upload;
+  static const bool separate_constants = std::getenv("LEGO_NATIVE_NO_DRAW_ARENA") != nullptr;
+  if (separate_constants) {
+    auto* host = HostDevice::Device();
+    if (!host) return false;
+    separate_upload = host->createBuffer(plume::RenderBufferDesc::UploadBuffer(
+        kSharedOffset + sizeof(SharedConstants)));
+    if (!separate_upload) return false;
+    upload = {separate_upload.get(), separate_upload->map(), 0};
+  } else upload = HostDevice::AllocateDrawUpload(kSharedOffset + sizeof(SharedConstants));
+  if (!upload) return false;
+  // Build in cacheable CPU memory. Persistently mapped UPLOAD storage is
+  // write-combined and must never be read for diagnostics or partial updates.
+  alignas(16) std::array<u8, kSharedOffset + sizeof(SharedConstants)> data;
+  auto* mapped = data.data();
   const auto* bytes = reinterpret_cast<const u8*>(device);
-  CopyBigEndianDwords(mapped, bytes + 0x700, kConstantsSize);
-  CopyBigEndianDwords(mapped + kConstantsSize, bytes + 0x1700,
+  CopyBigEndianDwords(mapped, bytes + 0x780, kConstantsSize);
+  CopyBigEndianDwords(mapped + kConstantsSize, bytes + 0x1780,
                       kConstantsSize);
 
+  // Filtering the CoC pass alone leaves the tonemap's sharp/mip blend active.
+  // Change only this draw's immutable host constants, never guest state.
+  if (BoundShaderHash(ShaderStage::kPixel) == 0x3A47E5DDE66B42C6ull &&
+      ApplyDisabledTonemapDof(BoundShaderHash(ShaderStage::kPixel),
+          rex::cvar::GetFlagByName("depth_of_field") == "false",
+          mapped + kConstantsSize, kConstantsSize)) {
+    static bool logged_disabled_mix = false;
+    if (!logged_disabled_mix) {
+      logged_disabled_mix = true;
+      REXLOG_INFO("Native GPU: disabled DoF sharp/mip tonemap mix");
+    }
+  }
+
   SharedConstants shared;
+  // Isolated ABI624 candidate; requires the matching viewport shader banks.
+  static const bool viewport_candidate = std::getenv("LEGO_NATIVE_VIEWPORT") != nullptr;
+  if (viewport_candidate && u32(*reinterpret_cast<const be_u32*>(bytes + 10572)) == 0x400) {
+    const auto target = ResolveTextureResource(bindings.render_targets[0]);
+    if (!target.width || !target.height) return false;
+    shared.viewport_mode = 1;
+    shared.viewport_scale_x = 2.0f / target.width;
+    shared.viewport_scale_y = -2.0f / target.height;
+  }
+  const auto alpha = DecodeNativeAlphaState(
+      *reinterpret_cast<const be_u32*>(bytes + 10556),
+      *reinterpret_cast<const be_f32*>(bytes + 10620));
+  shared.alpha_function = alpha.function;
+  shared.alpha_threshold = alpha.reference;
   std::fill(std::begin(shared.texture_3d), std::end(shared.texture_3d), 1u);
   std::fill(std::begin(shared.texture_cube), std::end(shared.texture_cube), 2u);
-  const auto* bools = reinterpret_cast<const be_u32*>(bytes + 0x2700);
+  const auto* bools = reinterpret_cast<const be_u32*>(bytes + 0x2780);
   for (u32 i = 0; i < 8; ++i) {
     shared.booleans[i] = bools[i];
   }
@@ -147,12 +261,73 @@ std::shared_ptr<plume::RenderBuffer> BindConstants(
   shared.swapped_blend_weights = declaration.swapped_blend_weights;
   shared.swapped_positions = declaration.swapped_positions;
   shared.sint_texcoords = declaration.sint_texcoords;
+  for (u32 i = 0; i < kNativeRenderTargets; ++i)
+    shared.color_output_scale[i] = SurfaceColorOutputScale(bindings.render_targets[i]);
+  static const bool force_all_textures = std::getenv("LEGO_NATIVE_ALL_TEXTURES") != nullptr;
+  const u32 texture_mask = force_all_textures ? ~u32{0} :
+      BoundShaderTextureMask(ShaderStage::kVertex) | BoundShaderTextureMask(ShaderStage::kPixel);
+  static const bool trace_constants = std::getenv("LEGO_GPU_SNAPSHOT_DIR") != nullptr;
+  if (trace_constants && BoundShaderHash(ShaderStage::kPixel) == 0x3A47E5DDE66B42C6ull) {
+    static u32 tonemap_logs = 0;
+    if (tonemap_logs++ < 12) {
+      const auto* vs = reinterpret_cast<const float*>(mapped);
+      const auto* ps = reinterpret_cast<const float*>(mapped + kConstantsSize);
+      for (u32 index : {4u, 10u, 11u, 12u})
+        REXLOG_INFO("Native tonemap constants c{} VS={},{},{},{} PS={},{},{},{}", index,
+            vs[index*4], vs[index*4+1], vs[index*4+2], vs[index*4+3],
+            ps[index*4], ps[index*4+1], ps[index*4+2], ps[index*4+3]);
+    }
+  }
   for (u32 i = 0; i < kNativeTextureSlots; ++i) {
+    if (!(texture_mask & (1u << i))) continue;
     const auto texture = ResolveTextureResource(bindings.textures[i]);
     if (!texture.texture || texture.descriptor_index == ~u32{0}) {
+      if (LongProbeEnabled() && bindings.textures[i] &&
+          LongProbeOnce(0xBAD1000000000000ull | bindings.textures[i]))
+        LongProbeEvent("unavailable_bound_texture", true, "slot=", i,
+            "guest=", bindings.textures[i]);
       continue;
     }
-    UploadTextureResource(bindings.textures[i], commands);
+    // Vertex textures contain animation data. Until every writer participates
+    // in invalidation, verify their contents at every use, even with watches.
+    UploadTextureResource(bindings.textures[i], commands,
+        (BoundShaderTextureMask(ShaderStage::kVertex) & (1u << i)) != 0);
+    TextureFetchWords fetch;
+    for (u32 word = 0; word < fetch.size(); ++word) fetch[word] = device->fetch_constants[i].dword[word];
+    if (LongProbeEnabled()) {
+      static std::unordered_map<u64, TextureFetchWords> seen;
+      const u64 identity=(u64(bindings.textures[i])<<8)|i;
+      const auto found=seen.find(identity);
+      if (found==seen.end() || found->second!=fetch) {
+        seen[identity]=fetch;
+        LongProbeEvent("texture_fetch_changed", false, "slot=", i,
+            "guest=", bindings.textures[i], "descriptor=", texture.descriptor_index,
+            "VS=", BoundShaderHash(ShaderStage::kVertex),
+            "PS=", BoundShaderHash(ShaderStage::kPixel), "fetch=", LongProbeHex(fetch));
+      }
+    }
+    shared.samplers[i] = HostDevice::RegisterSampler(fetch);
+    if (trace_constants && i >= 16) {
+      struct VertexTextureTrace {
+        u32 guest = 0;
+        TextureFetchWords fetch{};
+        bool valid = false;
+      };
+      // Trace changes rather than the first 32 draws of the same character.
+      // These are the active fetch words, which can differ from its header.
+      static std::array<VertexTextureTrace, kNativeTextureSlots> previous{};
+      static u32 vertex_texture_logs = 0;
+      auto& seen = previous[i];
+      if (!seen.valid || seen.guest != bindings.textures[i] || seen.fetch != fetch) {
+        seen = {bindings.textures[i], fetch, true};
+        if (vertex_texture_logs++ < 64) REXLOG_INFO(
+            "Native vertex texture slot={} guest={:08X} descriptor={} sampler={} VS={:016X} "
+            "fetch={:08X},{:08X},{:08X},{:08X},{:08X},{:08X}",
+            i, bindings.textures[i], texture.descriptor_index, shared.samplers[i],
+            BoundShaderHash(ShaderStage::kVertex),
+            fetch[0], fetch[1], fetch[2], fetch[3], fetch[4], fetch[5]);
+      }
+    }
     if (texture.d3d_type ==
         static_cast<u32>(D3DResourceType::kVolumeTexture)) {
       shared.texture_3d[i] = texture.descriptor_index;
@@ -164,17 +339,27 @@ std::shared_ptr<plume::RenderBuffer> BindConstants(
     }
   }
   std::memcpy(mapped + kSharedOffset, &shared, sizeof(shared));
-  upload->unmap();
-  commands->setGraphicsRootDescriptor(upload->at(0), 0);
-  commands->setGraphicsRootDescriptor(upload->at(kConstantsSize), 1);
-  commands->setGraphicsRootDescriptor(upload->at(kSharedOffset), 2);
-  return upload;
+  if (capture_constants && HostDevice::LongProbeSnapshotActive()) {
+    const auto name=SaveLongProbeConstants({mapped,kSharedOffset+sizeof(shared)});
+    if (!name.empty()) LongProbeEvent("draw_constants", false, "file=", name,
+        "VS=", BoundShaderHash(ShaderStage::kVertex),
+        "PS=", BoundShaderHash(ShaderStage::kPixel), "texture_mask=", texture_mask,
+        "VS_offset=0 PS_offset=4096 shared_offset=8192 shared_bytes=624");
+  }
+  std::memcpy(upload.mapped, data.data(), data.size());
+  commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset), 0);
+  commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset + kConstantsSize), 1);
+  commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset + kSharedOffset), 2);
+  if (separate_upload) {
+    separate_upload->unmap();
+    HostDevice::RetireResource(std::move(separate_upload));
+  }
+  return true;
 }
 
 plume::RenderPipeline* GetPipeline(
     const PipelineKey& key, const VertexDeclarationView& declaration,
-    plume::RenderPrimitiveTopology topology, plume::RenderFormat color_format,
-    plume::RenderFormat depth_format) {
+    plume::RenderPrimitiveTopology topology, plume::RenderFormat depth_format) {
   std::lock_guard lock(g_pipeline_mutex);
   if (const auto it = g_pipelines.find(key); it != g_pipelines.end()) {
     return it->second.get();
@@ -192,51 +377,252 @@ plume::RenderPipeline* GetPipeline(
   desc.pipelineLayout = HostDevice::PipelineLayout();
   desc.vertexShader = vs;
   desc.pixelShader = ps;
-  desc.depthFunction = plume::RenderComparisonFunction::LESS_EQUAL;
-  desc.depthEnabled = depth_format != plume::RenderFormat::UNKNOWN;
-  desc.depthWriteEnabled = desc.depthEnabled;
-  desc.depthClipEnabled = true;
+  const auto depth_state = DecodeDepthState(
+      key.depth_control, depth_format != plume::RenderFormat::UNKNOWN);
+  constexpr plume::RenderComparisonFunction comparisons[] = {
+      plume::RenderComparisonFunction::NEVER, plume::RenderComparisonFunction::LESS,
+      plume::RenderComparisonFunction::EQUAL, plume::RenderComparisonFunction::LESS_EQUAL,
+      plume::RenderComparisonFunction::GREATER, plume::RenderComparisonFunction::NOT_EQUAL,
+      plume::RenderComparisonFunction::GREATER_EQUAL, plume::RenderComparisonFunction::ALWAYS};
+  desc.depthFunction = comparisons[depth_state.comparison];
+  desc.depthEnabled = depth_state.enabled;
+  desc.depthWriteEnabled = depth_state.write;
+  desc.depthClipEnabled = !key.clip_disable;
+  const auto stencil = DecodeStencilState(key.depth_control, key.stencil_masks,
+      depth_format == plume::RenderFormat::D32_FLOAT_S8_UINT,
+      topology == plume::RenderPrimitiveTopology::TRIANGLE_LIST ||
+      topology == plume::RenderPrimitiveTopology::TRIANGLE_STRIP);
+  desc.stencilEnabled = stencil.enabled;
+  if (stencil.enabled) {
+    constexpr plume::RenderStencilOp ops[] = {
+        plume::RenderStencilOp::KEEP, plume::RenderStencilOp::ZERO,
+        plume::RenderStencilOp::REPLACE, plume::RenderStencilOp::INCREMENT_AND_CLAMP,
+        plume::RenderStencilOp::DECREMENT_AND_CLAMP, plume::RenderStencilOp::INVERT,
+        plume::RenderStencilOp::INCREMENT_AND_WRAP, plume::RenderStencilOp::DECREMENT_AND_WRAP};
+    const auto face = [&](const StencilFaceState& value) {
+      plume::RenderStencilFaceDesc result;
+      result.compareFunction = comparisons[value.comparison];
+      result.failOp = ops[value.fail];
+      result.passOp = ops[value.pass];
+      result.depthFailOp = ops[value.depth_fail];
+      return result;
+    };
+    desc.stencilFrontFace = face(stencil.front);
+    desc.stencilBackFace = face(stencil.back);
+    desc.stencilReadMask = stencil.read_mask;
+    desc.stencilWriteMask = stencil.write_mask;
+    desc.frontFace = key.stencil_face ? plume::RenderFrontFace::CLOCKWISE
+                                     : plume::RenderFrontFace::COUNTER_CLOCKWISE;
+  }
   desc.primitiveTopology = topology;
   desc.cullMode = plume::RenderCullMode::NONE;
   desc.fillMode = plume::RenderFillMode::SOLID;
-  desc.renderTargetCount = color_format == plume::RenderFormat::UNKNOWN ? 0 : 1;
-  desc.renderTargetFormat[0] = color_format;
-  desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy();
+  desc.renderTargetCount = 0;
+  for (u32 i = 0; i < kNativeRenderTargets; ++i) {
+    const auto color_format = static_cast<plume::RenderFormat>(key.render_target_formats[i]);
+    if (color_format == plume::RenderFormat::UNKNOWN) continue;
+    desc.renderTargetCount = i + 1;
+    desc.renderTargetFormat[i] = color_format;
+    desc.renderTargetBlend[i] = plume::RenderBlendDesc::Copy();
+    desc.renderTargetBlend[i].renderTargetWriteMask = (key.color_write_mask >> (i * 4)) & 15u;
+    const auto blend = DecodeColorBlend(key.blend_controls[i]);
+    if (!blend.supported) {
+      static std::atomic<u32> unsupported_blends{0};
+      if (unsupported_blends.fetch_add(1) < 16)
+        REXLOG_WARN("Native GPU: unsupported blend state {:08X}", key.blend_controls[i]);
+      return nullptr;
+    }
+    using B = plume::RenderBlend;
+    constexpr B factors[] = {B::ZERO, B::ONE, B::UNKNOWN, B::UNKNOWN,
+        B::SRC_COLOR, B::INV_SRC_COLOR, B::SRC_ALPHA, B::INV_SRC_ALPHA,
+        B::DEST_COLOR, B::INV_DEST_COLOR, B::DEST_ALPHA, B::INV_DEST_ALPHA,
+        B::BLEND_FACTOR, B::INV_BLEND_FACTOR, B::UNKNOWN, B::UNKNOWN, B::SRC_ALPHA_SAT};
+    using O = plume::RenderBlendOperation;
+    constexpr O operations[] = {O::ADD, O::SUBTRACT, O::MIN, O::MAX, O::REV_SUBTRACT};
+    auto alpha_factor = [&](u32 factor) {
+      // Xenos color factors reduce to their alpha component in the alpha equation.
+      if (factor == 4 || factor == 5 || factor == 8 || factor == 9) factor += 2;
+      return factor == 16 ? B::ONE : factors[factor];
+    };
+    auto& host_blend = desc.renderTargetBlend[i];
+    host_blend.blendEnabled = blend.enabled;
+    host_blend.srcBlend = factors[blend.source];
+    host_blend.dstBlend = factors[blend.destination];
+    host_blend.blendOp = operations[blend.operation];
+    host_blend.srcBlendAlpha = alpha_factor(blend.alpha_source);
+    host_blend.dstBlendAlpha = alpha_factor(blend.alpha_destination);
+    host_blend.blendOpAlpha = operations[blend.alpha_operation];
+  }
   desc.depthTargetFormat = depth_format;
   desc.inputElements = declaration.elements;
   desc.inputElementsCount = declaration.element_count;
   desc.inputSlots = slots.data();
   desc.inputSlotsCount = static_cast<u32>(slots.size());
   auto pipeline = HostDevice::Device()->createGraphicsPipeline(desc);
-  if (!pipeline) {
-    return nullptr;
+    if (!pipeline || !static_cast<plume::D3D12GraphicsPipeline*>(pipeline.get())->d3d) {
+      static std::atomic<u32> failed_pipeline_logs{0};
+      if (failed_pipeline_logs.fetch_add(1) < 16)
+        REXLOG_ERROR("Native GPU: D3D12 rejected pipeline VS=0x{:08X} PS=0x{:08X} decl=0x{:08X}",
+                     key.vertex_shader, key.pixel_shader, key.declaration);
+      return nullptr;
   }
   auto* result = pipeline.get();
+  static u32 mrt_logs = 0;
+  if (desc.renderTargetCount > 1 && mrt_logs++ < 16)
+    REXLOG_INFO("Native GPU: MRT pipeline targets={} formats={},{},{},{} masks={:04X} blends={:08X},{:08X},{:08X},{:08X}",
+        desc.renderTargetCount, key.render_target_formats[0], key.render_target_formats[1],
+        key.render_target_formats[2], key.render_target_formats[3], key.color_write_mask,
+        key.blend_controls[0], key.blend_controls[1], key.blend_controls[2], key.blend_controls[3]);
   g_pipelines.emplace(key, std::move(pipeline));
   return result;
 }
 
+u32 TraceMeshDraw(D3DDevice* device, u32 primitive, bool indexed,
+                  u32 start, u32 count, i32 base_vertex) {
+  static const bool enabled = std::getenv("LEGO_NATIVE_MESH_TRACE") != nullptr;
+  if (!enabled || !device) return 0;
+  const u64 vs = BoundShaderHash(ShaderStage::kVertex);
+  const u64 ps = BoundShaderHash(ShaderStage::kPixel);
+  const bool brick_trace =
+      (vs == 0x5FE844BD78B0368Cull && ps == 0xDAB518E550952312ull) ||
+      (vs == 0xC0C579A9C7D3ACD9ull && ps == 0xBC67385D86BABED8ull) ||
+      (vs == 0xFFC80E0F8B28B228ull &&
+          (ps == 0x1998CB74A7EDAFE1ull || ps == 0x336657153E47E0C3ull)) ||
+      (vs == 0x362C416499E90A3Bull && ps == 0x0705E91ECC4BA3BBull);
+  const bool primary_trace = vs == 0x35DB03916F21B74Full || vs == 0xA9FDB0A3AC4591FEull ||
+      vs == 0x0406577EF00099BDull || vs == 0x5EFE4DB4D125C51Cull ||
+      vs == 0x91007ACB3E640E3Dull || vs == 0xC3958E2D1B795ED9ull;
+  if (vs != 0x35DB03916F21B74Full && vs != 0xA9FDB0A3AC4591FEull &&
+      vs != 0x0406577EF00099BDull && vs != 0x5EFE4DB4D125C51Cull &&
+      vs != 0x91007ACB3E640E3Dull && vs != 0xC3958E2D1B795ED9ull &&
+      vs != 0xF8F4E64872206660ull && vs != 0x386847F375C2779Cull &&
+      vs != 0xC9DB3F6A8FA579E2ull && vs != 0xE634DE2932FB7C5Full &&
+      vs != 0x6E25E88B9AE03E54ull && vs != 0x5D70ED051F7FFB7Cull &&
+      vs != 0x07ADC2A5768CE459ull && !brick_trace) return 0;
+  const auto bindings = SnapshotDrawBindings();
+  // Sample distinct geometry ranges, before any native draw rejection. Do not
+  // hash changing bone matrices into the key and flood every animated frame.
+  const auto* bytes = reinterpret_cast<const u8*>(device);
+  const float alpha = *reinterpret_cast<const be_f32*>(bytes + 0x780 + 73 * 16 + 12);
+  // The brick shaders are shared by other assets. Retain a bounded temporal
+  // sample and distinct instance matrices rather than attributing every hit
+  // to PrologueC or exhausting the quota on its first cloud of bricks.
+  const u64 frame_bucket = brick_trace ? g_probe_frame.load() / 60 : 0;
+  u64 instance_signature = 0;
+  if (brick_trace) {
+    // Diagnostic-only FNV over the exact guest c48..54 bytes.
+    instance_signature = 14695981039346656037ull;
+    for (u32 i = 0; i < 7 * 16; ++i)
+      instance_signature = (instance_signature ^ bytes[0x780 + 48 * 16 + i]) * 1099511628211ull;
+  }
+  using Signature = std::array<u64, 15>;
+  const auto& stream = bindings.vertex_streams[0];
+  const Signature signature{vs, BoundShaderHash(ShaderStage::kPixel),
+      stream.buffer, stream.offset, stream.stride, bindings.index_buffer,
+      start, count, u32(base_vertex), primitive, indexed, device->vertex_declaration,
+      alpha <= 0.0f ? 0u : alpha < 0.5f ? 1u : alpha < 0.99f ? 2u : 3u,
+      frame_bucket, instance_signature};
+  static std::set<Signature> seen;  // Recording lock serializes callers.
+  // Shared body shader hashes can also occur on earlier scenery. Keep their
+  // quota separate so they cannot exhaust the arm/logo trace before Vortech.
+  static std::array<u32, 3> group_counts{};
+  static u64 brick_bucket = UINT64_MAX;
+  static u32 brick_bucket_count = 0;
+  if (brick_trace && brick_bucket != frame_bucket) {
+    brick_bucket = frame_bucket;
+    brick_bucket_count = 0;
+  }
+  const u32 group = brick_trace ? 2 : primary_trace ? 0 : 1;
+  if (group_counts[group] >= (brick_trace ? 512u : 256u) ||
+      (brick_trace && brick_bucket_count >= 8) || !seen.insert(signature).second) return 0;
+  ++group_counts[group];
+  if (brick_trace) ++brick_bucket_count;
+  const u32 id = u32(seen.size());
+  const auto read = [&](u32 offset) { return u32(*reinterpret_cast<const be_u32*>(bytes + offset)); };
+  REXLOG_INFO("Native mesh: id={} VS={:016X} PS={:016X} primitive={} indexed={} start={} count={} base={} decl={:08X} IB={:08X} RT={:08X} DS={:08X} depth={:08X} stencil={:08X} raster={:08X} mask={:08X} blend={:08X} tex5={:08X}",
+      id, vs, signature[1], primitive, indexed, start, count, base_vertex,
+      u32(device->vertex_declaration), bindings.index_buffer, bindings.render_targets[0],
+      bindings.depth_stencil, read(10548), read(10496), read(10568), read(10460), read(10552), bindings.textures[5]);
+  REXLOG_INFO("Native mesh: id={} alpha_control={:08X} alpha_ref={}", id,
+      read(10556), float(*reinterpret_cast<const be_f32*>(bytes + 10620)));
+  REXLOG_INFO("Native mesh: id={} frame={} brick_pair={}", id,
+      g_probe_frame.load(), brick_trace);
+  for (u32 i = 0; i < bindings.vertex_streams.size(); ++i) {
+    const auto& s = bindings.vertex_streams[i];
+    if (s.buffer) REXLOG_INFO("Native mesh: id={} stream={} VB={:08X} offset={} stride={}", id, i, s.buffer, s.offset, s.stride);
+  }
+  // Camera and the complete 52-bone palette are needed to evaluate skinned
+  // positions offline. The first four rows alone miss influences 2..51.
+  for (u32 c = 0; c < 256; ++c) {
+    if (!(c <= 8 || (c >= 16 && c <= 24) || (c >= 43 && c <= 54) ||
+          c == 73 || (!brick_trace && c >= 92 && c <= 247) ||
+          (brick_trace && ((c >= 36 && c <= 42) || c == 72)))) continue;
+    const auto* value = reinterpret_cast<const be_f32*>(bytes + 0x780 + c * 16);
+    REXLOG_INFO("Native mesh: id={} c{}=({},{},{},{})", id, c,
+        float(value[0]), float(value[1]), float(value[2]), float(value[3]));
+  }
+  for (u32 c : {4u,45u,53u,54u,55u,56u}) {
+    const auto* value = reinterpret_cast<const be_f32*>(bytes + 0x1780 + c * 16);
+    REXLOG_INFO("Native mesh: id={} PS c{}=({},{},{},{})", id, c,
+        float(value[0]), float(value[1]), float(value[2]), float(value[3]));
+  }
+  return id;
+}
+
 bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
                   u32 start, u32 count, i32 base_vertex) {
+  DrawTimer timing;
+  const u32 mesh_trace = TraceMeshDraw(device, primitive_type, indexed, start, count, base_vertex);
+  // Report early exits as well as successful submissions for each sample.
+  struct MeshTraceResult {
+    u32 id;
+    bool submitted = false;
+    ~MeshTraceResult() { if (id) REXLOG_INFO("Native mesh: id={} submitted={}", id, submitted); }
+  } mesh_result{mesh_trace};
   if (!device || !count) {
     return false;
   }
+  if (g_logo_capture_api && indexed && count == 6 && start == 24 && base_vertex == 16 &&
+      BoundShaderHash(ShaderStage::kVertex) == 0x91007ACB3E640E3Dull &&
+      BoundShaderHash(ShaderStage::kPixel) == 0xEEE573BE160E037Dull &&
+      float(*reinterpret_cast<const be_f32*>(reinterpret_cast<const u8*>(device) +
+          0x780 + 73 * 16 + 12)) >= 0.95f) RequestLogoCapture();
+  if (ShouldSkipBoundPixelShader()) return true;
   plume::RenderPrimitiveTopology topology;
   if (!MapTopology(primitive_type, topology)) {
+    LongProbeEvent("unsupported_primitive", true, "type=", primitive_type);
     if (g_unsupported_draw_logs++ < 20) {
       REXLOG_WARN("Native GPU: skipped unsupported primitive type {}",
                   primitive_type);
     }
     return false;
   }
-  const DrawBindings bindings = SnapshotDrawBindings();
+  DrawBindings bindings = SnapshotDrawBindings();
+  if (g_logo_capture_api) {
+    static std::set<std::array<u64, 6>> traced;
+    const std::array<u64, 6> identity{BoundShaderHash(ShaderStage::kVertex),
+        BoundShaderHash(ShaderStage::kPixel), bindings.render_targets[0],
+        bindings.depth_stencil, u32(device->render_targets[0]), u32(device->depth_stencil)};
+    if (traced.size() < 96 && traced.insert(identity).second)
+      REXLOG_INFO("Native attachment capture: VS={:016X} PS={:016X} count={} native_rt={:08X} native_ds={:08X} shadow_rt={:08X} shadow_ds={:08X} depth_control={:08X}",
+          identity[0], identity[1], count, identity[2], identity[3], identity[4], identity[5],
+          u32(*reinterpret_cast<const be_u32*>(reinterpret_cast<const u8*>(device)+10548)));
+  }
+  // SetFVF and inline SDK paths may write the declaration without invoking
+  // the public SetVertexDeclaration hook.
+  bindings.vertex_declaration = device->vertex_declaration;
   const auto declaration = ResolveVertexDeclaration(bindings.vertex_declaration);
-  const auto color = ResolveTextureResource(bindings.render_targets[0]);
+  std::array<TextureResourceView, kNativeRenderTargets> colors;
+  for (u32 i = 0; i < colors.size(); ++i)
+    colors[i] = ResolveTextureResource(bindings.render_targets[i]);
+  const auto& color = colors[0];
+  if (!PrepareSurfaceDepthAlias(bindings.depth_stencil)) return false;
   const auto depth = ResolveTextureResource(bindings.depth_stencil);
-  auto* framebuffer = ResolveFramebuffer(bindings.render_targets[0],
-                                         bindings.depth_stencil);
+  auto* framebuffer = ResolveFramebuffer(bindings.render_targets, bindings.depth_stencil);
   if (!declaration.supported || !declaration.elements || !framebuffer ||
       (!color.texture && !depth.texture)) {
+    LongProbeEvent("incomplete_draw_bindings", true, "decl=", bindings.vertex_declaration,
+        "rt=", bindings.render_targets[0], "ds=", bindings.depth_stencil);
     if (g_unsupported_draw_logs++ < 20) {
       REXLOG_WARN("Native GPU: skipped draw with incomplete bindings "
                   "(decl=0x{:08X}, rt=0x{:08X}, ds=0x{:08X})",
@@ -245,16 +631,18 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
     }
     return false;
   }
+  timing.Next();
   auto* commands = HostDevice::BeginFrameCommands();
   if (!commands) {
     return false;
   }
+  timing.Next();
 
-  std::array<plume::RenderTextureBarrier, 2> texture_barriers;
+  std::array<plume::RenderTextureBarrier, kNativeRenderTargets + 1> texture_barriers;
   u32 barrier_count = 0;
-  if (color.texture) {
-    texture_barriers[barrier_count++] = plume::RenderTextureBarrier(
-        color.texture, plume::RenderTextureLayout::COLOR_WRITE);
+  for (const auto& target : colors) {
+    if (target.texture) texture_barriers[barrier_count++] = plume::RenderTextureBarrier(
+        target.texture, plume::RenderTextureLayout::COLOR_WRITE);
   }
   if (depth.texture) {
     texture_barriers[barrier_count++] = plume::RenderTextureBarrier(
@@ -267,39 +655,114 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
   PipelineKey key;
   key.vertex_shader = BoundShaderAddress(ShaderStage::kVertex);
   key.pixel_shader = BoundShaderAddress(ShaderStage::kPixel);
+  key.vertex_shader_hash = BoundShaderHash(ShaderStage::kVertex);
+  key.pixel_shader_hash = BoundShaderHash(ShaderStage::kPixel);
+  key.declaration_hash = declaration.content_hash;
   key.declaration = bindings.vertex_declaration;
-  key.render_target_format = static_cast<u32>(color.format);
+  for (u32 i = 0; i < colors.size(); ++i)
+    key.render_target_formats[i] = static_cast<u32>(colors[i].format);
   key.depth_format = static_cast<u32>(depth.format);
+  static const bool viewport_candidate = std::getenv("LEGO_NATIVE_VIEWPORT") != nullptr;
+  if (viewport_candidate)
+    key.clip_disable = (u32(*reinterpret_cast<const be_u32*>(
+        reinterpret_cast<const u8*>(device) + 10564)) >> 16) & 1u;
+  // TU23 stencil enable / two-sided / z-fail setters update the
+  // RB_DEPTHCONTROL shadow at +10548 (0x2934), with hardware bit positions.
+  // Reading the shadow also covers inlined setters, unlike hooking one caller.
+  key.depth_control = *reinterpret_cast<const be_u32*>(
+      reinterpret_cast<const u8*>(device) + 0x2934) & 0x76u;
+  // Isolated candidate until cutscene/menu pixels have been compared. The
+  // normal path retains its previous depth-only pipeline identity.
+  static const bool enable_stencil = std::getenv("LEGO_NATIVE_STENCIL") != nullptr;
+  if (enable_stencil) {
+    key.depth_control = *reinterpret_cast<const be_u32*>(
+        reinterpret_cast<const u8*>(device) + 10548);
+    if ((key.depth_control & 1u) && depth.format == plume::RenderFormat::D32_FLOAT_S8_UINT) {
+      // D3D12 has one mask pair. With native culling disabled, follow Xenia's
+      // front-face mask choice. Per-face mask differences remain unsupported.
+      key.stencil_masks = *reinterpret_cast<const be_u32*>(
+          reinterpret_cast<const u8*>(device) + 10496) & 0x00FFFF00u;
+      key.stencil_face = (*reinterpret_cast<const be_u32*>(
+          reinterpret_cast<const u8*>(device) + 10568) >> 2) & 1u;
+      static std::unordered_set<u64> logged_stencil;
+      const u64 identity = (u64(key.depth_control) << 32) | key.stencil_masks;
+      if (logged_stencil.size() < 32 && logged_stencil.insert(identity).second)
+        REXLOG_INFO("Native GPU: stencil candidate control={:08X} masks={:06X} ref={} VS={:016X} PS={:016X}",
+            key.depth_control, key.stencil_masks, reinterpret_cast<const u8*>(device)[10499],
+            key.vertex_shader_hash, key.pixel_shader_hash);
+    }
+  }
+  // TU23 setter 83FB8A68 and getter 83FB8AA0 prove the requested RT0
+  // channel mask at +12292. Unlike the hardware shadow it is independent of
+  // whether an RT was attached when the setter ran (SetRT is hooked here).
+  for (u32 i = 0; i < kNativeRenderTargets; ++i)
+    key.color_write_mask |= (*reinterpret_cast<const be_u32*>(
+        reinterpret_cast<const u8*>(device) + 12292 + 4 * i) & 15u) << (4 * i);
+  // TU23 83FB7E38 folds enable/separate-alpha into RB_BLENDCONTROL0:
+  // disabling blending writes 0x00010001 at +10552. Read this folded state.
+  // 83FB7E38 updates RT0 at +10552 and RT1..3 at +10584/+10588/+10592.
+  constexpr u32 blend_offsets[] = {10552, 10584, 10588, 10592};
+  for (u32 i = 0; i < kNativeRenderTargets; ++i)
+    key.blend_controls[i] = *reinterpret_cast<const be_u32*>(
+        reinterpret_cast<const u8*>(device) + blend_offsets[i]) & 0x1FFF1FFFu;
+  static std::atomic<u32> depth_state_logs{0};
+  if (depth_state_logs.fetch_add(1) < 32)
+    REXLOG_INFO("Native GPU: draw depth-control=0x{:02X} target=0x{:08X}",
+                key.depth_control, bindings.depth_stencil);
   key.topology = static_cast<u32>(topology);
   key.spec_constants = declaration.has_r11g11b10_normal ? 1u : 0u;
+  const auto* state_bytes = reinterpret_cast<const u8*>(device);
+  const auto alpha = DecodeNativeAlphaState(
+      *reinterpret_cast<const be_u32*>(state_bytes + 10556),
+      *reinterpret_cast<const be_f32*>(state_bytes + 10620));
+  if (alpha.enabled) key.spec_constants |= 2u;
   for (u32 i = 0; i < kNativeVertexStreams; ++i) {
     key.strides[i] = bindings.vertex_streams[i].stride;
   }
-  auto* pipeline = GetPipeline(key, declaration, topology, color.format,
-                               depth.format);
+  auto* pipeline = GetPipeline(key, declaration, topology, depth.format);
   if (!pipeline) {
+    LongProbeEvent("draw_pipeline_unavailable", true, "VS=", key.vertex_shader_hash,
+        "PS=", key.pixel_shader_hash, "decl_hash=", key.declaration_hash);
     return false;
   }
+  timing.Next();
 
   commands->setGraphicsPipelineLayout(HostDevice::PipelineLayout());
+  if (enable_stencil && (key.depth_control & 1u))
+    static_cast<plume::D3D12CommandList*>(commands)->d3d->OMSetStencilRef(
+        reinterpret_cast<const u8*>(device)[10499]);
   commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 0);
   commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 1);
   commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 2);
   commands->setGraphicsDescriptorSet(HostDevice::SamplerDescriptorSet(), 3);
-  auto constants = BindConstants(commands, device, bindings, declaration);
+  auto constants = BindConstants(commands, device, bindings, declaration, count<=6);
   if (!constants) {
     return false;
   }
+  // A raw depth texture view may need a conversion draw while binding textures.
+  // Restore the guest framebuffer after that helper's temporary target.
+  commands->barriers(plume::RenderBarrierStage::GRAPHICS, texture_barriers.data(), barrier_count);
+  commands->setFramebuffer(framebuffer);
+  timing.Next();
   commands->setPipeline(pipeline);
+  // 83FB82F0 writes normalized R,G,B,A blend constants at +10464..10476.
+  const auto* guest_blend_factor = reinterpret_cast<const be_f32*>(
+      reinterpret_cast<const u8*>(device) + 10464);
+  const float blend_factor[] = {guest_blend_factor[0], guest_blend_factor[1],
+                               guest_blend_factor[2], guest_blend_factor[3]};
+  static_cast<plume::D3D12CommandList*>(commands)->d3d->OMSetBlendFactor(blend_factor);
 
   const float viewport_width = static_cast<u32>(device->viewport.width);
   const float viewport_height = static_cast<u32>(device->viewport.height);
+  const bool screen_space = viewport_candidate && u32(*reinterpret_cast<const be_u32*>(
+      reinterpret_cast<const u8*>(device) + 10572)) == 0x400;
   commands->setViewports(plume::RenderViewport(
-      static_cast<u32>(device->viewport.x), static_cast<u32>(device->viewport.y),
-      viewport_width > 0 ? viewport_width : float(color.width),
-      viewport_height > 0 ? viewport_height : float(color.height),
-      static_cast<float>(device->viewport.min_z),
-      static_cast<float>(device->viewport.max_z)));
+      screen_space ? 0 : static_cast<u32>(device->viewport.x),
+      screen_space ? 0 : static_cast<u32>(device->viewport.y),
+      screen_space ? float(color.width) : viewport_width > 0 ? viewport_width : float(color.width),
+      screen_space ? float(color.height) : viewport_height > 0 ? viewport_height : float(color.height),
+      screen_space ? 0.0f : static_cast<float>(device->viewport.min_z),
+      screen_space ? 1.0f : static_cast<float>(device->viewport.max_z)));
   const i32 scissor_right = static_cast<i32>(device->scissor.right);
   const i32 scissor_bottom = static_cast<i32>(device->scissor.bottom);
   commands->setScissors(plume::RenderRect(
@@ -311,22 +774,226 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
   std::array<plume::RenderVertexBufferView, kNativeVertexStreams> views;
   std::array<plume::RenderInputSlot, kNativeVertexStreams> slots;
   auto* null_buffer = HostDevice::NullVertexBuffer();
+  std::array<BufferResourceView, kNativeVertexStreams> vertex_buffers;
+  static const bool window_uploads = [] {
+    const char* value = std::getenv("LEGO_NATIVE_BUFFER_WINDOWS");
+    const bool enabled = value ? std::strcmp(value, "0") != 0 : REXCVAR_GET(gpu_native_buffer_windows);
+    REXLOG_INFO("Native buffer range cache: {} ({})", enabled,
+        value ? "environment override" : "saved configuration");
+    return enabled;
+  }();
+  std::array<VertexBufferWindow, kNativeVertexStreams> vertex_windows{};
+  bool use_windows = false;
+  u32 min_index = 0, index_offset = 0, index_bytes = 0;
+  if (window_uploads && indexed && count &&
+      primitive_type != static_cast<u32>(rex::graphics::xenos::PrimitiveType::kQuadList)) {
+    const auto index = InspectBufferResource(bindings.index_buffer, BufferKind::kIndex);
+    const u32 element_size = index.guest_format == 1 ? 2 : 4;
+    if (index.mirror_address && u64(start) * element_size <= index.length &&
+        u64(count) * element_size <= index.length - u64(start) * element_size) {
+      index_offset = start * element_size;
+      index_bytes = count * element_size;
+      const auto* source = REX_KERNEL_MEMORY()->TranslateVirtual<const u8*>(index.mirror_address + index_offset);
+      min_index = UINT32_MAX;
+      u32 max_index = 0;
+      for (u32 i = 0; i < count; ++i) {
+        const u32 value = element_size == 2 ?
+            u32(reinterpret_cast<const be_u16*>(source)[i]) :
+            u32(reinterpret_cast<const be_u32*>(source)[i]);
+        min_index = std::min(min_index, value);
+        max_index = std::max(max_index, value);
+      }
+      // D3D12's signed base vertex must represent -min_index exactly.
+      use_windows = min_index <= INT32_MAX;
+      for (u32 i = 0; use_windows && i < kNativeVertexStreams; ++i) {
+        const auto& binding = bindings.vertex_streams[i];
+        if (!binding.buffer) continue;  // Synthetic/optional zero stream.
+        const bool used = std::any_of(declaration.elements,
+            declaration.elements + declaration.element_count,
+            [i](const auto& e) { return e.slotIndex == i; });
+        if (!used) continue;
+        for (u32 e = 0; e < declaration.element_count; ++e) {
+          const auto& element = declaration.elements[e];
+          if (element.slotIndex != i) continue;
+          const u32 width = plume::RenderFormatSize(element.format);
+          if (!width || element.alignedByteOffset > binding.stride ||
+              width > binding.stride - element.alignedByteOffset) use_windows = false;
+        }
+        if (!use_windows) break;
+        const auto metadata = InspectBufferResource(binding.buffer, BufferKind::kVertex);
+        vertex_windows[i] = DrawVertexWindow(min_index, max_index, base_vertex,
+            binding.stride, binding.offset, metadata.length);
+        if (!vertex_windows[i].length) use_windows = false;
+      }
+    }
+  }
   for (u32 i = 0; i < kNativeVertexStreams; ++i) {
     const auto& binding = bindings.vertex_streams[i];
-    const auto buffer = ResolveBufferResourceView(binding.buffer,
-                                                  BufferKind::kVertex);
-    const u32 offset = std::min(binding.offset, buffer.length);
+    // D3D keeps old stream bindings across declaration changes. A stream with
+    // no input element cannot be fetched by this draw. Do not scan/upload a
+    // previously bound world mesh for a fullscreen quad using another stream.
+    const bool used = std::any_of(declaration.elements,
+        declaration.elements + declaration.element_count,
+        [i](const auto& element) { return element.slotIndex == i; });
+    slots[i] = plume::RenderInputSlot(i, binding.stride ? binding.stride : 16);
+    if (!used) {
+      views[i] = plume::RenderVertexBufferView(null_buffer->at(0), 256);
+      continue;
+    }
+    std::vector<u32> reversed_elements;
+    for (u32 e = 0; e < std::min(declaration.element_count, 64u); ++e) {
+      if ((declaration.reversed_byte_elements & (u64{1} << e)) &&
+          declaration.elements[e].slotIndex == i)
+        reversed_elements.push_back(declaration.elements[e].alignedByteOffset);
+    }
+    std::sort(reversed_elements.begin(), reversed_elements.end());
+    reversed_elements.erase(std::unique(reversed_elements.begin(), reversed_elements.end()),
+                             reversed_elements.end());
+    const auto window = vertex_windows[i];
+    const auto buffer = use_windows && window.length ?
+        ResolveBufferResourceWindow(binding.buffer, BufferKind::kVertex,
+            window.offset, window.length, {binding.stride, 0, reversed_elements}) :
+        ResolveBufferResourceView(binding.buffer,
+            BufferKind::kVertex, {binding.stride, binding.offset, reversed_elements});
+    vertex_buffers[i] = buffer;
+    const u32 offset = use_windows && window.length ? 0 : std::min(binding.offset, buffer.length);
     views[i] = buffer.buffer
                    ? plume::RenderVertexBufferView(buffer.buffer->at(offset),
                                                    buffer.length - offset)
                    : plume::RenderVertexBufferView(null_buffer->at(0), 256);
-    slots[i] = plume::RenderInputSlot(i, binding.stride ? binding.stride : 16);
   }
   commands->setVertexBuffers(0, views.data(), views.size(), slots.data());
+  if (mesh_trace) {
+    for (u32 i = 0; i < vertex_buffers.size(); ++i) {
+      const auto& buffer = vertex_buffers[i];
+      if (buffer.buffer)
+        REXLOG_INFO("Native mesh: id={} stream={} data={:08X} bytes={} content={:016X} conversion={:016X}",
+            mesh_trace, i, buffer.mirror_address, buffer.length,
+            buffer.content_hash, buffer.byte_order_hash);
+    }
+    i64 first_vertex = start;
+    if (indexed) {
+      const auto index = ResolveBufferResourceView(bindings.index_buffer, BufferKind::kIndex);
+      REXLOG_INFO("Native mesh: id={} index_data={:08X} bytes={} format={} content={:016X}",
+          mesh_trace, index.mirror_address, index.length, index.guest_format, index.content_hash);
+      const u32 size = index.guest_format == 1 ? 2 : 4;
+      if (index.buffer && (u64(start) + 1) * size <= index.length) {
+        const auto* raw = static_cast<const u8*>(index.buffer->map());
+        if (raw) {
+          first_vertex = (size == 2 ? reinterpret_cast<const u16*>(raw)[start]
+                                   : reinterpret_cast<const u32*>(raw)[start]) + i64(base_vertex);
+          index.buffer->unmap();
+        } else first_vertex = -1;
+      } else first_vertex = -1;
+    }
+    REXLOG_INFO("Native mesh: id={} first_vertex={} packed_normal={}", mesh_trace,
+        first_vertex, declaration.has_r11g11b10_normal);
+    for (u32 e = 0; e < declaration.element_count; ++e) {
+      const auto& element = declaration.elements[e];
+      if (element.slotIndex >= vertex_buffers.size()) continue;
+      const auto& binding = bindings.vertex_streams[element.slotIndex];
+      const auto buffer = vertex_buffers[element.slotIndex];
+      const u64 absolute_offset = first_vertex < 0 ? UINT64_MAX :
+          u64(binding.offset) + u64(first_vertex) * binding.stride + element.alignedByteOffset;
+      const u64 origin = use_windows ? vertex_windows[element.slotIndex].offset : 0;
+      const u64 offset = absolute_offset >= origin ? absolute_offset - origin : UINT64_MAX;
+      u32 words[2]{};
+      bool sampled = false;
+      if (buffer.buffer && offset <= buffer.length && buffer.length - offset >= sizeof(words)) {
+        if (const auto* raw = static_cast<const u8*>(buffer.buffer->map())) {
+          std::memcpy(words, raw + offset, sizeof(words));
+          buffer.buffer->unmap();
+          sampled = true;
+        }
+      }
+      REXLOG_INFO("Native mesh: id={} attr={}{} slot={} offset={} stride={} format={} reversed={} length={} sampled={} words={:08X},{:08X}",
+          mesh_trace, element.semanticName, element.semanticIndex, element.slotIndex,
+          element.alignedByteOffset, binding.stride, u32(element.format),
+          (declaration.reversed_byte_elements >> e) & 1, buffer.length, sampled, words[0], words[1]);
+    }
+  }
+  timing.Next();
 
-  if (indexed) {
-    const auto index = ResolveBufferResourceView(bindings.index_buffer,
-                                                 BufferKind::kIndex);
+  // Opt-in, bounded startup trace: separate texture color from vertex tint,
+  // float constants, and declaration decoding without altering the draw.
+  static const bool trace_color = [] {
+    char* value = nullptr;
+    size_t length = 0;
+    _dupenv_s(&value, &length, "LEGO_NATIVE_COLOR_TRACE");
+    const bool enabled = value && *value;
+    std::free(value);
+    return enabled;
+  }();
+  static u32 color_traces = 0;  // Serialized by LockRecording.
+  if (trace_color && color_traces++ < 8) {
+    const auto* constant_bytes = reinterpret_cast<const u8*>(device);
+    const auto* tint = reinterpret_cast<const be_f32*>(constant_bytes + 0x780 + 73 * 16);
+    const auto* ps45 = reinterpret_cast<const be_f32*>(constant_bytes + 0x1780 + 45 * 16);
+    const auto* ps32 = reinterpret_cast<const be_f32*>(constant_bytes + 0x1780 + 32 * 16);
+    REXLOG_INFO("Native color: draw={} VS={:08X} PS={:08X} start={} base={} indexed={} "
+                "VS73=({},{},{},{}) PS45=({},{},{},{}) PS32.x={}",
+        color_traces, key.vertex_shader, key.pixel_shader, start, base_vertex, indexed,
+        float(tint[0]), float(tint[1]), float(tint[2]), float(tint[3]),
+        float(ps45[0]), float(ps45[1]), float(ps45[2]), float(ps45[3]), float(ps32[0]));
+    for (u32 i = 0; i < declaration.element_count; ++i) {
+      const auto& element = declaration.elements[i];
+      if (element.slotIndex >= bindings.vertex_streams.size()) continue;
+      const auto& binding = bindings.vertex_streams[element.slotIndex];
+      const auto buffer = vertex_buffers[element.slotIndex];
+      const u64 offset = (use_windows ? 0 : u64(binding.offset)) + element.alignedByteOffset;
+      if (!buffer.buffer || offset + 4 > buffer.length) continue;
+      const auto* mapped_input = static_cast<const u8*>(buffer.buffer->map());
+      if (!mapped_input) continue;
+      u32 word = 0;
+      std::memcpy(&word, mapped_input + offset, sizeof(word));
+      buffer.buffer->unmap();
+      REXLOG_INFO("Native color: {}{} slot={} offset={} stride={} format={} host-word={:08X}",
+          element.semanticName, element.semanticIndex, element.slotIndex,
+          element.alignedByteOffset, binding.stride, u32(element.format), word);
+    }
+  }
+
+  if (primitive_type == static_cast<u32>(rex::graphics::xenos::PrimitiveType::kQuadList)) {
+    if (count % 4 != 0) return false;
+    const auto source = indexed
+        ? ResolveBufferResourceView(bindings.index_buffer, BufferKind::kIndex)
+        : BufferResourceView{};
+    const u32 element_size = source.guest_format == 1 ? 2 : 4;
+    if (indexed && (!source.buffer || u64(start + u64(count)) * element_size > source.length))
+      return false;
+    auto expanded = std::shared_ptr<plume::RenderBuffer>(HostDevice::Device()->createBuffer(
+        plume::RenderBufferDesc::IndexBuffer(u64(count / 4) * 6 * sizeof(u32),
+                                            plume::RenderHeapType::UPLOAD)).release());
+    if (!expanded) return false;
+    auto* output = static_cast<u32*>(expanded->map());
+    const auto* input = indexed ? static_cast<const u8*>(source.buffer->map()) : nullptr;
+    if (!output || (indexed && !input)) {
+      if (output) expanded->unmap();
+      return false;
+    }
+    constexpr u32 order[] = {0, 1, 2, 0, 2, 3};
+    for (u32 quad = 0; quad < count / 4; ++quad) {
+      for (u32 corner = 0; corner < 6; ++corner) {
+        const u32 index = start + quad * 4 + order[corner];
+        output[quad * 6 + corner] = !indexed ? index
+            : element_size == 2 ? reinterpret_cast<const u16*>(input)[index]
+                                : reinterpret_cast<const u32*>(input)[index];
+      }
+    }
+    if (indexed) source.buffer->unmap();
+    expanded->unmap();
+    const plume::RenderIndexBufferView index_view(expanded->at(0),
+        u64(count / 4) * 6 * sizeof(u32), plume::RenderFormat::R32_UINT);
+    commands->setIndexBuffer(&index_view);
+    {
+      QueryDrawScope query_scope(commands);
+      commands->drawIndexedInstanced(count / 4 * 6, 1, 0, indexed ? base_vertex : 0, 0);
+    }
+    HostDevice::RetireResource(std::move(expanded));
+  } else if (indexed) {
+    const auto index = use_windows ?
+        ResolveBufferResourceWindow(bindings.index_buffer, BufferKind::kIndex, index_offset, index_bytes) :
+        ResolveBufferResourceView(bindings.index_buffer, BufferKind::kIndex);
     if (!index.buffer) {
       return false;
     }
@@ -336,39 +1003,77 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
     const plume::RenderIndexBufferView index_view(index.buffer->at(0),
                                                   index.length, index_format);
     commands->setIndexBuffer(&index_view);
-    commands->drawIndexedInstanced(count, 1, start, base_vertex, 0);
+    QueryDrawScope query_scope(commands);
+    commands->drawIndexedInstanced(count, 1, use_windows ? 0 : start,
+        use_windows ? -i32(min_index) : base_vertex, 0);
   } else {
+    QueryDrawScope query_scope(commands);
     commands->drawInstanced(count, 1, start, 0);
   }
-  HostDevice::RetireResource(std::move(constants));
+  timing.Next();
+  static const bool trace_stages = [] {
+    char* value = nullptr;
+    size_t length = 0;
+    _dupenv_s(&value, &length, "LEGO_GPU_SNAPSHOT_DIR");
+    const bool enabled = value && *value;
+    std::free(value);
+    return enabled;
+  }();
+  if (trace_stages) {
+    if (LongProbeEnabled() && (HostDevice::LongProbeSnapshotActive() ||
+        LongProbeOnce(key.vertex_shader_hash ^ key.pixel_shader_hash ^ declaration.content_hash)))
+      LongProbeEvent("draw_stage", false, "VS=", key.vertex_shader_hash,
+          "PS=", key.pixel_shader_hash, "count=", count, "indexed=", indexed,
+          "decl_hash=", declaration.content_hash, "rt=", bindings.render_targets[0],
+          "width=", color.width, "height=", color.height, "format=", u32(color.format),
+          "depth=", bindings.depth_stencil, "write_mask=", key.color_write_mask,
+          "textures=", LongProbeHex(bindings.textures));
+    static std::unordered_set<PipelineKey, PipelineKeyHash> traced;
+    if (traced.size() < 100 && traced.insert(key).second) {
+      REXLOG_INFO("Native stage: VS={:08X} PS={:08X} rt={:08X} fmt={} ds={:08X} z={:02X} "
+          "view={},{},{},{} scissor={},{},{},{} tex={:08X},{:08X},{:08X},{:08X} "
+          "tex4-7={:08X},{:08X},{:08X},{:08X} mask={:X} blend={:08X}",
+          key.vertex_shader, key.pixel_shader, bindings.render_targets[0], key.render_target_formats[0],
+          bindings.depth_stencil, key.depth_control, u32(device->viewport.x), u32(device->viewport.y),
+          u32(device->viewport.width), u32(device->viewport.height), i32(device->scissor.left),
+          i32(device->scissor.top), scissor_right, scissor_bottom,
+          bindings.textures[0], bindings.textures[1], bindings.textures[2], bindings.textures[3],
+          bindings.textures[4], bindings.textures[5], bindings.textures[6], bindings.textures[7],
+          key.color_write_mask, u32(*reinterpret_cast<const be_u32*>(
+              reinterpret_cast<const u8*>(device) + 10552)));
+    }
+    if (color.texture && (!LongProbeEnabled() || count <= 6))
+      HostDevice::SnapshotTexture(color.texture,
+          "draw-vs-" + std::to_string(key.vertex_shader) + "-ps-" +
+          std::to_string(key.pixel_shader), true);
+  }
+  static std::atomic<u32> issued_draw_logs{0};
+  if (issued_draw_logs.fetch_add(1) < 8)
+    REXLOG_INFO("Native GPU: issued draw primitive={} count={} indexed={} decl=0x{:08X}",
+                 primitive_type, count, indexed, bindings.vertex_declaration);
+  mesh_result.submitted = true;
+  for (u32 i = 0; i < kNativeRenderTargets; ++i)
+    if ((key.color_write_mask >> (i * 4)) & 15u) MarkSurfaceWritten(bindings.render_targets[i]);
+  if (DecodeDepthState(key.depth_control, depth.texture != nullptr).write)
+    MarkSurfaceWritten(bindings.depth_stencil);
   return true;
 }
 
 u32 ClearHook(D3DDevice* /*device*/, u32 rectangle_count,
               u32 rectangles_address, u32 flags, u32 color, f64 depth,
               u32 /*depth_gpr_slot*/, u32 stencil, u32 /*edram_clear*/) {
+  auto recording = HostDevice::LockRecording();
   const DrawBindings bindings = SnapshotDrawBindings();
-  const auto target = ResolveTextureResource(bindings.render_targets[0]);
-  const auto depth_target = ResolveTextureResource(bindings.depth_stencil);
-  auto* framebuffer = ResolveFramebuffer(bindings.render_targets[0],
-                                         bindings.depth_stencil);
   auto* commands = HostDevice::BeginFrameCommands();
-  if (!framebuffer || !commands) {
-    return 0;
-  }
-  std::array<plume::RenderTextureBarrier, 2> barriers;
-  u32 barrier_count = 0;
-  if (target.texture) {
-    barriers[barrier_count++] = plume::RenderTextureBarrier(
-        target.texture, plume::RenderTextureLayout::COLOR_WRITE);
-  }
-  if (depth_target.texture) {
-    barriers[barrier_count++] = plume::RenderTextureBarrier(
-        depth_target.texture, plume::RenderTextureLayout::DEPTH_WRITE);
-  }
-  commands->barriers(plume::RenderBarrierStage::GRAPHICS, barriers.data(),
-                     barrier_count);
-  commands->setFramebuffer(framebuffer);
+  if (!commands) return 0x80004005u;
+  static std::atomic<u32> clear_sequence{0};
+  const u32 clear_id = clear_sequence.fetch_add(1);
+  static u32 tiled_clear_logs = 0;
+  const bool trace_clear = clear_id < 128 || (g_seen_tiled_pass && tiled_clear_logs++ < 24);
+  if (trace_clear)
+    REXLOG_INFO("Native GPU: clear #{} flags={:X} rt={:08X},{:08X},{:08X},{:08X} ds={:08X} color={:08X} z={}",
+        clear_id, flags, bindings.render_targets[0], bindings.render_targets[1],
+        bindings.render_targets[2], bindings.render_targets[3], bindings.depth_stencil, color, depth);
 
   std::vector<plume::RenderRect> rectangles;
   if (rectangle_count && rectangles_address) {
@@ -385,72 +1090,251 @@ u32 ClearHook(D3DDevice* /*device*/, u32 rectangle_count,
   const plume::RenderRect* clear_rectangles =
       rectangles.empty() ? nullptr : rectangles.data();
   const u32 clear_rectangle_count = static_cast<u32>(rectangles.size());
-  if ((flags & 0x1) && target.texture) {
-    const float scale = 1.0f / 255.0f;
-    commands->clearColor(
-        0,
-        plume::RenderColor(float((color >> 16) & 0xFF) * scale,
-                           float((color >> 8) & 0xFF) * scale,
-                           float(color & 0xFF) * scale,
-                           float((color >> 24) & 0xFF) * scale),
-        clear_rectangles, clear_rectangle_count);
-  }
-  if ((flags & 0x30) && depth_target.texture) {
+  const float scale = 1.0f / 255.0f;
+  // X360 TARGET0..3 are independent bits 1/2/4/8. Bind each selected
+  // attachment alone for clear; its shader output slot is unchanged for draws.
+  for (u32 i = 0; i < kNativeRenderTargets; ++i) {
+    if (!(flags & (1u << i))) continue;
+    const auto target = ResolveTextureResource(bindings.render_targets[i]);
+    auto* framebuffer = ResolveFramebuffer(bindings.render_targets[i], 0);
+    if (!target.texture || !framebuffer) return 0x80004005u;
+    commands->barriers(plume::RenderBarrierStage::GRAPHICS,
+        plume::RenderTextureBarrier(target.texture, plume::RenderTextureLayout::COLOR_WRITE));
+    commands->setFramebuffer(framebuffer);
+    const bool fixed16 = target.format == plume::RenderFormat::R16G16B16A16_SNORM ||
+                         target.format == plume::RenderFormat::R16G16_SNORM;
+    const float clear_scale = fixed16 ? 1.0f / 32.0f : 1.0f;
+    const plume::RenderColor encoded_color(float((color >> 16) & 0xFF) * scale * clear_scale,
+        float((color >> 8) & 0xFF) * scale * clear_scale,
+        float(color & 0xFF) * scale * clear_scale,
+        float((color >> 24) & 0xFF) * scale * clear_scale);
+    commands->clearColor(0, encoded_color, clear_rectangles, clear_rectangle_count);
+    MarkSurfaceWritten(bindings.render_targets[i]);
+  }
+  if (flags & 0x30) {
+    if (!PrepareSurfaceDepthAlias(bindings.depth_stencil)) return 0x80004005u;
+    const auto depth_target = ResolveTextureResource(bindings.depth_stencil);
+    auto* framebuffer = ResolveFramebuffer(0, bindings.depth_stencil);
+    if (!depth_target.texture || !framebuffer) return 0x80004005u;
+    commands->barriers(plume::RenderBarrierStage::GRAPHICS,
+        plume::RenderTextureBarrier(depth_target.texture, plume::RenderTextureLayout::DEPTH_WRITE));
+    commands->setFramebuffer(framebuffer);
     commands->clearDepthStencil((flags & 0x10) != 0, (flags & 0x20) != 0,
-                                static_cast<float>(depth), stencil,
-                                clear_rectangles, clear_rectangle_count);
+        static_cast<float>(depth), stencil, clear_rectangles, clear_rectangle_count);
+    if (flags & 0x10) MarkSurfaceWritten(bindings.depth_stencil);
   }
+  if (trace_clear) REXLOG_INFO("Native GPU: clear #{} complete", clear_id);
   return 0;
 }
 
 u32 DrawVerticesHook(D3DDevice* device, u32 primitive_type, u32 start_vertex,
                      u32 vertex_count) {
-  DispatchDraw(device, primitive_type, false, start_vertex, vertex_count, 0);
+  auto recording = HostDevice::LockRecording();
+  if (ConsumeCpuPoolCopyDraw(primitive_type, start_vertex, vertex_count)) return 0;
+  if (!DispatchDraw(device, primitive_type, false, start_vertex, vertex_count, 0))
+    FailActiveQueries();
   return 0;
 }
 
 u32 DrawIndexedVerticesHook(D3DDevice* device, u32 primitive_type,
                             u32 base_vertex, u32 start_index, u32 index_count) {
-  DispatchDraw(device, primitive_type, true, start_index, index_count,
-               static_cast<i32>(base_vertex));
+  auto recording = HostDevice::LockRecording();
+  if (!DispatchDraw(device, primitive_type, true, start_index, index_count,
+                    static_cast<i32>(base_vertex))) FailActiveQueries();
   return 0;
 }
 
-u32 ResolveHook(D3DDevice* /*device*/, u32 /*flags*/, u32 /*source_rect*/,
-                u32 destination_texture, u32 /*destination_point*/,
+bool ReadResolveWords(u32 address, i32* output, u32 count) {
+  if (!address || u64(address) + count * 4 > 0x100000000ull) return false;
+  auto* memory = REX_KERNEL_MEMORY();
+  for (u64 at = address; at < u64(address) + count * 4; at = (at & ~u64(4095)) + 4096) {
+    auto* heap = memory->LookupHeap(u32(at));
+    rex::memory::HeapAllocationInfo info{};
+    if (!heap || !heap->QueryRegionInfo(u32(at), &info) ||
+        !(info.state & rex::memory::kMemoryAllocationCommit) ||
+        !(info.protect & rex::memory::kMemoryProtectRead)) return false;
+  }
+  const auto* words = memory->TranslateVirtual<const be_u32*>(address);
+  for (u32 i = 0; i < count; ++i) output[i] = i32(u32(words[i]));
+  return true;
+}
+
+u32 ResolveHook(u32 flags, u32 source_rect,
+                u32 destination_texture, u32 destination_point,
                 u32 destination_level, u32 destination_slice,
-                u32 /*clear_color*/, u32 /*clear_z_hi*/, u32 /*clear_z_lo*/,
-                u32 /*clear_stencil*/, u32 /*parameters*/) {
-  if (!ResolveTextureFromSurface(destination_texture, BoundRenderTarget(0),
-                                 destination_level, destination_slice) &&
+                u32 clear_color, f64 clear_z, u32 lr, u32 sp) {
+  auto recording = HostDevice::LockRecording();
+  i32 rect_words[4]{}, point_words[2]{};
+  if ((source_rect && !ReadResolveWords(source_rect, rect_words, 4)) ||
+      (destination_point && !ReadResolveWords(destination_point, point_words, 2)))
+    return 0x80070057u;
+  const ResolveRect rect{rect_words[0], rect_words[1], rect_words[2], rect_words[3]};
+  const ResolvePoint point{point_words[0], point_words[1]};
+  // Log distinct transfers, not the same loading-screen depth copy hundreds
+  // of times before the first shadow atlas appears.
+  using Transfer = std::array<u32, 10>;
+  static std::vector<Transfer> traced_transfers;
+  const Transfer transfer{flags, destination_texture, destination_level, destination_slice,
+      u32(rect.left), u32(rect.top), u32(rect.right), u32(rect.bottom), u32(point.x), u32(point.y)};
+  if (traced_transfers.size() < 128 &&
+      std::find(traced_transfers.begin(), traced_transfers.end(), transfer) == traced_transfers.end()) {
+    traced_transfers.push_back(transfer);
+    i32 backchain = 0, caller_lr = 0;
+    if (ReadResolveWords(sp, &backchain, 1) && u32(backchain) >= 8)
+      ReadResolveWords(u32(backchain) - 8, &caller_lr, 1);
+    REXLOG_INFO("Native GPU: Resolve ABI lr={:08X} caller={:08X} flags={:08X} "
+        "rect@{:08X}=({},{},{},{}) point@{:08X}=({},{}) clear@{:08X} z={}",
+        lr, u32(caller_lr), flags, source_rect, rect.left, rect.top, rect.right, rect.bottom,
+        destination_point, point.x, point.y, clear_color, clear_z);
+  }
+  // TU23 sub_83FBDF80 selects +12832 for index 4, or +12816+4*index for 0..3.
+  // Depth conversion remains explicit
+  // unsupported work; never feed it unrelated color data.
+  const u32 source = (flags & 4u) ? BoundDepthStencil() : BoundRenderTarget(flags & 3u);
+  const bool resolved = ResolveTextureFromSurface(destination_texture, source,
+                                 destination_level, destination_slice, flags,
+                                 source_rect ? &rect : nullptr,
+                                 destination_point ? &point : nullptr);
+  if (!resolved &&
       g_unsupported_draw_logs++ < 20) {
-    REXLOG_WARN("Native GPU: skipped unsupported resolve to 0x{:08X}",
-                destination_texture);
+    REXLOG_WARN("Native GPU: skipped resolve src=0x{:08X} dst=0x{:08X} "
+                "flags=0x{:08X} level={} slice={}", source,
+                destination_texture, flags, destination_level, destination_slice);
+  }
+  // RESOLVE_CLEARRENDERTARGET takes float RGBA, unlike Clear's packed ARGB.
+  // Clear only after the copy, so accumulated intermediate passes start clean.
+  if (resolved && (flags & 0x100u) && !(flags & 4u) && clear_color) {
+    i32 clear_words[4];
+    if (!ReadResolveWords(clear_color, clear_words, 4)) return 0x80070057u;
+    float rgba[4];
+    std::memcpy(rgba, clear_words, sizeof(rgba));
+    const auto target = ResolveTextureResource(source);
+    auto* framebuffer = ResolveFramebuffer(source, 0);
+    auto* commands = HostDevice::BeginFrameCommands();
+    if (!target.texture || !framebuffer || !commands) return 0x80004005u;
+    const bool fixed16 = target.format == plume::RenderFormat::R16G16B16A16_SNORM ||
+                         target.format == plume::RenderFormat::R16G16_SNORM;
+    const float scale = fixed16 ? 1.0f / 32.0f : 1.0f;
+    commands->barriers(plume::RenderBarrierStage::GRAPHICS,
+        plume::RenderTextureBarrier(target.texture, plume::RenderTextureLayout::COLOR_WRITE));
+    commands->setFramebuffer(framebuffer);
+    const plume::RenderRect area(rect.left, rect.top, rect.right, rect.bottom);
+    commands->clearColor(0, plume::RenderColor(rgba[0]*scale, rgba[1]*scale,
+        rgba[2]*scale, rgba[3]*scale), source_rect ? &area : nullptr, source_rect ? 1 : 0);
+    MarkSurfaceWritten(source);
+    static std::atomic<u32> clear_logs{0};
+    if (clear_logs.fetch_add(1) < 8)
+      REXLOG_INFO("Native GPU: resolve cleared source={:08X} rgba={},{},{},{}",
+          source, rgba[0], rgba[1], rgba[2], rgba[3]);
   }
   return 0;
 }
 
 u32 SwapHook(D3DDevice* /*device*/, u32 front_buffer, u32 /*parameters*/) {
+  HostDevice::PollCompletionCallbacks();
+  auto recording = HostDevice::LockRecording();
   const auto texture = ResolveTextureResource(front_buffer);
   if (!HostDevice::PresentTexture(texture.texture, texture.descriptor_index) &&
       g_unsupported_draw_logs++ < 20) {
+    LogUnknownTexture(front_buffer, "present");
     REXLOG_WARN("Native GPU: skipped present for front buffer 0x{:08X}",
                 front_buffer);
   }
+  recording.unlock();
+  HostDevice::PollCompletionCallbacks();
   return 0;
 }
 
+// Sep'13 tile.obj: Xbox replays these draws through a worker command queue
+// to fit EDRAM. Host render targets hold the whole image, so draw once between
+// the same clear/resolve boundaries without entering that Xbox worker queue.
+u32 BeginTilingHook(D3DDevice* device, u32 flags, u32 count,
+                    u32 rectangles, mapped_f32 clear_color, f64 clear_z,
+                    u32 /*z_gpr_slot*/, u32 clear_stencil) {
+  u32 color = 0;
+  if (const be_f32* rgba = clear_color) {
+    const auto pack = [](float value) {
+      return static_cast<u32>(std::clamp(value, 0.0f, 1.0f) * 255.0f + 0.5f);
+    };
+    color = (pack(rgba[3]) << 24) | (pack(rgba[0]) << 16) |
+            (pack(rgba[1]) << 8) | pack(rgba[2]);
+  }
+  auto recording = HostDevice::LockRecording();
+  auto* memory = REX_KERNEL_MEMORY();
+  const u64 end = u64(rectangles) + u64(count) * sizeof(D3DRect);
+  bool readable = rectangles && count && count <= 16 && end <= 0x100000000ull;
+  for (u64 at = rectangles; readable && at < end; at = (at & ~u64(4095)) + 4096) {
+    auto* heap = memory->LookupHeap(static_cast<u32>(at));
+    rex::memory::HeapAllocationInfo info{};
+    readable = heap && heap->QueryRegionInfo(static_cast<u32>(at), &info) &&
+        (info.state & rex::memory::kMemoryAllocationCommit) &&
+        (info.protect & rex::memory::kMemoryProtectRead);
+  }
+  std::array<NativeTileRect, 16> tiles;
+  if (!readable) return 0x80004001u;
+  const auto* guest_tiles = memory->TranslateVirtual<const D3DRect*>(rectangles);
+  for (u32 i = 0; i < count; ++i)
+    tiles[i] = {i32(guest_tiles[i].left), i32(guest_tiles[i].top),
+                i32(guest_tiles[i].right), i32(guest_tiles[i].bottom)};
+  const auto extent = LogicalTileExtent(std::span(tiles.data(), count));
+  if (!extent.width) return 0x80004001u;
+  const auto bindings = SnapshotDrawBindings();
+  // The observed scene clears AFTER BeginTiling(flags=1), before geometry.
+  // Allocate synchronously here so the clear covers the new complete target.
+  // Do not enable Xbox tiling count/flags or invoke its worker queue.
+  for (u32 surface : bindings.render_targets)
+    if (!PromoteTiledSurface(surface, extent.width, extent.height)) return 0x80004001u;
+  if (!PromoteTiledSurface(bindings.depth_stencil, extent.width, extent.height))
+    return 0x80004001u;
+  g_seen_tiled_pass = true;
+  HostDevice::BeginTiledPassSnapshot();
+  static std::atomic<u32> logs{0};
+  if (logs.fetch_add(1) < 8)
+    REXLOG_INFO("Native GPU: BeginTiling flags={:X} tiles={} host={}x{} viewport={}x{} guest_count={}",
+        flags, count, extent.width, extent.height, u32(device->viewport.width),
+        u32(device->viewport.height), u32(*reinterpret_cast<const be_u32*>(
+            reinterpret_cast<const u8*>(device) + 13124)));
+  // The TU23 implementation branches past its initial clear when bit 0 is set.
+  if (flags & 1) return 0;
+  return ClearHook(device, 0, 0, 0x31, color, clear_z, 0, clear_stencil, 0);
+}
+
+u32 EndTilingHook(D3DDevice* /*device*/, u32 flags, u32 /*rectangles*/,
+                  u32 destination, u32 /*clear_color*/, f64 /*clear_z*/,
+                  u32 /*z_gpr_slot*/, u32 /*clear_stencil*/, u32 /*parameters*/) {
+  if (!destination) return 0;
+  const u32 source = (flags & 4) ? BoundDepthStencil() : BoundRenderTarget(flags & 3);
+  const bool resolved = ResolveTextureFromSurface(destination, source, 0, 0, flags);
+  static std::atomic<u32> logs{0};
+  if (logs.fetch_add(1) < 8)
+    REXLOG_INFO("Native GPU: EndTiling source=0x{:08X} destination=0x{:08X} resolved={}",
+                 source, destination, resolved);
+  return resolved ? 0 : 0x80004001u;
+}
+
 }  // namespace
 
+DrawTiming ConsumeDrawTiming() {
+  auto recording = HostDevice::LockRecording();
+  return std::exchange(g_draw_timing, {});
+}
+
 void ResetDrawResources() {
   std::lock_guard lock(g_pipeline_mutex);
   g_pipelines.clear();
+  g_seen_tiled_pass = false;
 }
 
 }  // namespace legodimensions::gpu_native
 
 REX_HOOK(sub_83FC6640, legodimensions::gpu_native::DrawVerticesHook);
 REX_HOOK(sub_83FC6A58, legodimensions::gpu_native::DrawIndexedVerticesHook);
-REX_HOOK(sub_83FBF0D8, legodimensions::gpu_native::ResolveHook);
+REX_HOOK_RAW(sub_83FBF0D8) {
+  ctx.r3.u64 = legodimensions::gpu_native::ResolveHook(ctx.r4.u32, ctx.r5.u32,
+      ctx.r6.u32, ctx.r7.u32, ctx.r8.u32, ctx.r9.u32, ctx.r10.u32,
+      ctx.f1.f64, ctx.lr, ctx.r1.u32);
+}
 REX_HOOK(sub_83FB33E0, legodimensions::gpu_native::SwapHook);
 REX_HOOK(sub_83FBC9D8, legodimensions::gpu_native::ClearHook);
+REX_HOOK(sub_83FBCD28, legodimensions::gpu_native::BeginTilingHook);
+REX_HOOK(sub_83FBD098, legodimensions::gpu_native::EndTilingHook);
diff --git a/rexlego/src/gpu_native/draw.h b/rexlego/src/gpu_native/draw.h
index efaed6d..5b406ab 100644
--- a/rexlego/src/gpu_native/draw.h
+++ b/rexlego/src/gpu_native/draw.h
@@ -1,7 +1,18 @@
 #pragma once
 
+#include <array>
+#include <cstdint>
+
 namespace legodimensions::gpu_native {
 
+// Host elapsed time, not GPU timestamps. Serialized by LockRecording.
+struct DrawTiming {
+  uint64_t calls = 0;
+  // Bindings, begin commands, pipeline, constants/textures, vertices, issue, tail.
+  std::array<double, 7> stages_ms{};
+};
+DrawTiming ConsumeDrawTiming();
+
 // Must run before shader/device teardown because cached PSOs refer to both.
 void ResetDrawResources();
 
diff --git a/rexlego/src/gpu_native/format.cpp b/rexlego/src/gpu_native/format.cpp
index 0601344..19d4911 100644
--- a/rexlego/src/gpu_native/format.cpp
+++ b/rexlego/src/gpu_native/format.cpp
@@ -14,10 +14,12 @@ enum class D3DFormat : u32 {
   kA16B16G16R16FAlt = 0x1A2201BF,
   kA8B8G8R8 = 0x1A200186,
   kA8R8G8B8 = 0x18280186,
+  kA8R8G8B8Alt = 0x18280106,
   kX8R8G8B8 = 0x28280086,
   kTT8888 = 0x28280106,
   kTT8888Alt = 0x28280186,
   kTT2101010As16161616 = 0x182801B6,
+  kTT16161616 = 0x1A20AB55,
   kD24FS8 = 0x1A220197,
   kD24S8 = 0x2D200196,
   kR32F = 0x2DA2ABA4,
@@ -49,10 +51,13 @@ plume::RenderFormat ConvertGuestTextureFormat(u32 guest_format) {
       return RF::R16G16B16A16_FLOAT;
     case D3DFormat::kA8B8G8R8:
     case D3DFormat::kA8R8G8B8:
+    case D3DFormat::kA8R8G8B8Alt:
     case D3DFormat::kX8R8G8B8:
     case D3DFormat::kTT8888:
     case D3DFormat::kTT8888Alt:
       return RF::R8G8B8A8_UNORM;
+    case D3DFormat::kTT16161616:
+      return RF::R16G16B16A16_SNORM;
     case D3DFormat::kD24FS8:
     case D3DFormat::kD24S8:
       // LEGO uses stencil shadows, so retaining the stencil plane is required.
@@ -110,6 +115,14 @@ plume::RenderFormat ConvertXenosTextureFormat(u32 xenos_format) {
       return RF::R16G16_FLOAT;
     case TF::k_16_16_16_16_FLOAT:
       return RF::R16G16B16A16_FLOAT;
+    case TF::k_16_16_16_16:
+      return RF::R16G16B16A16_UNORM;
+    case TF::k_16_16:
+      return RF::R16G16_UNORM;
+    case TF::k_16_16_EDRAM:
+      return RF::R16G16_SNORM;
+    case TF::k_16_16_16_16_EDRAM:
+      return RF::R16G16B16A16_SNORM;
     case TF::k_32_FLOAT:
       return RF::R32_FLOAT;
     case TF::k_32_32_FLOAT:
@@ -141,8 +154,11 @@ bool IsRenderTargetFormat(plume::RenderFormat format) {
     case RF::B8G8R8A8_UNORM:
     case RF::R16_FLOAT:
     case RF::R16G16_FLOAT:
+    case RF::R16G16_UNORM:
     case RF::R16G16B16A16_FLOAT:
     case RF::R16G16B16A16_UNORM:
+    case RF::R16G16B16A16_SNORM:
+    case RF::R16G16_SNORM:
     case RF::R32_FLOAT:
     case RF::R32G32_FLOAT:
     case RF::R32G32B32A32_FLOAT:
diff --git a/rexlego/src/gpu_native/hooks_buffer.cpp b/rexlego/src/gpu_native/hooks_buffer.cpp
index 77fd488..ec5a91b 100644
--- a/rexlego/src/gpu_native/hooks_buffer.cpp
+++ b/rexlego/src/gpu_native/hooks_buffer.cpp
@@ -1,15 +1,73 @@
-#include <rex/hook.h>
+#include "gpu_native/renderer_route.h"
 #include <rex/ppc/context.h>
 #include <rex/types.h>
 
+#include <atomic>
+#include <algorithm>
+#include <cstring>
+#include <cstdlib>
+#include <xxhash.h>
+
+#include <rex/logging.h>
+#include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
+
 #include "gpu_native/buffers.h"
 #include "gpu_native/textures.h"
+#include "gpu_native/state.h"
+#include "gpu_native/device.h"
+#include "gpu_native/shaders.h"
+#include "gpu_native/memory_watch.h"
 
 extern "C" void __imp__sub_83FC49E8(PPCContext& __restrict ctx, u8* base);
+extern "C" void __imp__sub_82BD2868(PPCContext& __restrict ctx, u8* base);
 
 namespace legodimensions::gpu_native {
 namespace {
 
+thread_local PoolCopyDraws* g_cpu_pool_copy = nullptr;
+
+bool PhysicalRange(u32 address, u32 size, PhysicalCopyRange& result) {
+  auto* memory = REX_KERNEL_MEMORY();
+  const u32 physical = address < 0x20000000u ? address : memory->GetPhysicalAddress(address);
+  if (!size || physical == UINT32_MAX || u64(physical) + size > 0x20000000ull ||
+      u64(address) + size > 0x100000000ull) return false;
+  // Do not cross a virtual heap/alias boundary while assuming linear mapping.
+  const u32 last = address < 0x20000000u ? address + size - 1 :
+      memory->GetPhysicalAddress(address + size - 1);
+  if (last != physical + size - 1) return false;
+  result = {physical, size};
+  return true;
+}
+
+bool AccessiblePhysicalRange(PhysicalCopyRange range, bool write) {
+  auto* heap = REX_KERNEL_MEMORY()->GetPhysicalHeap();
+  for (u64 at = range.address; at < u64(range.address) + range.length;) {
+    rex::memory::HeapAllocationInfo info{};
+    if (!heap->QueryRegionInfo(u32(at), &info) ||
+        !(info.state & rex::memory::kMemoryAllocationCommit) ||
+        !(info.protect & rex::memory::kMemoryProtectRead) ||
+        (write && !(info.protect & (rex::memory::kMemoryProtectWrite |
+                                   rex::memory::kMemoryProtectWriteCombine)))) return false;
+    const u64 page = heap->heap_base() +
+        ((at - heap->heap_base()) / heap->page_size()) * heap->page_size();
+    const u64 next = std::min(page + info.region_size,
+                             u64(heap->heap_base()) + heap->heap_size());
+    if (next <= at) return false;
+    at = next;
+  }
+  return true;
+}
+
+struct PoolCopyScope {
+  PoolCopyDraws draws;
+  PoolCopyDraws* previous;
+  explicit PoolCopyScope(u32 bytes) : draws{bytes / 64, 0}, previous(g_cpu_pool_copy) {
+    g_cpu_pool_copy = &draws;
+  }
+  ~PoolCopyScope() { g_cpu_pool_copy = previous; }
+};
+
 u32 CreateVertexBufferHook(u32 length, u32 usage, u32 fvf, u32 pool) {
   return CreateBufferResource(length, usage, fvf, pool, BufferKind::kVertex);
 }
@@ -26,13 +84,100 @@ u32 LockIndexBufferHook(u32 buffer, u32 offset, u32 size, u32 flags) {
   return LockBufferResource(buffer, offset, size, flags, BufferKind::kIndex);
 }
 
+void BeginExportHook(u32 /*device*/, u32 index, u32 resource, u32 format) {
+  BeginExportBinding(index, resource, format);
+  if (g_cpu_pool_copy) return;
+  static std::atomic<u32> warning_count{0};
+  if (warning_count.fetch_add(1, std::memory_order_relaxed) < 8) {
+    REXLOG_WARN(
+        "Native GPU: captured memexport begin index={} resource=0x{:08X} "
+        "format={} (GPU writeback pending implementation)",
+        index, resource, format);
+  }
+}
+
+void EndExportHook(u32 /*device*/, u32 index, u32 resource, u32 format) {
+  EndExportBinding(index, resource, format);
+}
+
 }  // namespace
+
+bool GuestRangeOverlapsPoolCopy(u32 address, u32 size, PhysicalCopyRange copy) {
+  PhysicalCopyRange resource;
+  return PhysicalRange(address, size, resource) && CopyRangesOverlap(resource, copy);
+}
+
+bool ConsumeCpuPoolCopyDraw(u32 primitive, u32 start, u32 count) {
+  return g_cpu_pool_copy && g_cpu_pool_copy->Consume(primitive, start, count);
+}
+
+void PoolCopyHook(PPCContext& ctx, u8* base) {
+  auto recording = HostDevice::LockRecording();
+  const u32 destination = ctx.r3.u32, source = ctx.r4.u32, bytes = ctx.r5.u32;
+  const u32 caller = ctx.lr == 0x82BD2B44 ?
+      u32(*REX_KERNEL_MEMORY()->TranslateVirtual<const be_u32*>(ctx.r1.u32 + 152)) : u32(ctx.lr);
+  PhysicalCopyRange src, dst;
+  // 82BD2AA0 supplies already rounded chunks and traverses them backwards.
+  // Keep the original helper's headers, constants, ring advancement, bindings
+  // and final stream unbind; replace only its POINTLIST copy work.
+  const bool accessible = bytes && !(bytes & 63) &&
+      PhysicalRange(source, bytes, src) && PhysicalRange(destination, bytes, dst) &&
+      AccessiblePhysicalRange(src, false) && AccessiblePhysicalRange(dst, true);
+  const bool gpu_owned = accessible &&
+      (PoolCopyHasHostTextureSource(src) || PoolCopyHasHostTextureSource(dst));
+  if (!accessible || gpu_owned) {
+    REXLOG_ERROR("Native GPU: pool copy unsupported dst={:08X} src={:08X} bytes={} "
+                 "lr={:08X} accessible={} overlaps_host_resolve={}",
+                 destination, source, bytes, ctx.lr, accessible, gpu_owned);
+    __imp__sub_82BD2868(ctx, base);
+    return;
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  static const bool verify_copy = [] {
+    char* value = nullptr;
+    size_t length = 0;
+    _dupenv_s(&value, &length, "LEGO_DUMP_MISSING_SHADERS");
+    const bool enabled = value && length > 1;
+    std::free(value);
+    return enabled;
+  }();
+  const u64 source_hash = verify_copy ?
+      XXH3_64bits(memory->TranslatePhysical(src.address), bytes) : 0;
+  // Canonical physical pointers make memmove safe even across A/C/E aliases
+  // whose separate host mappings would hide an overlapping physical range.
+  InvalidateCpuPhysicalMemory(dst.address, bytes);
+  std::memmove(memory->TranslatePhysical(dst.address),
+               memory->TranslatePhysical(src.address), bytes);
+  if (verify_copy && source_hash != XXH3_64bits(memory->TranslatePhysical(dst.address), bytes))
+    REXLOG_ERROR("Native GPU: pool copy byte verification failed dst={:08X} src={:08X} bytes={}",
+                 destination, source, bytes);
+  const u32 buffers = InvalidatePoolCopyBuffers(dst);
+  const u32 textures = InvalidatePoolCopyTextures(dst);
+  const u32 shaders = InvalidatePoolCopyShaders(dst);
+  static std::atomic<u32> logs{0};
+  if (logs.fetch_add(1, std::memory_order_relaxed) < 128)
+    REXLOG_INFO("Native GPU: CPU pool copy dst={:08X} src={:08X} bytes={} "
+        "physical={:08X}<-{:08X} lr={:08X} caller={:08X} VS={:016X} verify={} invalidated VB/IB={} textures={} shaders={}",
+        destination, source, bytes, dst.address, src.address, ctx.lr, caller,
+        BoundShaderHash(ShaderStage::kVertex), verify_copy, buffers, textures, shaders);
+  PoolCopyScope scope(bytes);
+  __imp__sub_82BD2868(ctx, base);
+  if (scope.draws.consumed != scope.draws.records)
+    REXLOG_ERROR("Native GPU: pool copy POINTLIST contract mismatch consumed={} expected={}",
+                 scope.draws.consumed, scope.draws.records);
+}
 }  // namespace legodimensions::gpu_native
 
+REX_HOOK_RAW(sub_82BD2868) {
+  legodimensions::gpu_native::PoolCopyHook(ctx, base);
+}
+
 REX_HOOK(sub_83FC4D58, legodimensions::gpu_native::CreateVertexBufferHook);
 REX_HOOK(sub_83FC4E30, legodimensions::gpu_native::CreateIndexBufferHook);
 REX_HOOK(sub_83FC60D8, legodimensions::gpu_native::LockVertexBufferHook);
 REX_HOOK(sub_83FC6128, legodimensions::gpu_native::LockIndexBufferHook);
+REX_HOOK(sub_83FC53D0, legodimensions::gpu_native::BeginExportHook);
+REX_HOOK(sub_83FC54C8, legodimensions::gpu_native::EndExportHook);
 
 REX_HOOK_RAW(sub_83FC49E8) {
   const u32 resource = ctx.r3.u32;
@@ -44,4 +189,8 @@ REX_HOOK_RAW(sub_83FC49E8) {
     return;
   }
   __imp__sub_83FC49E8(ctx, base);
+  // This caller controls which header field XGOffsetResource relocates.
+  if (ctx.lr == 0x83F9B660 && legodimensions::gpu_native::IsNativeBuffer(resource))
+    REXLOG_INFO("Native GPU: placement buffer relocation type resource={:08X} type={} lr={:08X}",
+                resource, ctx.r3.u32, ctx.lr);
 }
diff --git a/rexlego/src/gpu_native/hooks_device.cpp b/rexlego/src/gpu_native/hooks_device.cpp
index bddf2ed..38b77e5 100644
--- a/rexlego/src/gpu_native/hooks_device.cpp
+++ b/rexlego/src/gpu_native/hooks_device.cpp
@@ -1,12 +1,21 @@
+#include <algorithm>
+#include <array>
 #include <cstring>
+#include <cstdlib>
+#include <unordered_set>
 
-#include <rex/hook.h>
+#include "gpu_native/renderer_route.h"
+#include <rex/ppc.h>
+#include <rex/system/function_dispatcher.h>
 #include <rex/system/kernel_state.h>
 #include <rex/types.h>
 
 #include "gpu_native/d3d.h"
 #include "gpu_native/device.h"
 #include "gpu_native/state.h"
+#include "gpu_native/buffers.h"
+#include "gpu_native/textures.h"
+#include "gpu_native/shaders.h"
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -27,6 +36,91 @@ void CopyStateDispatch(D3DDevice* device, u8* membase) {
   }
 }
 
+bool InitializeRenderDefaults(u32 guest_address, u8* membase) {
+  // Only statically audited leaf alpha/blend/write-mask setters. Use the actual
+  // TU23 descriptor defaults, not guessed reset values or full queue init.
+  constexpr u32 indices[] = {15,16,18,19,20,21,22,23,24,26,53,54,55,56,59};
+  constexpr u32 setters[] = {0x83FB7E38,0x83FB81C8,0x83FB7F58,0x83FB7FE8,
+      0x83FB7EC8,0x83FB80E8,0x83FB8158,0x83FB8078,0x83FB7E00,0x83FB82C0,
+      0x83FB8A68,0x83FB8AA8,0x83FB8AE8,0x83FB8B28,0x83FB92B8};
+  const auto* table = reinterpret_cast<const be_u32*>(membase + kRenderStateTable);
+  auto* kernel_state = REX_KERNEL_STATE();
+  auto* dispatcher = kernel_state ? kernel_state->function_dispatcher() : nullptr;
+  if (!dispatcher) return false;
+  for (u32 i = 0; i < 15; ++i) {
+    const u32 address = table[indices[i] * 3 + 1];
+    if (address != setters[i] || !dispatcher->GetFunction(address)) {
+      REXLOG_ERROR("Native GPU: unexpected render default setter {:08X}", address);
+      return false;
+    }
+  }
+  for (u32 i = 0; i < 15; ++i) {
+    const u32 address = table[indices[i] * 3 + 1];
+    const u32 value = table[indices[i] * 3 + 2];
+    rex::ppc::GuestToHostFunction<void>(dispatcher->GetFunction(address), guest_address, value);
+    REXLOG_INFO("Native GPU: render default setter={:08X} raw={:08X}", address, value);
+  }
+  return true;
+}
+
+bool InitializeSamplerDefaults(D3DDevice* device, u32 guest_address, u8* membase) {
+  // Only these statically verified leaf setters are safe on the detached
+  // device. Never enter full 83FC9450, which also initializes Xbox queues.
+  constexpr std::array<u32, kSamplerStateCount> allowed{
+      0x83FB9DB8, 0x83FB9E08, 0x83FB9E58, 0x83FB97E8, 0x83FB9640,
+      0x83FB9988, 0x83FB9A88, 0x83FB9BA8, 0x83FB9C48, 0x83FB9CC8,
+      0x83FB9B00, 0x83FB9D48, 0x83FB9EA8, 0x83FB9FB0, 0x83FB9F00,
+      0x83FB9F58, 0x83FBA008, 0x83FB98E8, 0x83FB9740, 0x83FB99E0};
+  auto* kernel = rex::system::kernel_state();
+  auto* dispatcher = kernel ? kernel->function_dispatcher() : nullptr;
+  if (!dispatcher) return false;
+  const auto* table = reinterpret_cast<const be_u32*>(membase + kSamplerStateTable);
+  // Check the entire table before executing any entry, including game data.
+  for (u32 k = 0; k < kSamplerStateCount; ++k) {
+    const u32 address = table[k * 3 + 1];
+    if (std::find(allowed.begin(), allowed.end(), address) == allowed.end() ||
+        !dispatcher->GetFunction(address)) {
+      REXLOG_ERROR("Native GPU: unsafe sampler initializer k={} setter={:08X}", k, address);
+      return false;
+    }
+    REXLOG_INFO("Native sampler default k={} setter={:08X} value={:08X}",
+                k, address, u32(table[k * 3 + 2]));
+  }
+  // Sampler-major order matches the original. Min/max mip requests are saved
+  // even with no bound texture, then applied by SetNativeTexture on a bind.
+  for (u32 s = 0; s < kFetchConstantCount; ++s) {
+    for (u32 k = 0; k < kSamplerStateCount; ++k) {
+      auto* setter = dispatcher->GetFunction(table[k * 3 + 1]);
+      rex::ppc::GuestToHostFunction<void>(setter, guest_address, s, u32(table[k * 3 + 2]));
+    }
+    SetNativeTexture(device, s, 0, u64{1} << (31 - s));
+  }
+  REXLOG_INFO("Native GPU: initialized {} sampler fetch shadows", kFetchConstantCount);
+  return true;
+}
+
+bool InitializeStencilDefaults(u32 guest_address, u8* membase) {
+  // TU23 descriptors 27..42 are CPU-only leaves. Do not enter queue init.
+  constexpr u32 setters[] = {0x83FB8570,0x83FB85B0,0x83FB8618,0x83FB8650,
+      0x83FB8688,0x83FB85E8,0x83FB8788,0x83FB87A8,0x83FB87C8,0x83FB86E8,
+      0x83FB8720,0x83FB8758,0x83FB86B8,0x83FB87E8,0x83FB8808,0x83FB8828};
+  const auto* table = reinterpret_cast<const be_u32*>(membase + kRenderStateTable);
+  auto* kernel = REX_KERNEL_STATE();
+  auto* dispatcher = kernel ? kernel->function_dispatcher() : nullptr;
+  if (!dispatcher) return false;
+  for (u32 i = 0; i < 16; ++i) {
+    const u32 address = table[(27 + i) * 3 + 1];
+    if (address != setters[i] || !dispatcher->GetFunction(address)) return false;
+  }
+  for (u32 i = 0; i < 16; ++i) {
+    const u32 address = table[(27 + i) * 3 + 1];
+    rex::ppc::GuestToHostFunction<void>(dispatcher->GetFunction(address), guest_address,
+                                     u32(table[(27 + i) * 3 + 2]));
+  }
+  REXLOG_INFO("Native GPU: initialized isolated stencil candidate defaults");
+  return true;
+}
+
 u32 Direct3DCreateDeviceHook(u32 /*adapter*/, u32 /*device_type*/, u32 /*focus_window*/,
                              u32 /*behavior_flags*/, u32 /*present_params*/,
                              mapped_u32 out_device) {
@@ -45,6 +139,20 @@ u32 Direct3DCreateDeviceHook(u32 /*adapter*/, u32 /*device_type*/, u32 /*focus_w
 
   auto* device = reinterpret_cast<D3DDevice*>(memory->virtual_membase() + guest_address);
   CopyStateDispatch(device, memory->virtual_membase());
+  if (std::getenv("LEGO_NATIVE_VIEWPORT")) {
+    // CPU register defaults from TU23 83FCA320; 83FB8FD0 subsequently toggles
+    // the viewport enable bits and CLIP_DISABLE without entering GPU init.
+    auto* bytes = reinterpret_cast<u8*>(device);
+    *reinterpret_cast<be_u32*>(bytes + 10564) = 1u << 19;
+    *reinterpret_cast<be_u32*>(bytes + 10572) = 0x43F;
+  }
+  if (!InitializeRenderDefaults(guest_address, memory->virtual_membase()) ||
+      !InitializeSamplerDefaults(device, guest_address, memory->virtual_membase()) ||
+      (std::getenv("LEGO_NATIVE_STENCIL") &&
+       !InitializeStencilDefaults(guest_address, memory->virtual_membase()))) {
+    memory->SystemHeapFree(guest_address);
+    return 0x80004005u;
+  }
   device->viewport.width = 1280;
   device->viewport.height = 720;
   device->viewport.max_z = 1.0f;
@@ -54,7 +162,116 @@ u32 Direct3DCreateDeviceHook(u32 /*adapter*/, u32 /*device_type*/, u32 /*focus_w
   return 0;
 }
 
+// The Xenos implementation dirties the emulated ring/device globals. Native
+// buffers are CPU-visible mirrors and D3D12 barriers are recorded explicitly,
+// so there is no guest GPU cache to invalidate.
+void InvalidateGpuCacheHook(D3DDevice* /*device*/, u32 /*address*/,
+                            u32 /*size*/, u32 /*flags*/) {}
+
+// Register allocation is an Xenos microcode scheduling knob. AOT DXIL has
+// already gone through the desktop compiler's register allocation.
+void SetShaderGprAllocationHook(D3DDevice* /*device*/, u32 /*vertex_gprs*/,
+                                u32 /*pixel_gprs*/) {}
+
+// Presentation pacing is owned by the host swap chain. The original helper
+// writes to the Xenos ring globals that do not exist in detached-native mode.
+void SynchronizeToPresentationIntervalHook(D3DDevice* /*device*/,
+                                           u32 /*interval*/) {}
+
+void BlockUntilIdleHook(D3DDevice* /*device*/) {
+  HostDevice::Synchronize(SyncReason::kIdle);
+  HostDevice::PollCompletionCallbacks();
+}
+
+// TU23 83FBF8A0 takes the fence marker in r3, NOT a device pointer.
+// Both original wrappers fetch an Xbox-only global device before reaching
+// CBlocker. A full host drain is conservative (waits for more work), but keeps
+// ordering without reading that nonexistent ring or claiming immediate completion.
+void BlockOnFenceHook(u32 /*fence_marker*/) {
+  if (!HostDevice::Synchronize(SyncReason::kFence))
+    REXLOG_ERROR("Native GPU: BlockOnFence failed; host device unavailable");
+  HostDevice::PollCompletionCallbacks();
+}
+
+void BlockUntilNotBusyHook(u32 resource) {
+  auto recording = HostDevice::LockRecording();
+  // Opt-in until the controlled cutscene/title test passes. Native uploads
+  // copy CPU data into GPU-owned storage and retain that storage at fences.
+  // Updating those guest bytes cannot overwrite an earlier GPU draw.
+  static const bool asynchronous_cpu = std::getenv("LEGO_NATIVE_ASYNC_CPU_RESOURCES") != nullptr;
+  if (asynchronous_cpu && (IsNativeBuffer(resource) || IsNativeShader(resource) ||
+                           IsCpuUploadedTexture(resource))) {
+    HostDevice::RecordCpuResourceWaitSkip();
+    return;
+  }
+  // Unknown resources and host-resolved textures keep the full wait.
+  if (!HostDevice::Synchronize(SyncReason::kResource))
+    REXLOG_ERROR("Native GPU: BlockUntilNotBusy failed; host device unavailable");
+  recording.unlock();
+  HostDevice::PollCompletionCallbacks();
+}
+
+void KickOffSegmentHook(D3DDevice* /*device*/) {
+  HostDevice::SubmitRecordedWork();
+  HostDevice::PollCompletionCallbacks();
+}
+
+u32 KickOffHook(D3DDevice* device) {
+  HostDevice::SubmitRecordedWork();
+  HostDevice::PollCompletionCallbacks();
+  if (!device) {
+    return 0;
+  }
+  // Callers store the returned ring marker before comparing it later. Give
+  // them a stable, device-owned scratch marker while Plume owns the real
+  // submission fence.
+  auto* base = REX_KERNEL_MEMORY()->virtual_membase();
+  return static_cast<u32>(reinterpret_cast<u8*>(device) - base) + 0x6000;
+}
+
+// Return without waiting, like Xenos. All callbacks (including profiler
+// markers) run after their containing submission completes, on a guest hook
+// thread. No early timestamp shortcut or worker-thread guest invocation.
+void InsertCallbackHook(D3DDevice* /*device*/, u32 flags,
+                        u32 callback, u32 callback_data) {
+  auto recording = HostDevice::LockRecording();
+  if (!callback) {
+    return;
+  }
+  auto* kernel = rex::system::kernel_state();
+  auto* dispatcher = kernel ? kernel->function_dispatcher() : nullptr;
+  auto* function = dispatcher ? dispatcher->GetFunction(callback) : nullptr;
+  if (!function) {
+    REXLOG_ERROR("Native GPU: callback 0x{:08X} is not recompiled", callback);
+    return;
+  }
+  static std::unordered_set<u32> logged_callbacks;
+  if (logged_callbacks.size() < 32 && logged_callbacks.insert(callback).second)
+    REXLOG_INFO("Native GPU: deferred callback={:08X} context={:08X} flags={:08X}",
+                callback, callback_data, flags);
+  if (!HostDevice::EnqueueCompletionCallback([function, callback_data] {
+        rex::ppc::GuestToHostFunction<void>(function, callback_data);
+      })) {
+    REXLOG_WARN("Native GPU: callback 0x{:08X} skipped; device is unavailable", callback);
+    return;
+  }
+  recording.unlock();
+  HostDevice::PollCompletionCallbacks();
+}
+
 }  // namespace
 }  // namespace legodimensions::gpu_native
 
 REX_HOOK(sub_83FAF9A0, legodimensions::gpu_native::Direct3DCreateDeviceHook);
+REX_HOOK(sub_83FC5A58,
+         legodimensions::gpu_native::InvalidateGpuCacheHook);
+REX_HOOK(sub_83FB6DD0,
+         legodimensions::gpu_native::SetShaderGprAllocationHook);
+REX_HOOK(sub_83FB2F20,
+         legodimensions::gpu_native::SynchronizeToPresentationIntervalHook);
+REX_HOOK(sub_83FC0EC0, legodimensions::gpu_native::BlockUntilIdleHook);
+REX_HOOK(sub_83FBF8A0, legodimensions::gpu_native::BlockOnFenceHook);
+REX_HOOK(sub_83FBF900, legodimensions::gpu_native::BlockUntilNotBusyHook);
+REX_HOOK(sub_83FC08D0, legodimensions::gpu_native::KickOffSegmentHook);
+REX_HOOK(sub_83FC0C10, legodimensions::gpu_native::KickOffHook);
+REX_HOOK(sub_83FC0F38, legodimensions::gpu_native::InsertCallbackHook);
diff --git a/rexlego/src/gpu_native/hooks_shader.cpp b/rexlego/src/gpu_native/hooks_shader.cpp
index a5df44f..15a2e53 100644
--- a/rexlego/src/gpu_native/hooks_shader.cpp
+++ b/rexlego/src/gpu_native/hooks_shader.cpp
@@ -1,4 +1,4 @@
-#include <rex/hook.h>
+#include "gpu_native/renderer_route.h"
 #include <rex/ppc/context.h>
 #include <rex/types.h>
 
@@ -10,6 +10,8 @@
 
 extern "C" void __imp__sub_83FC4CE0(PPCContext& __restrict ctx, u8* base);
 extern "C" void __imp__sub_83FC59D8(PPCContext& __restrict ctx, u8* base);
+extern "C" void __imp__sub_83FB7658(PPCContext& __restrict ctx, u8* base);
+extern "C" void __imp__sub_83FB67F0(PPCContext& __restrict ctx, u8* base);
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -44,6 +46,16 @@ REX_HOOK(sub_83FB7750, legodimensions::gpu_native::CreateVertexShaderHook);
 REX_HOOK(sub_83FB6828, legodimensions::gpu_native::SetPixelShaderHook);
 REX_HOOK(sub_83FB6A30, legodimensions::gpu_native::SetVertexShaderHook);
 
+REX_HOOK_RAW(sub_83FB7658) {
+  legodimensions::gpu_native::InvalidatePlacementShader(ctx.r3.u32);
+  __imp__sub_83FB7658(ctx, base);
+}
+
+REX_HOOK_RAW(sub_83FB67F0) {
+  legodimensions::gpu_native::InvalidatePlacementShader(ctx.r3.u32);
+  __imp__sub_83FB67F0(ctx, base);
+}
+
 // Resource AddRef/Release serve all D3D resource types. Only intercept objects
 // created by the native shader hooks; every other pointer retains the exact
 // Sep'13 implementation.
diff --git a/rexlego/src/gpu_native/hooks_state.cpp b/rexlego/src/gpu_native/hooks_state.cpp
index b90239f..efcdeb2 100644
--- a/rexlego/src/gpu_native/hooks_state.cpp
+++ b/rexlego/src/gpu_native/hooks_state.cpp
@@ -1,21 +1,44 @@
-#include <rex/hook.h>
+#include "gpu_native/renderer_route.h"
+#include <rex/system/kernel_state.h>
 #include <rex/types.h>
 
 #include "gpu_native/d3d.h"
 #include "gpu_native/state.h"
 #include "gpu_native/textures.h"
 #include "gpu_native/vertex_declarations.h"
+#include "gpu_native/sampler_state.h"
+#include "gpu_native/depth_state.h"
+#include "gpu_native/device.h"
+#include <rex/logging.h>
 
-namespace legodimensions::gpu_native {
-namespace {
+extern "C" void __imp__sub_83FBB110(PPCContext& __restrict ctx, u8* base);
 
-void SetTextureHook(D3DDevice* device, u32 sampler, u32 texture) {
-  if (device && sampler < kNativeTextureSlots) {
-    device->textures[sampler] = texture;
+namespace legodimensions::gpu_native {
+void SetNativeTexture(D3DDevice* device, u32 sampler, u32 texture, u64 dirty) {
+  if (!device || sampler >= kNativeTextureSlots) return;
+  auto* base = REX_KERNEL_MEMORY()->virtual_membase();
+  TextureFetchWords words;
+  for (u32 i = 0; i < words.size(); ++i) words[i] = device->fetch_constants[sampler].dword[i];
+  if (texture) {
+    const auto* header = reinterpret_cast<const D3DTexture*>(base + texture);
+    TextureFetchWords texture_words;
+    for (u32 i = 0; i < texture_words.size(); ++i) texture_words[i] = header->format.dword[i];
+    const auto* bytes = reinterpret_cast<const u8*>(device);
+    words = MergeTextureFetch(words, texture_words, bytes[12356 + sampler], bytes[12382 + sampler]);
+    RefreshTextureHeader(texture);
+    auto* dirty_mask = reinterpret_cast<be_u64*>(reinterpret_cast<u8*>(device) + 24);
+    *dirty_mask = u64(*dirty_mask) | dirty;
+  } else {
+    // Original NULL branch clears only type and binding, keeping sampler bits.
+    ClearTextureFetch(words);
   }
+  for (u32 i = 0; i < words.size(); ++i) device->fetch_constants[sampler].dword[i] = words[i];
+  device->textures[sampler] = texture;
   BindTexture(sampler, texture);
 }
 
+namespace {
+
 void SetStreamSourceHook(D3DDevice* device, u32 stream, u32 buffer,
                          u32 offset, u32 stride) {
   if (device && stream < 18) {
@@ -52,6 +75,11 @@ void SetRenderTargetHook(D3DDevice* device, u32 index, u32 surface) {
 void SetDepthStencilHook(D3DDevice* device, u32 surface) {
   if (device) {
     device->depth_stencil = surface;
+    auto* bytes = reinterpret_cast<u8*>(device);
+    auto* control = reinterpret_cast<be_u32*>(bytes + 10548);
+    *control = RebindDepthStencilControl(*control,
+        *reinterpret_cast<const be_u32*>(bytes + 12308),
+        *reinterpret_cast<const be_u32*>(bytes + 12312), surface != 0);
   }
   BindDepthStencil(surface);
 }
@@ -84,19 +112,46 @@ void SetScissorRectHook(D3DDevice* device, const D3DRect* rect) {
   }
 }
 
+// The native backend does not execute Xbox query predicates yet. Entering the
+// original XDK routine is unsafe because it dereferences private command-buffer
+// arrays that do not exist in our deliberately minimal guest D3DDevice.
+void SetPredicationHook(D3DDevice* /*device*/, u32 /*predicate*/) {}
+
+// Emits an Xbox command-buffer format/HiZ packet in the Sep'13 runtime. Native
+// render-target transitions own this responsibility on the host backend.
+void SetupFormatHook(D3DDevice* /*device*/, u32 /*mode*/) {}
+
 }  // namespace
 }  // namespace legodimensions::gpu_native
 
-REX_HOOK(sub_83FB58A8, legodimensions::gpu_native::SetTextureHook);
+REX_HOOK(sub_83FB58A8, legodimensions::gpu_native::SetNativeTexture);
 REX_HOOK(sub_83FBA160, legodimensions::gpu_native::SetStreamSourceHook);
 REX_HOOK(sub_83FBA308, legodimensions::gpu_native::SetIndicesHook);
 REX_HOOK(sub_83FB6D50,
          legodimensions::gpu_native::CreateVertexDeclarationHook);
-REX_HOOK(sub_83FB7840,
+REX_HOOK(sub_83FB6C48,
          legodimensions::gpu_native::SetVertexDeclarationHook);
 REX_HOOK(sub_83FBAAA8, legodimensions::gpu_native::SetRenderTargetHook);
 REX_HOOK(sub_83FBAE38, legodimensions::gpu_native::SetDepthStencilHook);
+// SetRenderTargets is a separate bulk setter, not a caller of the individual
+// setters above. Preserve its XDK shadow/viewport updates, then publish all
+// five attachments to the native renderer. Save r3 before the guest call.
+REX_HOOK_RAW(sub_83FBB110) {
+  auto recording = legodimensions::gpu_native::HostDevice::LockRecording();
+  auto* device = reinterpret_cast<legodimensions::gpu_native::D3DDevice*>(base + ctx.r3.u32);
+  __imp__sub_83FBB110(ctx, base);
+  for (u32 i = 0; i < legodimensions::gpu_native::kNativeRenderTargets; ++i)
+    legodimensions::gpu_native::BindRenderTarget(i, device->render_targets[i]);
+  legodimensions::gpu_native::BindDepthStencil(device->depth_stencil);
+  static u32 logs = 0;
+  if (logs++ < 12)
+    REXLOG_INFO("Native GPU: bulk attachments rt={:08X},{:08X},{:08X},{:08X} ds={:08X}",
+        u32(device->render_targets[0]), u32(device->render_targets[1]),
+        u32(device->render_targets[2]), u32(device->render_targets[3]), u32(device->depth_stencil));
+}
 REX_HOOK(sub_83FBA398, legodimensions::gpu_native::GetRenderTargetHook);
 REX_HOOK(sub_83FBA3E0, legodimensions::gpu_native::GetDepthStencilHook);
 REX_HOOK(sub_83FBA978, legodimensions::gpu_native::SetViewportHook);
 REX_HOOK(sub_83FBA068, legodimensions::gpu_native::SetScissorRectHook);
+REX_HOOK(sub_83FBCBB8, legodimensions::gpu_native::SetPredicationHook);
+REX_HOOK(sub_83FAF5D8, legodimensions::gpu_native::SetupFormatHook);
diff --git a/rexlego/src/gpu_native/hooks_texture.cpp b/rexlego/src/gpu_native/hooks_texture.cpp
index 2b74d1f..405a3f4 100644
--- a/rexlego/src/gpu_native/hooks_texture.cpp
+++ b/rexlego/src/gpu_native/hooks_texture.cpp
@@ -1,4 +1,4 @@
-#include <rex/hook.h>
+#include "gpu_native/renderer_route.h"
 #include <rex/ppc/context.h>
 #include <rex/types.h>
 
diff --git a/rexlego/src/gpu_native/linked_shader_cache.h b/rexlego/src/gpu_native/linked_shader_cache.h
index a3806d7..d5b3407 100644
--- a/rexlego/src/gpu_native/linked_shader_cache.h
+++ b/rexlego/src/gpu_native/linked_shader_cache.h
@@ -29,3 +29,9 @@ extern const size_t g_linkedShaderCacheEntryCount;
 extern const uint8_t g_compressedLinkedDxilCache[];
 extern const size_t g_linkedDxilCacheCompressedSize;
 extern const size_t g_linkedDxilCacheDecompressedSize;
+
+extern LinkedShaderCacheEntry g_runtimeLinkedShaderCacheEntries[];
+extern const size_t g_runtimeLinkedShaderCacheEntryCount;
+extern const uint8_t g_runtimeCompressedLinkedDxilCache[];
+extern const size_t g_runtimeLinkedDxilCacheCompressedSize;
+extern const size_t g_runtimeLinkedDxilCacheDecompressedSize;
diff --git a/rexlego/src/gpu_native/long_probe.h b/rexlego/src/gpu_native/long_probe.h
new file mode 100644
index 0000000..53764bb
--- /dev/null
+++ b/rexlego/src/gpu_native/long_probe.h
@@ -0,0 +1,89 @@
+// Opt-in long-run diagnostics. No guest or rendering state is modified here.
+#pragma once
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iomanip>
+#include <mutex>
+#include <sstream>
+#include <span>
+#include <string>
+#include <string_view>
+#include <unordered_set>
+
+namespace legodimensions::gpu_native {
+inline const std::filesystem::path& LongProbeDirectory() {
+  static const std::filesystem::path path = [] {
+    char* value = nullptr; size_t length = 0;
+    _dupenv_s(&value, &length, "LEGO_NATIVE_TRACE_DIR");
+    std::filesystem::path result(value ? value : "");
+    std::free(value); return result;
+  }();
+  return path;
+}
+inline bool LongProbeEnabled() { return !LongProbeDirectory().empty(); }
+inline std::atomic<uint32_t> g_probe_frame{0};
+inline std::atomic<bool> g_probe_capture_requested{false};
+inline double LongProbeMilliseconds() {
+  static const auto start = std::chrono::steady_clock::now();
+  return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now()-start).count();
+}
+inline void WriteLongProbeEvent(std::string_view kind, bool anomaly, std::string_view payload);
+template<class... T>
+inline void LongProbeEvent(std::string_view kind, bool anomaly, const T&... values) {
+  if (!LongProbeEnabled()) return;
+  std::ostringstream payload;
+  ((payload << values << ' '), ...);
+  WriteLongProbeEvent(kind, anomaly, payload.str());
+}
+inline void WriteLongProbeEvent(std::string_view kind, bool anomaly, std::string_view payload) {
+  struct Writer { std::mutex mutex; std::ofstream file; uint64_t rows=0;
+    Writer() {
+      std::error_code error; std::filesystem::create_directories(LongProbeDirectory(),error);
+      file.open(LongProbeDirectory()/"events.tsv",std::ios::out|std::ios::app);
+      file << "elapsed_ms\tframe\tevent\tanomaly_candidate\tdetails\n";
+      std::ofstream start(LongProbeDirectory()/"trace-start.json");
+      start << "{\"unix_ms\":" << std::chrono::duration_cast<std::chrono::milliseconds>(
+          std::chrono::system_clock::now().time_since_epoch()).count() << "}";
+    }
+  };
+  static Writer writer;
+  std::lock_guard lock(writer.mutex);
+  writer.file << std::fixed << std::setprecision(3) << LongProbeMilliseconds() << '\t'
+      << g_probe_frame.load() << '\t' << kind << '\t' << anomaly << '\t' << payload << '\n';
+  if (anomaly || ++writer.rows % 256 == 0 || kind=="frame") writer.file.flush();
+  if (anomaly) g_probe_capture_requested.store(true);
+}
+inline bool LongProbeOnce(uint64_t key) {
+  static std::mutex mutex; static std::unordered_set<uint64_t> seen;
+  std::lock_guard lock(mutex); return seen.insert(key).second;
+}
+template<class Words> inline std::string LongProbeHex(const Words& words) {
+  std::ostringstream out; out << std::hex << std::setfill('0');
+  for (auto word : words) out << std::setw(8) << uint32_t(word) << ',';
+  return out.str();
+}
+inline std::string SaveLongProbeConstants(std::span<const uint8_t> bytes) {
+  static std::atomic<uint64_t> sequence{0};
+  static std::mutex budget_mutex;
+  static uint32_t budget_frame=~0u,count=0;
+  {
+    std::lock_guard lock(budget_mutex);
+    const auto frame=g_probe_frame.load();
+    if(frame!=budget_frame) { budget_frame=frame; count=0; }
+    if(count++>=128) {
+      if(count==129) LongProbeEvent("constants_frame_budget",false,"frame=",frame,"limit=128");
+      return {};
+    }
+  }
+  const auto name="constants-f"+std::to_string(g_probe_frame.load())+"-"+
+      std::to_string(sequence.fetch_add(1))+".bin";
+  std::error_code error; std::filesystem::create_directories(LongProbeDirectory(),error);
+  std::ofstream file(LongProbeDirectory()/name,std::ios::binary);
+  file.write(reinterpret_cast<const char*>(bytes.data()),bytes.size());
+  return file ? name : "WRITE_FAILED";
+}
+} // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/memory_watch.cpp b/rexlego/src/gpu_native/memory_watch.cpp
new file mode 100644
index 0000000..8cf7034
--- /dev/null
+++ b/rexlego/src/gpu_native/memory_watch.cpp
@@ -0,0 +1,129 @@
+#include "gpu_native/memory_watch.h"
+
+#include <algorithm>
+#include <array>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <rex/cvar.h>
+#include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
+
+REXCVAR_DEFINE_BOOL(gpu_native_texture_watch, false, "GPU",
+    "Reuse unchanged native pixel textures with physical page write invalidation")
+    .lifecycle(rex::cvar::Lifecycle::kRequiresRestart);
+
+namespace legodimensions::gpu_native {
+namespace {
+constexpr uint32_t kPageSize = 4096, kPhysicalSize = 0x20000000;
+// Protected by the SDK global critical region. Fault callbacks never take
+// renderer/resource locks and never call the driver.
+std::array<uint64_t, kPhysicalSize / kPageSize> g_page_versions{};
+rex::thread::global_critical_region g_watch_region;
+rex::memory::Memory* g_memory = nullptr;
+void* g_callback = nullptr;
+
+bool SelectedMemoryWatch() {
+  // Only pixel textures may use this path. Vertex animation textures and
+  // vertex/index buffers retain every-use content checks.
+  static const bool enabled = [] {
+    const char* value = std::getenv("LEGO_NATIVE_STATIC_TEXTURE_WATCH");
+    const bool selected = value ? std::strcmp(value, "0") != 0 :
+        REXCVAR_GET(gpu_native_texture_watch);
+    return selected && !std::getenv("LEGO_NATIVE_NO_MEMORY_WATCH");
+  }();
+  return enabled;
+}
+
+void Invalidate(uint32_t address, uint32_t length) {
+  if (!length || address >= kPhysicalSize) return;
+  const uint64_t end = std::min(uint64_t(address) + length, uint64_t(kPhysicalSize));
+  for (uint32_t page = address / kPageSize; uint64_t(page) * kPageSize < end; ++page)
+    ++g_page_versions[page];
+}
+
+std::pair<uint32_t, uint32_t> MemoryWritten(void*, uint32_t address,
+                                          uint32_t length, bool) {
+  Invalidate(address, length);
+  // Unprotect only the notified pages. Returning a wider range would leave
+  // other stamps falsely clean when the SDK removes their write watches.
+  return {address, length};
+}
+
+bool PhysicalSpan(rex::memory::Memory* memory, CpuMemorySpan input, CpuMemorySpan& output) {
+  if (!input.length || uint64_t(input.address) + input.length > 0x100000000ull) return false;
+  const auto physical = [&](uint32_t at) {
+    return at < kPhysicalSize ? at : memory->GetPhysicalAddress(at);
+  };
+  const uint32_t first = physical(input.address);
+  if (first == UINT32_MAX || uint64_t(first) + input.length > kPhysicalSize ||
+      physical(input.address + input.length - 1) != first + input.length - 1) return false;
+  auto* heap = memory->GetPhysicalHeap();
+  for (uint64_t at = first, end = uint64_t(first) + input.length; at < end;) {
+    rex::memory::HeapAllocationInfo info{};
+    if (!heap->QueryRegionInfo(uint32_t(at), &info) ||
+        !(info.state & rex::memory::kMemoryAllocationCommit) ||
+        !(info.protect & rex::memory::kMemoryProtectRead)) return false;
+    const uint64_t page = heap->heap_base() + ((at - heap->heap_base()) / heap->page_size()) * heap->page_size();
+    const uint64_t next = page + info.region_size;
+    if (next <= at) return false;
+    at = std::min(next, end);
+  }
+  output = {first, input.length};
+  return true;
+}
+}  // namespace
+
+bool CpuMemoryWatchEnabled() { return SelectedMemoryWatch(); }
+
+CpuMemoryStamp WatchCpuMemory(std::span<const CpuMemorySpan> spans) {
+  CpuMemoryStamp stamp;
+  if (!CpuMemoryWatchEnabled()) return stamp;
+  auto lock = g_watch_region.Acquire();
+  auto* memory = REX_KERNEL_MEMORY();
+  if (!memory) return stamp;
+  std::vector<CpuMemorySpan> physical;
+  for (const auto span : spans) {
+    if (!span.length) continue;
+    CpuMemorySpan decoded;
+    if (!PhysicalSpan(memory, span, decoded)) return {};
+    physical.push_back(decoded);
+  }
+  if (physical.empty()) return {};
+  if (!g_callback) {
+    g_memory = memory;
+    g_callback = memory->RegisterPhysicalMemoryInvalidationCallback(MemoryWritten, nullptr);
+  }
+  for (const auto span : physical) {
+    // Arm BEFORE hashing or copying, so an intervening write makes this stamp
+    // unusable rather than caching a mixture as permanently clean.
+    memory->EnablePhysicalMemoryAccessCallbacks(span.address, span.length, true, false);
+    const uint32_t last = (span.address + span.length - 1) / kPageSize;
+    for (uint32_t page = span.address / kPageSize; page <= last; ++page)
+      stamp.pages.emplace_back(page, g_page_versions[page]);
+  }
+  return stamp;
+}
+
+bool CpuMemoryUnchanged(const CpuMemoryStamp& stamp) {
+  if (stamp.pages.empty()) return false;
+  auto lock = g_watch_region.Acquire();
+  for (const auto [page, version] : stamp.pages)
+    if (g_page_versions[page] != version) return false;
+  return true;
+}
+
+void InvalidateCpuPhysicalMemory(uint32_t address, uint32_t length) {
+  auto lock = g_watch_region.Acquire();
+  Invalidate(address, length);
+}
+
+void ShutdownCpuMemoryWatch() {
+  auto lock = g_watch_region.Acquire();
+  if (g_callback) g_memory->UnregisterPhysicalMemoryInvalidationCallback(g_callback);
+  g_callback = nullptr;
+  g_memory = nullptr;
+  // Invalidate stamps even if a new device is created with the same Memory.
+  for (auto& version : g_page_versions) ++version;
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/memory_watch.h b/rexlego/src/gpu_native/memory_watch.h
new file mode 100644
index 0000000..1727ab1
--- /dev/null
+++ b/rexlego/src/gpu_native/memory_watch.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <cstdint>
+#include <span>
+#include <utility>
+#include <vector>
+
+namespace legodimensions::gpu_native {
+
+struct CpuMemorySpan { uint32_t address = 0, length = 0; };
+struct CpuMemoryStamp {
+  std::vector<std::pair<uint32_t, uint64_t>> pages;
+};
+bool CpuMemoryWatchEnabled();
+
+// Uses the SDK's physical write notifications across A/C/E aliases. Unknown
+// virtual storage has no stamp and must still be hashed at each use.
+CpuMemoryStamp WatchCpuMemory(std::span<const CpuMemorySpan> spans);
+bool CpuMemoryUnchanged(const CpuMemoryStamp& stamp);
+// Host writes through TranslatePhysical bypass protected guest aliases.
+void InvalidateCpuPhysicalMemory(uint32_t address, uint32_t length);
+void ShutdownCpuMemoryWatch();
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/pixel_shader_filter.h b/rexlego/src/gpu_native/pixel_shader_filter.h
new file mode 100644
index 0000000..5f32b5b
--- /dev/null
+++ b/rexlego/src/gpu_native/pixel_shader_filter.h
@@ -0,0 +1,26 @@
+#pragma once
+#include <charconv>
+#include <cctype>
+#include <cstdint>
+#include <string_view>
+#include <unordered_set>
+
+namespace legodimensions::gpu_native {
+inline std::unordered_set<uint64_t> ParsePixelShaderFilter(std::string_view list) {
+  std::unordered_set<uint64_t> hashes;
+  while (!list.empty()) {
+    const auto comma = list.find(',');
+    auto token = list.substr(0, comma);
+    while (!token.empty() && std::isspace(static_cast<unsigned char>(token.front()))) token.remove_prefix(1);
+    while (!token.empty() && std::isspace(static_cast<unsigned char>(token.back()))) token.remove_suffix(1);
+    if (token.starts_with("0x") || token.starts_with("0X")) token.remove_prefix(2);
+    uint64_t value = 0;
+    const auto result = std::from_chars(token.data(), token.data() + token.size(), value, 16);
+    if (!token.empty() && result.ec == std::errc{} && result.ptr == token.data() + token.size())
+      hashes.insert(value);
+    if (comma == std::string_view::npos) break;
+    list.remove_prefix(comma + 1);
+  }
+  return hashes;
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/pool_copy.h b/rexlego/src/gpu_native/pool_copy.h
new file mode 100644
index 0000000..56c3c18
--- /dev/null
+++ b/rexlego/src/gpu_native/pool_copy.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+
+struct PhysicalCopyRange {
+  uint32_t address = 0;
+  uint32_t length = 0;
+};
+
+constexpr bool CopyRangesOverlap(PhysicalCopyRange a, PhysicalCopyRange b) {
+  return a.length && b.length && uint64_t(a.address) < uint64_t(b.address) + b.length &&
+         uint64_t(b.address) < uint64_t(a.address) + a.length;
+}
+
+// The chunk helper emits POINTLIST batches of at most 16384 64-byte records.
+struct PoolCopyDraws {
+  uint32_t records = 0;
+  uint32_t consumed = 0;
+  bool Consume(uint32_t primitive, uint32_t start, uint32_t count) {
+    if (primitive != 1 || start != consumed || !count || count > 16384 ||
+        consumed > records || count > records - consumed) return false;
+    consumed += count;
+    return true;
+  }
+};
+
+// Includes the SDK's +0x1000 adjustment for E/F physical aliases.
+bool GuestRangeOverlapsPoolCopy(uint32_t address, uint32_t size,
+                               PhysicalCopyRange copy);
+
+// Only active inside the original TU23 82BD2868 copy helper, after its bytes
+// have been copied by the CPU. Ordinary memexport shaders aren't suppressed.
+bool ConsumeCpuPoolCopyDraw(uint32_t primitive, uint32_t start, uint32_t count);
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/queries.cpp b/rexlego/src/gpu_native/queries.cpp
new file mode 100644
index 0000000..5b5e16b
--- /dev/null
+++ b/rexlego/src/gpu_native/queries.cpp
@@ -0,0 +1,204 @@
+#include "gpu_native/queries.h"
+#include "gpu_native/device.h"
+#include <plume_d3d12.h>
+#include <wrl/client.h>
+#include "gpu_native/renderer_route.h"
+#include <rex/logging.h>
+#include <rex/system/kernel_state.h>
+#include <algorithm>
+#include <atomic>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+
+namespace legodimensions::gpu_native {
+namespace {
+using Microsoft::WRL::ComPtr;
+constexpr u32 kInvalid = 0x8876086Cu, kUnsupported = 0x80004001u, kFailure = 0x80004005u;
+struct Samples { ComPtr<ID3D12QueryHeap> heap; ComPtr<ID3D12Resource> readback; };
+struct Query {
+  bool active = false, ended = false, failed = false;
+  std::vector<std::shared_ptr<Samples>> draws;
+  ComPtr<ID3D12Fence> completion;
+};
+std::mutex mutex;
+std::unordered_map<u32, std::shared_ptr<Query>> queries;
+std::atomic<u32> diagnostics{0};
+be_u32* Header(u32 address) { return REX_KERNEL_MEMORY()->TranslateVirtual<be_u32*>(address); }
+u32 Unsupported(const char* operation, u32 value) {
+  if (diagnostics.fetch_add(1) < 32)
+    REXLOG_ERROR("Native GPU: unsupported query {}={} (no fabricated result)", operation, value);
+  return kUnsupported;
+}
+u32 CreateQuery(u32 device_address, u32 type, u32 tiles) {
+  auto recording = HostDevice::LockRecording();
+  if (type != 9 || HostDevice::BackendName() != "D3D12") {
+    Unsupported("type/backend", type); return 0;
+  }
+  auto* device = static_cast<plume::D3D12Device*>(HostDevice::Device());
+  if (!device) return 0;
+  auto query = std::make_shared<Query>();
+  if (FAILED(device->d3d->CreateFence(0, D3D12_FENCE_FLAG_NONE,
+                                    IID_PPV_ARGS(&query->completion)))) return 0;
+  auto* memory = REX_KERNEL_MEMORY();
+  const u32 address = memory->SystemHeapAlloc(156, 16);
+  if (!address) return 0;
+  memory->Zero(address, 156);
+  auto* words = Header(address);
+  words[0] = device_address; words[1] = type; words[3] = 1;
+  words[37] = 1; words[38] = tiles;  // Native target is whole, no Xbox tile replay.
+  std::lock_guard lock(mutex);
+  queries.emplace(address, std::move(query));
+  if (diagnostics.fetch_add(1) < 32)
+    REXLOG_INFO("Native GPU: created hardware occlusion query 0x{:08X} tiles={} with asynchronous readback generations", address, tiles);
+  return address;
+}
+u32 IssueQuery(u32 address, u32 flags) {
+  auto recording = HostDevice::LockRecording();
+  std::shared_ptr<Query> query;
+  {
+    std::lock_guard lock(mutex);
+    auto it = queries.find(address);
+    if (it == queries.end()) return kInvalid;
+    query = it->second;
+    if (flags == 2 && query->active) return kInvalid;
+  }
+  if (flags == 2) {
+    auto* device = static_cast<plume::D3D12Device*>(HostDevice::Device());
+    auto next = std::make_shared<Query>();
+    if (!device || FAILED(device->d3d->CreateFence(0, D3D12_FENCE_FLAG_NONE,
+                                                  IID_PPV_ARGS(&next->completion)))) return kFailure;
+    // Each BEGIN owns fresh query/readback storage. Older GPU work keeps its
+    // generation alive at every pending submission, without draining the GPU.
+    next->active = true;
+    HostDevice::RetireResource(query);
+    {
+      std::lock_guard lock(mutex);
+      queries[address] = std::move(next);
+    }
+    return 0;
+  }
+  if (flags != 1) return Unsupported("issue flags", flags);
+  {
+    std::lock_guard lock(mutex);
+    if (!query->active) return kInvalid;
+    query->active = false;
+  }
+  // Each draw has already ended/resolved its query. Submit before signaling.
+  const bool submitted = HostDevice::SubmitRecordedWork();
+  auto* queue = static_cast<plume::D3D12CommandQueue*>(HostDevice::Queue());
+  const bool signaled = submitted && queue &&
+      SUCCEEDED(queue->d3d->Signal(query->completion.Get(), 1));
+  std::lock_guard lock(mutex);
+  query->ended = true; query->failed |= !signaled;
+  return signaled ? 0 : kFailure;
+}
+u32 GetQueryData(u32 address, u32 output, u32 size, u32 /*flags*/) {
+  auto recording = HostDevice::LockRecording();
+  std::lock_guard lock(mutex);
+  auto it = queries.find(address);
+  if (it == queries.end()) return kInvalid;
+  const auto& query = *it->second;
+  if (size != 4 || !output) return Unsupported("GetData size", size);
+  if (query.failed) return kFailure;
+  if (!query.ended) return 1;
+  const u64 completed = query.completion->GetCompletedValue();
+  if (completed == UINT64_MAX) return kFailure;  // Device removal isn't completion.
+  if (completed < 1) return 1;
+  u64 sum = 0;
+  for (const auto& draw : query.draws) {
+    const D3D12_RANGE read_range{0, sizeof(u64)};
+    void* mapped = nullptr;
+    if (FAILED(draw->readback->Map(0, &read_range, &mapped))) return kFailure;
+    sum += *static_cast<const u64*>(mapped);
+    const D3D12_RANGE no_writes{0, 0};
+    draw->readback->Unmap(0, &no_writes);
+  }
+  *Header(output) = static_cast<u32>(std::min<u64>(sum, UINT32_MAX));
+  if (diagnostics.fetch_add(1) < 32)
+    REXLOG_INFO("Native GPU: query 0x{:08X} complete draws={} samples={}", address, query.draws.size(), sum);
+  return 0;
+}
+u32 ReleaseQuery(u32 address) {
+  auto recording = HostDevice::LockRecording();
+  std::shared_ptr<Query> released;
+  {
+    std::lock_guard lock(mutex);
+    if (!queries.contains(address)) return 0;
+    const u32 refs = Header(address)[3];
+    if (refs > 1) { Header(address)[3] = refs - 1; return refs - 1; }
+    released = std::move(queries[address]);
+    released->active = false;
+    queries.erase(address);
+  }
+  // GPU commands refer to host heaps/readbacks, never the guest header.
+  // Retain the generation even for release-before-END; return the header now.
+  HostDevice::RetireResource(std::move(released));
+  REX_KERNEL_MEMORY()->SystemHeapFree(address);
+  return 0;
+}
+}
+struct QueryDrawScope::Impl {
+  plume::D3D12CommandList* commands;
+  std::shared_ptr<Samples> samples;
+  std::vector<std::shared_ptr<Query>> consumers;
+};
+QueryDrawScope::QueryDrawScope(plume::RenderCommandList* commands) {
+  auto scope = std::make_unique<Impl>();
+  {
+    std::lock_guard lock(mutex);
+    for (const auto& [address, query] : queries)
+      if (query->active) scope->consumers.push_back(query);
+  }
+  if (scope->consumers.empty()) return;
+  scope->commands = static_cast<plume::D3D12CommandList*>(commands);
+  auto* device = static_cast<plume::D3D12Device*>(HostDevice::Device());
+  auto samples = std::make_shared<Samples>();
+  D3D12_QUERY_HEAP_DESC query_desc{};
+  query_desc.Type = D3D12_QUERY_HEAP_TYPE_OCCLUSION; query_desc.Count = 1;
+  D3D12_HEAP_PROPERTIES heap{};
+  heap.Type = D3D12_HEAP_TYPE_READBACK; heap.CreationNodeMask = heap.VisibleNodeMask = 1;
+  D3D12_RESOURCE_DESC desc{};
+  desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; desc.Width = sizeof(u64);
+  desc.Height = desc.DepthOrArraySize = desc.MipLevels = desc.SampleDesc.Count = 1;
+  desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
+  if (!device || !commands ||
+      FAILED(device->d3d->CreateQueryHeap(&query_desc, IID_PPV_ARGS(&samples->heap))) ||
+      FAILED(device->d3d->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc,
+          D3D12_RESOURCE_STATE_COPY_DEST, nullptr, IID_PPV_ARGS(&samples->readback)))) {
+    std::lock_guard lock(mutex);
+    for (auto& query : scope->consumers) query->failed = true;
+    return;
+  }
+  scope->samples = std::move(samples);
+  scope->commands->d3d->BeginQuery(scope->samples->heap.Get(), D3D12_QUERY_TYPE_OCCLUSION, 0);
+  impl_ = std::move(scope);
+}
+QueryDrawScope::~QueryDrawScope() {
+  if (!impl_) return;
+  auto& scope = *impl_;
+  scope.commands->d3d->EndQuery(scope.samples->heap.Get(), D3D12_QUERY_TYPE_OCCLUSION, 0);
+  scope.commands->d3d->ResolveQueryData(scope.samples->heap.Get(), D3D12_QUERY_TYPE_OCCLUSION,
+                                      0, 1, scope.samples->readback.Get(), 0);
+  std::lock_guard lock(mutex);
+  for (auto& query : scope.consumers) query->draws.push_back(scope.samples);
+}
+void FailActiveQueries() {
+  std::lock_guard lock(mutex);
+  for (auto& [address, query] : queries) {
+    if (!query->active) continue;
+    if (!query->failed && diagnostics.fetch_add(1) < 32)
+      REXLOG_WARN("Native GPU: query 0x{:08X} invalidated by failed draw", address);
+    query->failed = true;
+  }
+}
+void ResetQueryResources() {
+  std::lock_guard lock(mutex);
+  for (const auto& [address, query] : queries) REX_KERNEL_MEMORY()->SystemHeapFree(address);
+  queries.clear();
+}
+}
+REX_HOOK(sub_83FB2360, legodimensions::gpu_native::CreateQuery);
+REX_HOOK(sub_83FB0738, legodimensions::gpu_native::IssueQuery);
+REX_HOOK(sub_83FB09E8, legodimensions::gpu_native::GetQueryData);
+REX_HOOK(sub_83FB05B0, legodimensions::gpu_native::ReleaseQuery);
diff --git a/rexlego/src/gpu_native/queries.h b/rexlego/src/gpu_native/queries.h
new file mode 100644
index 0000000..27be6f8
--- /dev/null
+++ b/rexlego/src/gpu_native/queries.h
@@ -0,0 +1,19 @@
+#pragma once
+#include <memory>
+namespace plume { struct RenderCommandList; }
+namespace legodimensions::gpu_native {
+// One hardware query per actual draw, shared by all active guest scopes.
+// Hardware BEGIN/END never cross a command-list submission boundary.
+class QueryDrawScope {
+ public:
+  explicit QueryDrawScope(plume::RenderCommandList* commands);
+  ~QueryDrawScope();
+  QueryDrawScope(const QueryDrawScope&) = delete;
+  QueryDrawScope& operator=(const QueryDrawScope&) = delete;
+ private:
+  struct Impl;
+  std::unique_ptr<Impl> impl_;
+};
+void FailActiveQueries();
+void ResetQueryResources();
+}
diff --git a/rexlego/src/gpu_native/renderdoc_probe.h b/rexlego/src/gpu_native/renderdoc_probe.h
new file mode 100644
index 0000000..9b261e9
--- /dev/null
+++ b/rexlego/src/gpu_native/renderdoc_probe.h
@@ -0,0 +1,54 @@
+#pragma once
+
+// Opt-in single-frame diagnostic. No RenderDoc dependency is loaded in normal
+// runs; the module must be attached before creating the native D3D12 device.
+#include <cstdlib>
+#include <filesystem>
+#include <windows.h>
+#include <renderdoc/renderdoc_app.h>
+#include <rex/logging.h>
+
+namespace legodimensions::gpu_native {
+inline RENDERDOC_API_1_0_0* g_logo_capture_api = nullptr;
+
+inline void InitializeLogoCapture() {
+  const auto* dll = std::getenv("LEGO_NATIVE_RENDERDOC_DLL");
+  const auto* path = std::getenv("LEGO_NATIVE_RENDERDOC_CAPTURE");
+  if (!dll || !*dll || !path || !*path) return;
+  if (!std::filesystem::path(dll).is_absolute() ||
+      !std::filesystem::path(path).is_absolute()) {
+    REXLOG_ERROR("Native logo capture requires absolute DLL and output paths");
+    return;
+  }
+  // Keep the capture module loaded through all D3D12 resource destruction.
+  const auto module = LoadLibraryW(std::filesystem::path(dll).c_str());
+  const auto get_api = module ? reinterpret_cast<pRENDERDOC_GetAPI>(
+      GetProcAddress(module, "RENDERDOC_GetAPI")) : nullptr;
+  if (!get_api || !get_api(eRENDERDOC_API_Version_1_0_0,
+                          reinterpret_cast<void**>(&g_logo_capture_api))) {
+    REXLOG_ERROR("Native logo capture could not initialize RenderDoc");
+    g_logo_capture_api = nullptr;
+    return;
+  }
+  g_logo_capture_api->SetLogFilePathTemplate(path);
+  g_logo_capture_api->MaskOverlayBits(0, 0);
+  REXLOG_INFO("Native logo capture armed: {}", path);
+}
+
+inline void RequestLogoCapture() {
+  static bool requested = false;
+  if (!g_logo_capture_api || requested) return;
+  requested = true;
+  // Automatic capture starts at the next present, recording a complete frame
+  // with the logo fully faded in, including its preceding depth-writing draws.
+  g_logo_capture_api->TriggerCapture();
+  REXLOG_INFO("Native logo capture requested at TT opacity >= 0.95");
+}
+
+inline void ReportLogoCapture() {
+  static bool reported = false;
+  if (!g_logo_capture_api || reported || !g_logo_capture_api->GetNumCaptures()) return;
+  reported = true;
+  REXLOG_INFO("Native logo capture saved (one frame)");
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/renderer_mode.cpp b/rexlego/src/gpu_native/renderer_mode.cpp
new file mode 100644
index 0000000..b9ff9de
--- /dev/null
+++ b/rexlego/src/gpu_native/renderer_mode.cpp
@@ -0,0 +1,7 @@
+#include <rex/cvar.h>
+
+// The PM4 route prioritizes the SDK's complete Xenos rendering semantics.
+// The detached native renderer remains available for development and A/B tests.
+REXCVAR_DEFINE_BOOL(gpu_native_pm4, false, "GPU",
+    "Use the original XDK command stream and SDK GPU renderer in native builds")
+    .lifecycle(rex::cvar::Lifecycle::kRequiresRestart);
diff --git a/rexlego/src/gpu_native/renderer_mode.h b/rexlego/src/gpu_native/renderer_mode.h
new file mode 100644
index 0000000..dfb1627
--- /dev/null
+++ b/rexlego/src/gpu_native/renderer_mode.h
@@ -0,0 +1,25 @@
+#pragma once
+#include <cstdlib>
+#include <cstring>
+#include <rex/logging.h>
+#include <rex/cvar.h>
+
+REXCVAR_DECLARE(bool, gpu_native_pm4);
+
+namespace legodimensions::gpu_native {
+// Select once, before runtime setup and the first XDK hook. Mixing native
+// objects and original XDK objects in one session is invalid.
+inline bool UsePm4Reference() {
+  static const bool enabled = [] {
+    const char* value = std::getenv("LEGO_NATIVE_PM4_REFERENCE");
+    // An explicit 0/1 override supports controlled A/B probes. Normal launches
+    // use the persisted setting, so the tested renderer is reproducible.
+    const bool selected = value && std::strcmp(value, "1") == 0 ? true :
+        value && std::strcmp(value, "0") == 0 ? false : REXCVAR_GET(gpu_native_pm4);
+    REXLOG_INFO("GPU route: {}", selected ? "original XDK PM4 / SDK renderer" :
+                "detached native D3D12 renderer");
+    return selected;
+  }();
+  return enabled;
+}
+}
diff --git a/rexlego/src/gpu_native/renderer_route.h b/rexlego/src/gpu_native/renderer_route.h
new file mode 100644
index 0000000..2b47c90
--- /dev/null
+++ b/rexlego/src/gpu_native/renderer_route.h
@@ -0,0 +1,29 @@
+#pragma once
+#include <rex/hook.h>
+#include "gpu_native/renderer_mode.h"
+
+// Local to native GPU translation units: every replacement, including resource
+// creation, synchronization and raw placement hooks, takes the same route.
+#undef REX_HOOK
+#define REX_HOOK(name, function) \
+  extern "C" REX_FUNC(__imp__##name); \
+  extern "C" REX_FUNC(name) { \
+    if (legodimensions::gpu_native::UsePm4Reference()) { \
+      __imp__##name(ctx, base); \
+      return; \
+    } \
+    rex::ppc::HostToGuestFunction<function>(ctx, base); \
+  }
+
+#undef REX_HOOK_RAW
+#define REX_HOOK_RAW(name) \
+  extern "C" REX_FUNC(__imp__##name); \
+  static REX_FUNC(lego_native_##name); \
+  extern "C" REX_FUNC(name) { \
+    if (legodimensions::gpu_native::UsePm4Reference()) { \
+      __imp__##name(ctx, base); \
+      return; \
+    } \
+    lego_native_##name(ctx, base); \
+  } \
+  static REX_FUNC(lego_native_##name)
diff --git a/rexlego/src/gpu_native/resolve_region.h b/rexlego/src/gpu_native/resolve_region.h
new file mode 100644
index 0000000..0b63671
--- /dev/null
+++ b/rexlego/src/gpu_native/resolve_region.h
@@ -0,0 +1,28 @@
+#pragma once
+#include <algorithm>
+#include <cstdint>
+#include <optional>
+
+namespace legodimensions::gpu_native {
+struct ResolveRect { int32_t left, top, right, bottom; };
+struct ResolvePoint { int32_t x, y; };
+struct ResolveRegion { uint32_t left, top, width, height, x, y; };
+
+// NULL rect selects the logical extent fitting both padded source and target.
+// Explicit rectangles must fit exactly; silently clipping breaks atlas tiles.
+inline std::optional<ResolveRegion> CheckedResolveRegion(
+    uint32_t sw, uint32_t sh, uint32_t dw, uint32_t dh,
+    const ResolveRect* rect = nullptr, const ResolvePoint* point = nullptr) {
+  const ResolvePoint p = point ? *point : ResolvePoint{0, 0};
+  if (p.x < 0 || p.y < 0 || uint32_t(p.x) >= dw || uint32_t(p.y) >= dh)
+    return {};
+  const ResolveRect r = rect ? *rect : ResolveRect{0, 0,
+      int32_t(std::min(sw, dw - p.x)), int32_t(std::min(sh, dh - p.y))};
+  if (r.left < 0 || r.top < 0 || r.right <= r.left || r.bottom <= r.top ||
+      uint32_t(r.right) > sw || uint32_t(r.bottom) > sh) return {};
+  const uint32_t w = r.right - r.left, h = r.bottom - r.top;
+  if (w > dw - p.x || h > dh - p.y) return {};
+  return ResolveRegion{uint32_t(r.left), uint32_t(r.top), w, h,
+                       uint32_t(p.x), uint32_t(p.y)};
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/runtime_linked_shader_cache.cpp b/rexlego/src/gpu_native/runtime_linked_shader_cache.cpp
new file mode 100644
index 0000000..2da3d7e
--- /dev/null
+++ b/rexlego/src/gpu_native/runtime_linked_shader_cache.cpp
@@ -0,0 +1,6 @@
+#define g_linkedShaderCacheEntries g_runtimeLinkedShaderCacheEntries
+#define g_linkedShaderCacheEntryCount g_runtimeLinkedShaderCacheEntryCount
+#define g_compressedLinkedDxilCache g_runtimeCompressedLinkedDxilCache
+#define g_linkedDxilCacheCompressedSize g_runtimeLinkedDxilCacheCompressedSize
+#define g_linkedDxilCacheDecompressedSize g_runtimeLinkedDxilCacheDecompressedSize
+#include "../../out/native-gpu/runtime-linked-dxil-cache.cpp"
diff --git a/rexlego/src/gpu_native/runtime_shader_cache.cpp b/rexlego/src/gpu_native/runtime_shader_cache.cpp
new file mode 100644
index 0000000..6ab9274
--- /dev/null
+++ b/rexlego/src/gpu_native/runtime_shader_cache.cpp
@@ -0,0 +1,9 @@
+#define g_shaderCacheEntries g_runtimeShaderCacheEntries
+#define g_shaderCacheEntryCount g_runtimeShaderCacheEntryCount
+#define g_compressedDxilCache g_runtimeCompressedDxilCache
+#define g_dxilCacheCompressedSize g_runtimeDxilCacheCompressedSize
+#define g_dxilCacheDecompressedSize g_runtimeDxilCacheDecompressedSize
+#define g_compressedSpirvCache g_runtimeCompressedSpirvCache
+#define g_spirvCacheCompressedSize g_runtimeSpirvCacheCompressedSize
+#define g_spirvCacheDecompressedSize g_runtimeSpirvCacheDecompressedSize
+#include "../../out/native-gpu/runtime-cache.cpp"
diff --git a/rexlego/src/gpu_native/sampler_state.h b/rexlego/src/gpu_native/sampler_state.h
new file mode 100644
index 0000000..b1a78de
--- /dev/null
+++ b/rexlego/src/gpu_native/sampler_state.h
@@ -0,0 +1,102 @@
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+
+#include <plume_render_interface_types.h>
+
+namespace legodimensions::gpu_native {
+
+using TextureFetchWords = std::array<uint32_t, 6>;
+
+// TU23 83FB58A8, before its Xbox fence/reference tracking. Input words are
+// host endian; the texture header and the device shadow remain big endian.
+inline TextureFetchWords MergeTextureFetch(const TextureFetchWords& old,
+                                           const TextureFetchWords& header,
+                                           uint8_t requested_min,
+                                           uint8_t requested_max) {
+  TextureFetchWords result;
+  result[0] = (header[0] & ~0x003FFC00u) | (old[0] & 0x003FFC00u);
+  const auto physical_word = [](uint32_t value) {
+    return (value & 0x1FFFFFFFu) + (((value >> 20) + 512u) & 0x1000u);
+  };
+  result[1] = (physical_word(header[1]) & ~0x800u) | (old[1] & 0x800u);
+  result[2] = header[2];
+  result[3] = (header[3] & ~0x7FF80000u) | (old[3] & 0x7FF80000u);
+  result[4] = (header[4] & 0x3FCu) | (old[4] & ~0x3FCu);
+  const uint32_t mip_min = std::max((header[4] >> 2) & 15u, uint32_t(requested_min));
+  const uint32_t mip_max = std::min((header[4] >> 6) & 15u, uint32_t(requested_max));
+  result[4] = (result[4] & ~0x3FCu) | ((mip_min & 15u) << 2) | ((mip_max & 15u) << 6);
+  result[5] = ((header[5] & 0x1FFFFE00u) +
+               (((header[5] >> 20) + 512u) & 0x1000u)) | (old[5] & 0x1FFu);
+  return result;
+}
+
+inline void ClearTextureFetch(TextureFetchWords& words) { words[0] &= ~3u; }
+
+// Only fields consumed by the desktop sampler are keys. No padding or texture
+// addresses: cached descriptors never change until the host device is idle.
+struct NativeSamplerKey {
+  std::array<uint32_t, 5> words{};
+  bool operator==(const NativeSamplerKey&) const = default;
+};
+
+inline plume::RenderSamplerDesc DecodeTextureSampler(const TextureFetchWords& fetch) {
+  using Address = plume::RenderTextureAddressMode;
+  constexpr Address modes[] = {Address::WRAP, Address::MIRROR, Address::CLAMP,
+      Address::MIRROR_ONCE, Address::CLAMP, Address::MIRROR_ONCE,
+      Address::BORDER, Address::MIRROR_ONCE};
+  plume::RenderSamplerDesc desc;
+  const uint32_t dimension = (fetch[5] >> 9) & 3u;
+  desc.addressU = dimension == 3 ? Address::CLAMP : modes[(fetch[0] >> 10) & 7u];
+  desc.addressV = dimension == 0 || dimension == 3 ? Address::CLAMP : modes[(fetch[0] >> 13) & 7u];
+  desc.addressW = dimension == 2 ? modes[(fetch[0] >> 16) & 7u] : Address::CLAMP;
+  const uint32_t mag = (fetch[3] >> 19) & 3u;
+  const uint32_t min = (fetch[3] >> 21) & 3u;
+  const uint32_t mip = (fetch[3] >> 23) & 3u;
+  const uint32_t aniso = std::min((fetch[3] >> 25) & 7u, 5u);
+  desc.anisotropyEnabled = aniso != 0;
+  desc.maxAnisotropy = aniso ? 1u << (aniso - 1u) : 1u;
+  desc.magFilter = aniso || mag == 1 ? plume::RenderFilter::LINEAR : plume::RenderFilter::NEAREST;
+  desc.minFilter = aniso || min == 1 ? plume::RenderFilter::LINEAR : plume::RenderFilter::NEAREST;
+  desc.mipmapMode = aniso || mip == 1 ? plume::RenderMipmapMode::LINEAR : plume::RenderMipmapMode::NEAREST;
+  // Xenia adds this bias in generated shaders. Native currently emits Sample
+  // for implicit LOD, so use the host sampler there. Explicit register LOD is
+  // a separate compiler gap; this does not claim to implement it.
+  const int32_t bias = int32_t((fetch[4] >> 12) & 0x3FFu);
+  desc.mipLODBias = float(bias >= 512 ? bias - 1024 : bias) / 32.0f;
+  desc.borderColor = (fetch[5] & 3u) == 1 ? plume::RenderBorderColor::OPAQUE_WHITE
+                                         : plume::RenderBorderColor::TRANSPARENT_BLACK;
+
+  uint32_t longest_minus_one;
+  if (dimension == 0) longest_minus_one = fetch[2] & 0xFFFFFFu;
+  else if (dimension == 2)
+    longest_minus_one = std::max({fetch[2] & 0x7FFu, (fetch[2] >> 11) & 0x7FFu, fetch[2] >> 22});
+  else longest_minus_one = std::max(fetch[2] & 0x1FFFu, (fetch[2] >> 13) & 0x1FFFu);
+  const uint32_t size_max = std::bit_width(longest_minus_one + 1u) - 1u;
+  uint32_t min_level = 0, max_level = 0;
+  if ((fetch[5] >> 12) & 0x1FFFFu) {
+    min_level = std::min((fetch[4] >> 2) & 15u, size_max);
+    max_level = std::max(min_level, std::min((fetch[4] >> 6) & 15u, size_max));
+    if (max_level && !((fetch[1] >> 12) & 0x1FFFFu)) min_level = std::max(min_level, 1u);
+  }
+  desc.minLOD = float(min_level);
+  desc.maxLOD = float(std::max(min_level, max_level));
+  if (mip == 2) desc.maxLOD = desc.minLOD + (aniso ? 0.0f : 0.25f);
+  return desc;
+}
+
+inline NativeSamplerKey TextureSamplerKey(const plume::RenderSamplerDesc& desc) {
+  return {{{uint32_t(desc.addressU) | (uint32_t(desc.addressV) << 3) |
+             (uint32_t(desc.addressW) << 6) | (uint32_t(desc.minFilter) << 9) |
+             (uint32_t(desc.magFilter) << 11) | (uint32_t(desc.mipmapMode) << 13) |
+             (uint32_t(desc.anisotropyEnabled) << 15) | (desc.maxAnisotropy << 16) |
+             (uint32_t(desc.borderColor) << 21),
+            std::bit_cast<uint32_t>(desc.mipLODBias),
+            std::bit_cast<uint32_t>(desc.minLOD),
+            std::bit_cast<uint32_t>(desc.maxLOD), 0u}}};
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/shader_archive.cpp b/rexlego/src/gpu_native/shader_archive.cpp
index 4026fd7..1e5a9e5 100644
--- a/rexlego/src/gpu_native/shader_archive.cpp
+++ b/rexlego/src/gpu_native/shader_archive.cpp
@@ -12,6 +12,8 @@
 #include <memory>
 #include <mutex>
 #include <utility>
+#include <unordered_map>
+#include <vector>
 
 #define MINIZ_HEADER_FILE_ONLY
 #include <miniz.h>
@@ -40,16 +42,82 @@ size_t ShaderContainerByteLength(const void* shader_container) {
   const auto* bytes = static_cast<const uint8_t*>(shader_container);
   const uint32_t virtual_size = LoadBigEndianU32(bytes + sizeof(uint32_t));
   const uint32_t physical_size = LoadBigEndianU32(bytes + 2 * sizeof(uint32_t));
-  return size_t{virtual_size} + sizeof(uint32_t) + size_t{physical_size};
+  constexpr uint32_t kMaximumSectionSize = 4 * 1024 * 1024;
+  if (virtual_size > kMaximumSectionSize || physical_size > kMaximumSectionSize) {
+    return size_t{kMaximumSectionSize} * 2 + sizeof(uint32_t);
+  }
+  // Archive containers repeat physical_size at the section boundary, while
+  // shaders built by the game's runtime compiler omit that marker.
+  const uint32_t boundary = LoadBigEndianU32(bytes + virtual_size);
+  const size_t padding = boundary == physical_size ? sizeof(uint32_t) : 0;
+  return size_t{virtual_size} + padding + size_t{physical_size};
 }
 
 const ShaderCacheEntry* FindShader(uint64_t hash) {
-  const ShaderCacheEntry* const begin = g_shaderCacheEntries;
-  const ShaderCacheEntry* const end = begin + g_shaderCacheEntryCount;
-  const ShaderCacheEntry* const result = std::lower_bound(
-      begin, end, hash,
-      [](const ShaderCacheEntry& entry, uint64_t key) { return entry.hash < key; });
-  return result != end && result->hash == hash ? result : nullptr;
+  const auto find_in = [hash](ShaderCacheEntry* begin, size_t count) {
+    ShaderCacheEntry* const end = begin + count;
+    ShaderCacheEntry* const result = std::lower_bound(
+        begin, end, hash, [](const ShaderCacheEntry& entry, uint64_t key) {
+          return entry.hash < key;
+        });
+    return result != end && result->hash == hash ? result : nullptr;
+  };
+  if (auto* shader = find_in(g_shaderCacheEntries, g_shaderCacheEntryCount)) {
+    return shader;
+  }
+  return find_in(g_runtimeShaderCacheEntries, g_runtimeShaderCacheEntryCount);
+}
+
+uint32_t FindShaderTextureMask(uint64_t hash) {
+  static const auto masks = [] {
+    std::unordered_map<uint64_t, uint32_t> result;
+    for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) {
+      const auto& entry = g_shaderMicrocodeEntries[i];
+      result[entry.containerHash] |= entry.textureMask;
+    }
+    return result;
+  }();
+  const auto it = masks.find(hash);
+  return it == masks.end() ? ~uint32_t{0} : it->second;
+}
+
+const ShaderCacheEntry* FindShaderByMicrocode(const void* microcode,
+                                              uint32_t stage) {
+  if (!microcode) {
+    return nullptr;
+  }
+  using Index = std::unordered_multimap<uint64_t, const ShaderMicrocodeEntry*>;
+  static std::once_flag index_once;
+  static Index index;
+  std::call_once(index_once, [] {
+    index.reserve(g_shaderMicrocodeEntryCount);
+    const auto add = [](Index& target, ShaderMicrocodeEntry* entries, size_t count) {
+      for (size_t i = 0; i < count; ++i) {
+        if (entries[i].microcodeSize != 0) {
+          target.emplace(entries[i].prefixHash, &entries[i]);
+        }
+      }
+    };
+    add(index, g_shaderMicrocodeEntries, g_shaderMicrocodeEntryCount);
+  });
+
+  const uint64_t prefix = XXH3_64bits(microcode, 8);
+  const auto [first, last] = index.equal_range(prefix);
+  // Thousands of physical shader sections share eight zero prefix bytes.
+  // A missing placement shader used to re-hash the same bytes once for each
+  // candidate on every bind. Reuse hashes only within THIS lookup, keyed by
+  // the exact byte length; never cache across guest writes or bind calls.
+  std::unordered_map<uint32_t, uint64_t> hashes_by_size;
+  for (auto it = first; it != last; ++it) {
+    const ShaderMicrocodeEntry* entry = it->second;
+    if (entry->stage != stage) continue;
+    auto [hash, inserted] = hashes_by_size.try_emplace(entry->microcodeSize, 0);
+    if (inserted) hash->second = XXH3_64bits(microcode, entry->microcodeSize);
+    if (hash->second == entry->microcodeHash) {
+      if (const auto* shader = FindShader(entry->containerHash)) return shader;
+    }
+  }
+  return nullptr;
 }
 
 uint64_t HashShaderContainer(const void* shader_container) {
@@ -66,16 +134,28 @@ ShaderBytecode FindDxil(uint64_t hash, uint32_t spec_constants) {
     return {};
   }
 
-  static std::once_flag dxil_once;
-  static std::unique_ptr<uint8_t[]> dxil;
-  std::call_once(dxil_once, [] {
-    auto data = std::make_unique<uint8_t[]>(g_dxilCacheDecompressedSize);
-    const size_t result = ZSTD_decompress(data.get(), g_dxilCacheDecompressedSize,
-                                          g_compressedDxilCache,
-                                          g_dxilCacheCompressedSize);
-    if (ZSTD_isError(result) || result != g_dxilCacheDecompressedSize) {
-      REXLOG_ERROR("Native GPU DXIL archive decompression failed: {} of {} bytes",
-                   result, g_dxilCacheDecompressedSize);
+  const bool runtime = shader >= g_runtimeShaderCacheEntries &&
+                       shader < g_runtimeShaderCacheEntries +
+                                    g_runtimeShaderCacheEntryCount;
+  static std::once_flag main_dxil_once;
+  static std::once_flag runtime_dxil_once;
+  static std::unique_ptr<uint8_t[]> main_dxil;
+  static std::unique_ptr<uint8_t[]> runtime_dxil;
+  auto& once = runtime ? runtime_dxil_once : main_dxil_once;
+  auto& dxil = runtime ? runtime_dxil : main_dxil;
+  const size_t compressed_size = runtime ? g_runtimeDxilCacheCompressedSize
+                                         : g_dxilCacheCompressedSize;
+  const size_t decompressed_size = runtime ? g_runtimeDxilCacheDecompressedSize
+                                           : g_dxilCacheDecompressedSize;
+  const uint8_t* compressed = runtime ? g_runtimeCompressedDxilCache
+                                      : g_compressedDxilCache;
+  std::call_once(once, [&] {
+    auto data = std::make_unique<uint8_t[]>(decompressed_size);
+    const size_t result =
+        ZSTD_decompress(data.get(), decompressed_size, compressed, compressed_size);
+    if (ZSTD_isError(result) || result != decompressed_size) {
+      REXLOG_ERROR("Native GPU {} DXIL archive decompression failed: {} of {} bytes",
+                   runtime ? "runtime" : "main", result, decompressed_size);
       return;
     }
     dxil = std::move(data);
@@ -89,8 +169,11 @@ ShaderBytecode FindDxil(uint64_t hash, uint32_t spec_constants) {
   }
 
   const uint32_t masked = spec_constants & shader->specConstantsMask;
-  const LinkedShaderCacheEntry* const first = g_linkedShaderCacheEntries;
-  const LinkedShaderCacheEntry* const end = first + g_linkedShaderCacheEntryCount;
+  const LinkedShaderCacheEntry* const first = runtime
+      ? g_runtimeLinkedShaderCacheEntries : g_linkedShaderCacheEntries;
+  const size_t linked_count = runtime ? g_runtimeLinkedShaderCacheEntryCount
+                                      : g_linkedShaderCacheEntryCount;
+  const LinkedShaderCacheEntry* const end = first + linked_count;
   const auto key = std::pair{hash, masked};
   const LinkedShaderCacheEntry* const linked = std::lower_bound(
       first, end, key,
@@ -104,16 +187,27 @@ ShaderBytecode FindDxil(uint64_t hash, uint32_t spec_constants) {
     return {};
   }
 
-  static std::once_flag linked_once;
-  static std::unique_ptr<uint8_t[]> linked_dxil;
-  std::call_once(linked_once, [] {
-    auto data = std::make_unique<uint8_t[]>(g_linkedDxilCacheDecompressedSize);
-    mz_ulong size = static_cast<mz_ulong>(g_linkedDxilCacheDecompressedSize);
+  static std::once_flag main_linked_once;
+  static std::once_flag runtime_linked_once;
+  static std::unique_ptr<uint8_t[]> main_linked_dxil;
+  static std::unique_ptr<uint8_t[]> runtime_linked_dxil;
+  auto& linked_once = runtime ? runtime_linked_once : main_linked_once;
+  auto& linked_dxil = runtime ? runtime_linked_dxil : main_linked_dxil;
+  const size_t linked_compressed_size = runtime
+      ? g_runtimeLinkedDxilCacheCompressedSize : g_linkedDxilCacheCompressedSize;
+  const size_t linked_decompressed_size = runtime
+      ? g_runtimeLinkedDxilCacheDecompressedSize : g_linkedDxilCacheDecompressedSize;
+  const uint8_t* linked_compressed = runtime
+      ? g_runtimeCompressedLinkedDxilCache : g_compressedLinkedDxilCache;
+  std::call_once(linked_once, [&] {
+    auto data = std::make_unique<uint8_t[]>(linked_decompressed_size);
+    mz_ulong size = static_cast<mz_ulong>(linked_decompressed_size);
     const int result = mz_uncompress(
-        data.get(), &size, g_compressedLinkedDxilCache,
-        static_cast<mz_ulong>(g_linkedDxilCacheCompressedSize));
-    if (result != MZ_OK || size != g_linkedDxilCacheDecompressedSize) {
-      REXLOG_ERROR("Native GPU linked DXIL archive decompression failed");
+        data.get(), &size, linked_compressed,
+        static_cast<mz_ulong>(linked_compressed_size));
+    if (result != MZ_OK || size != linked_decompressed_size) {
+      REXLOG_ERROR("Native GPU {} linked DXIL archive decompression failed",
+                   runtime ? "runtime" : "main");
       return;
     }
     linked_dxil = std::move(data);
diff --git a/rexlego/src/gpu_native/shader_archive.h b/rexlego/src/gpu_native/shader_archive.h
index fb38a0b..fdb54a8 100644
--- a/rexlego/src/gpu_native/shader_archive.h
+++ b/rexlego/src/gpu_native/shader_archive.h
@@ -21,6 +21,18 @@ struct ShaderCacheEntry {
   const uint32_t specConstantsMask;
 };
 
+// Signatures cover both a complete physical shader section and its nested
+// instruction stream because both pointer forms are produced by XDK helpers.
+struct ShaderMicrocodeEntry {
+  const uint64_t prefixHash;
+  const uint64_t microcodeHash;
+  const uint64_t containerHash;
+  const uint32_t microcodeSize;
+  // 0 = vertex, 1 = pixel.
+  const uint32_t stage;
+  const uint32_t textureMask = ~uint32_t{0};
+};
+
 extern ShaderCacheEntry g_shaderCacheEntries[];
 extern const size_t g_shaderCacheEntryCount;
 
@@ -32,6 +44,15 @@ extern const uint8_t g_compressedSpirvCache[];
 extern const size_t g_spirvCacheCompressedSize;
 extern const size_t g_spirvCacheDecompressedSize;
 
+extern ShaderCacheEntry g_runtimeShaderCacheEntries[];
+extern const size_t g_runtimeShaderCacheEntryCount;
+extern const uint8_t g_runtimeCompressedDxilCache[];
+extern const size_t g_runtimeDxilCacheCompressedSize;
+extern const size_t g_runtimeDxilCacheDecompressedSize;
+
+extern ShaderMicrocodeEntry g_shaderMicrocodeEntries[];
+extern const size_t g_shaderMicrocodeEntryCount;
+
 namespace legodimensions::gpu_native {
 
 struct ShaderBytecode {
@@ -41,11 +62,18 @@ struct ShaderBytecode {
   explicit operator bool() const { return data != nullptr && size != 0; }
 };
 
-// Xbox shader containers place a four-byte separator between the virtual and
-// physical programs. LEGO's cache key covers that separator as well.
+// Archive containers repeat physical_size between their virtual and physical
+// programs. Runtime-created containers omit that marker; both layouts occur in
+// LEGO Dimensions and are detected from the container contents.
 size_t ShaderContainerByteLength(const void* shader_container);
 
 const ShaderCacheEntry* FindShader(uint64_t hash);
+uint32_t FindShaderTextureMask(uint64_t hash);
+
+// Matches an XDK placement shader, which retains only its physical microcode
+// pointer, back to the original container compiled into the AOT archive.
+const ShaderCacheEntry* FindShaderByMicrocode(const void* microcode,
+                                              uint32_t stage);
 
 // Hashes the exact Xbox container span used by the AOT generator.
 uint64_t HashShaderContainer(const void* shader_container);
diff --git a/rexlego/src/gpu_native/shader_texture_usage.h b/rexlego/src/gpu_native/shader_texture_usage.h
new file mode 100644
index 0000000..93ccdb6
--- /dev/null
+++ b/rexlego/src/gpu_native/shader_texture_usage.h
@@ -0,0 +1,47 @@
+#pragma once
+#include <algorithm>
+#include <cstdint>
+#include <span>
+
+namespace legodimensions::gpu_native {
+// Big-endian Xenos instruction stream, not the containing physical section.
+// Include every exec clause, even conditional and unreachable clauses. Unknown
+// or malformed input keeps all bindings; no predicate is evaluated here.
+inline uint32_t ShaderTextureUsage(std::span<const uint8_t> code) {
+  constexpr uint32_t all = ~uint32_t{0};
+  if (code.empty() || code.size() % 12) return all;
+  const auto word = [&](size_t offset) {
+    return uint32_t(code[offset]) << 24 | uint32_t(code[offset + 1]) << 16 |
+           uint32_t(code[offset + 2]) << 8 | uint32_t(code[offset + 3]);
+  };
+  size_t cf_end = code.size() / 12;
+  bool saw_exec = false;
+  uint32_t mask = 0;
+  for (size_t pair = 0; pair < cf_end; ++pair) {
+    const uint32_t a = word(pair * 12), b = word(pair * 12 + 4), c = word(pair * 12 + 8);
+    const uint32_t lo[] = {a, (b >> 16) | (c << 16)};
+    const uint32_t hi[] = {b & 0xFFFF, c >> 16};
+    for (size_t half = 0; half < 2; ++half) {
+      const uint32_t opcode = hi[half] >> 12;
+      if (!((opcode >= 1 && opcode <= 6) || opcode == 13 || opcode == 14)) continue;
+      const size_t address = lo[half] & 0xFFF;
+      const uint32_t count = (lo[half] >> 12) & 7;
+      const uint32_t sequence = (lo[half] >> 16) & 0xFFF;
+      if (count > 6 || address <= pair || address + count > code.size() / 12) return all;
+      cf_end = std::min(cf_end, address);
+      saw_exec = true;
+      for (uint32_t i = 0; i < count; ++i) {
+        if (!((sequence >> (i * 2)) & 1)) continue;
+        const uint32_t fetch = word((address + i) * 12);
+        const uint32_t fetch_opcode = fetch & 31;
+        if (fetch_opcode == 0) continue; // vertex buffer fetch
+        if (fetch_opcode != 1 && (fetch_opcode < 16 || fetch_opcode > 19) &&
+            (fetch_opcode < 24 || fetch_opcode > 26)) return all;
+        // Even texture-unit state operations retain their declared slot.
+        mask |= uint32_t{1} << ((fetch >> 20) & 31);
+      }
+    }
+  }
+  return saw_exec ? mask : all;
+}
+} // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/shaders.cpp b/rexlego/src/gpu_native/shaders.cpp
index d5c6fff..97aec0a 100644
--- a/rexlego/src/gpu_native/shaders.cpp
+++ b/rexlego/src/gpu_native/shaders.cpp
@@ -1,21 +1,33 @@
 #include "gpu_native/shaders.h"
+#include "gpu_native/long_probe.h"
 
+#include <algorithm>
 #include <array>
+#include <atomic>
+#include <cstdio>
+#include <cstdlib>
 #include <cstring>
+#include <filesystem>
+#include <fstream>
 #include <memory>
 #include <mutex>
 #include <unordered_map>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
 #include <plume_render_interface.h>
 #include <rex/logging.h>
+#include <rex/cvar.h>
 #include <rex/system/kernel_state.h>
 #include <rex/system/xmemory.h>
+#include <xxhash.h>
 
 #include "gpu_native/d3d.h"
 #include "gpu_native/device.h"
 #include "gpu_native/shader_archive.h"
+#include "gpu_native/pixel_shader_filter.h"
+#include "cheat_menu.h"
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -28,14 +40,19 @@ struct ShaderResource {
   u32 guest_address = 0;
   ShaderStage stage = ShaderStage::kVertex;
   u64 hash = 0;
+  u32 texture_mask = ~u32{0};
   const ShaderCacheEntry* cache_entry = nullptr;
   std::mutex shader_mutex;
   std::unordered_map<u32, std::unique_ptr<plume::RenderShader>> variants;
+  bool owns_guest_memory = true;
+  u32 physical_code_address = 0;
+  u32 physical_code_size = 0;
 };
 
 std::mutex g_registry_mutex;
 std::unordered_map<u32, std::shared_ptr<ShaderResource>> g_registry;
 std::array<std::shared_ptr<ShaderResource>, 2> g_bound_shaders;
+std::atomic<u32> g_shader_warning_count{0};
 
 size_t StageIndex(ShaderStage stage) {
   return stage == ShaderStage::kVertex ? 0 : 1;
@@ -48,12 +65,227 @@ std::shared_ptr<ShaderResource> FindResource(u32 guest_address) {
 }
 
 D3DResource* GuestHeader(u32 guest_address) {
+  return REX_KERNEL_MEMORY()->TranslateVirtual<D3DResource*>(guest_address);
+}
+
+void DumpMissingShader(const ShaderContainer* container, size_t byte_length,
+                       ShaderStage stage, u64 hash, const char* category = "");
+
+bool ReadableGuestRange(u32 address, u32 size) {
+  if (!size || uint64_t(address) + size > 0x100000000ull) return false;
   auto* memory = REX_KERNEL_MEMORY();
-  return reinterpret_cast<D3DResource*>(memory->virtual_membase() + guest_address);
+  for (uint64_t at = address; at < uint64_t(address) + size;) {
+    auto* heap = memory->LookupHeap(static_cast<u32>(at));
+    rex::memory::HeapAllocationInfo info{};
+    if (!heap || !heap->QueryRegionInfo(static_cast<u32>(at), &info) ||
+        !(info.state & rex::memory::kMemoryAllocationCommit) ||
+        !(info.protect & rex::memory::kMemoryProtectRead)) return false;
+    at = (at & ~uint64_t(4095)) + 4096;
+  }
+  return true;
+}
+
+std::shared_ptr<ShaderResource> AdoptShader(u32 guest_address,
+                                           ShaderStage expected_stage) {
+  if (const auto existing = FindResource(guest_address)) {
+    return existing;
+  }
+  if (!guest_address) {
+    return nullptr;
+  }
+  auto* memory = REX_KERNEL_MEMORY();
+  const auto* bytes = memory->TranslateVirtual<const uint8_t*>(guest_address);
+  const auto* header = reinterpret_cast<const D3DResource*>(bytes);
+  const u32 common = header->common;
+  // Game-owned XDK shaders use the bare resource type (6/7), while resources
+  // allocated by our create hooks also carry kShaderFlag.
+  if ((common & 0xFu) != static_cast<u32>(expected_stage)) {
+    if (g_shader_warning_count.fetch_add(1, std::memory_order_relaxed) < 16) {
+      const auto* words = reinterpret_cast<const be_u32*>(bytes);
+      REXLOG_WARN(
+          "Native GPU: cannot adopt {} shader 0x{:08X}, header "
+          "{:08X} {:08X} {:08X} {:08X} {:08X} {:08X} {:08X} {:08X} "
+          "{:08X} {:08X}",
+          expected_stage == ShaderStage::kVertex ? "vertex" : "pixel",
+          guest_address, static_cast<u32>(words[0]),
+          static_cast<u32>(words[1]), static_cast<u32>(words[2]),
+          static_cast<u32>(words[3]), static_cast<u32>(words[4]),
+          static_cast<u32>(words[5]), static_cast<u32>(words[6]),
+          static_cast<u32>(words[7]), static_cast<u32>(words[8]),
+          static_cast<u32>(words[9]));
+    }
+    return nullptr;
+  }
+  const size_t pointer_offset =
+      expected_stage == ShaderStage::kVertex ? 0x20 : 0x18;
+  const u32 container_address =
+      *reinterpret_cast<const be_u32*>(bytes + pointer_offset);
+  if (!container_address) {
+    return nullptr;
+  }
+  // XDK placement objects retain the copied physical microcode, not the
+  // original ShaderContainer. Associate it with the independently indexed
+  // AOT entry before creating any host object.
+  const auto* microcode =
+      memory->TranslateVirtual<const uint8_t*>(container_address);
+  const ShaderCacheEntry* cache_entry = FindShaderByMicrocode(
+      microcode, expected_stage == ShaderStage::kPixel ? 1u : 0u);
+  if (!cache_entry) {
+    // Diagnostic only, opt-in. Bound distinct objects rather than bind calls:
+    // repeated draws must not exhaust the capture budget on the first pair.
+    static const bool capture_enabled = [] {
+      char* value = nullptr;
+      size_t length = 0;
+      _dupenv_s(&value, &length, "LEGO_DUMP_MISSING_SHADERS");
+      const bool enabled = value && length > 1;
+      std::free(value);
+      return enabled;
+    }();
+    static std::mutex capture_mutex;
+    static std::unordered_set<u32> captured_objects;
+    bool capture_object = false;
+    if (capture_enabled) {
+      std::lock_guard lock(capture_mutex);
+      capture_object = captured_objects.size() < 256 &&
+                       captured_objects.insert(guest_address).second;
+    }
+    if (capture_object) {
+      DumpMissingShader(reinterpret_cast<const ShaderContainer*>(microcode),
+          std::min<size_t>(2048, 4096 - (container_address & 4095)),
+          expected_stage, XXH3_64bits(microcode, 8), "placement-raw");
+      // XGSet*ShaderHeader copies the virtual section immediately after the
+      // SDK object (TU23 83FB7618 / 83FB74F0). Validate both ranges before
+      // reconstructing a container from the separately allocated sections.
+      const u32 header_address = guest_address +
+          (expected_stage == ShaderStage::kVertex ? kVertexShaderGuestSize
+                                                  : kPixelShaderGuestSize);
+      const auto* virtual_header =
+          memory->TranslateVirtual<const ShaderContainer*>(header_address);
+      if (ReadableGuestRange(header_address, sizeof(ShaderContainer))) {
+        const u32 virtual_size = virtual_header->virtual_size;
+        const u32 physical_size = virtual_header->physical_size;
+        if ((u32(virtual_header->flags) & 0xFFFFFFFEu) == 0x102A1100u &&
+            virtual_size >= sizeof(ShaderContainer) && virtual_size <= 65536 &&
+            physical_size >= 8 && physical_size <= 65536 &&
+            ReadableGuestRange(header_address, virtual_size) &&
+            ReadableGuestRange(container_address, physical_size)) {
+          // Preserve the exact two sections, with an explicit archive marker.
+          // This is a capture, not a substitute shader or a guessed binding.
+          std::vector<uint8_t> capture(virtual_size + 4 + physical_size);
+          std::memcpy(capture.data(), virtual_header, virtual_size);
+          std::memcpy(capture.data() + virtual_size,
+                      reinterpret_cast<const uint8_t*>(virtual_header) + 8, 4);
+          std::memcpy(capture.data() + virtual_size + 4, microcode, physical_size);
+          DumpMissingShader(reinterpret_cast<const ShaderContainer*>(capture.data()),
+              capture.size(), expected_stage,
+              XXH3_64bits(capture.data(), capture.size()), "placement-containers");
+        }
+      }
+    }
+    if (g_shader_warning_count.fetch_add(1, std::memory_order_relaxed) < 16) {
+      const auto* words = reinterpret_cast<const be_u32*>(microcode);
+      REXLOG_WARN("Native GPU: placement {} shader 0x{:08X} backing "
+                  "0x{:08X} prefix=0x{:016X} words={:08X} {:08X} "
+                  "{:08X} {:08X} missing from AOT index",
+                  expected_stage == ShaderStage::kVertex ? "vertex" : "pixel",
+                  guest_address, container_address, XXH3_64bits(microcode, 8),
+                  static_cast<u32>(words[0]), static_cast<u32>(words[1]),
+                  static_cast<u32>(words[2]), static_cast<u32>(words[3]));
+    }
+    return nullptr;
+  }
+  auto resource = std::make_shared<ShaderResource>();
+  resource->guest_address = guest_address;
+  resource->stage = expected_stage;
+  resource->hash = cache_entry->hash;
+  resource->texture_mask = FindShaderTextureMask(resource->hash);
+  resource->cache_entry = cache_entry;
+  resource->owns_guest_memory = false;
+  resource->physical_code_address = container_address;
+  for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) {
+    const auto& entry = g_shaderMicrocodeEntries[i];
+    if (entry.containerHash == resource->hash &&
+        entry.stage == (expected_stage == ShaderStage::kPixel ? 1u : 0u))
+      resource->physical_code_size = std::max(resource->physical_code_size, entry.microcodeSize);
+  }
+  {
+    std::lock_guard lock(g_registry_mutex);
+    const auto [it, inserted] = g_registry.emplace(guest_address, resource);
+    if (!inserted) {
+      return it->second;
+    }
+  }
+  REXLOG_INFO("Native GPU: adopted {} shader 0x{:08X}, hash=0x{:016X}",
+              expected_stage == ShaderStage::kVertex ? "vertex" : "pixel",
+              guest_address, resource->hash);
+  return resource;
+}
+
+void DumpMissingShader(const ShaderContainer* container, size_t byte_length,
+                       ShaderStage stage, u64 hash, const char* category) {
+  char* root_buffer = nullptr;
+  size_t root_length = 0;
+  if (_dupenv_s(&root_buffer, &root_length, "LEGO_DUMP_MISSING_SHADERS") != 0 ||
+      !root_buffer || root_length <= 1) {
+    std::free(root_buffer);
+    return;
+  }
+  std::unique_ptr<char, decltype(&std::free)> root_owner(root_buffer,
+                                                          &std::free);
+  const char* root = root_owner.get();
+  std::error_code error;
+  const std::filesystem::path directory = std::filesystem::path(root) / category;
+  std::filesystem::create_directories(directory, error);
+  if (error) {
+    REXLOG_WARN("Native GPU: cannot create shader dump directory '{}': {}",
+                root, error.message());
+    return;
+  }
+  char name[48];
+  std::snprintf(name, sizeof(name), "%s-%016llX.bin",
+                stage == ShaderStage::kVertex ? "vs" : "ps",
+                static_cast<unsigned long long>(hash));
+  const auto path = directory / name;
+  if (std::filesystem::exists(path, error)) {
+    return;
+  }
+  std::ofstream output(path, std::ios::binary | std::ios::trunc);
+  output.write(reinterpret_cast<const char*>(container),
+               static_cast<std::streamsize>(byte_length));
+  if (!output) {
+    REXLOG_WARN("Native GPU: failed to dump missing shader '{}'", path.string());
+  } else {
+    REXLOG_INFO("Native GPU: dumped missing shader '{}'", path.string());
+  }
 }
 
 }  // namespace
 
+bool ShouldSkipBoundPixelShader() {
+  // Called under the recording lock; rebuild only when the user changes it.
+  static std::string previous;
+  static std::unordered_set<u64> containers;
+  auto list = rex::cvar::GetFlagByName("skip_pixel_shaders");
+  // Native mode may not link the Xenos command processor that registers its
+  // skip list cvar. The game's Graphics toggle itself is always registered.
+  if (rex::cvar::GetFlagByName("depth_of_field") == "false") {
+    if (!list.empty()) list += ',';
+    list += cheats::kDofPixelShaderHash;
+  }
+  if (list != previous) {
+    previous = list;
+    containers.clear();
+    const auto hashes = ParsePixelShaderFilter(list);
+    for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) {
+      const auto& entry = g_shaderMicrocodeEntries[i];
+      if (entry.stage == 1 && hashes.contains(entry.microcodeHash))
+        containers.insert(entry.containerHash);
+    }
+    REXLOG_INFO("Native GPU: pixel shader filter matches {} AOT containers", containers.size());
+  }
+  return containers.contains(BoundShaderHash(ShaderStage::kPixel));
+}
+
 u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) {
   if (!function || !HostDevice::IsReady()) {
     return 0;
@@ -72,6 +304,7 @@ u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) {
   const ShaderCacheEntry* cache_entry = FindShader(hash);
   if (!cache_entry) {
     REXLOG_WARN("Native GPU: shader not present in AOT cache, hash=0x{:016X}", hash);
+    DumpMissingShader(container, byte_length, stage, hash);
   }
 
   auto* memory = REX_KERNEL_MEMORY();
@@ -89,7 +322,7 @@ u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) {
   header->base_flush = 0xFFFF0000u;
 
   // Preserve the two SDK-private backing-pointer fields observed in TU23.
-  auto* bytes = memory->virtual_membase() + guest_address;
+  auto* bytes = memory->TranslateVirtual<uint8_t*>(guest_address);
   if (stage == ShaderStage::kVertex) {
     *reinterpret_cast<be_u32*>(bytes + 0x20) = function.guest_address();
   } else {
@@ -100,6 +333,7 @@ u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) {
   resource->guest_address = guest_address;
   resource->stage = stage;
   resource->hash = hash;
+  resource->texture_mask = FindShaderTextureMask(hash);
   resource->cache_entry = cache_entry;
   {
     std::lock_guard lock(g_registry_mutex);
@@ -111,10 +345,19 @@ u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) {
 bool BindShader(ShaderStage stage, u32 guest_address) {
   std::shared_ptr<ShaderResource> resource;
   if (guest_address) {
-    resource = FindResource(guest_address);
+    resource = AdoptShader(guest_address, stage);
     if (!resource || resource->stage != stage) {
-      REXLOG_WARN("Native GPU: unrecognized {} shader resource 0x{:08X}",
-                  stage == ShaderStage::kVertex ? "vertex" : "pixel", guest_address);
+      LongProbeEvent("unrecognized_shader", true, "guest=", guest_address,
+          "stage=", static_cast<u32>(stage));
+      {
+        std::lock_guard lock(g_registry_mutex);
+        g_bound_shaders[StageIndex(stage)].reset();
+      }
+      if (g_shader_warning_count.fetch_add(1, std::memory_order_relaxed) < 32) {
+        REXLOG_WARN("Native GPU: unrecognized {} shader resource 0x{:08X}",
+                    stage == ShaderStage::kVertex ? "vertex" : "pixel",
+                    guest_address);
+      }
       return false;
     }
   }
@@ -169,6 +412,62 @@ bool IsNativeShader(u32 guest_address) {
   return g_registry.contains(guest_address);
 }
 
+void InvalidatePlacementShader(u32 guest_address) {
+  auto recording = HostDevice::LockRecording();
+  std::shared_ptr<ShaderResource> previous;
+  {
+    std::lock_guard lock(g_registry_mutex);
+    const auto it = g_registry.find(guest_address);
+    if (it == g_registry.end() || it->second->owns_guest_memory) return;
+    previous = std::move(it->second);
+    g_registry.erase(it);
+  }
+  static std::atomic<u32> logs{0};
+  if (logs.fetch_add(1) < 16)
+    REXLOG_INFO("Native GPU: re-registered placement shader {:08X}, previous hash={:016X}",
+                guest_address, previous->hash);
+  HostDevice::RetireResource(std::move(previous));
+}
+
+u32 InvalidatePoolCopyShaders(PhysicalCopyRange destination) {
+  auto recording = HostDevice::LockRecording();
+  std::vector<std::shared_ptr<ShaderResource>> retired;
+  std::array<u32, 2> rebind{};
+  {
+    std::lock_guard lock(g_registry_mutex);
+    for (auto it = g_registry.begin(); it != g_registry.end();) {
+      const auto& resource = it->second;
+      const u32 header_size = resource->stage == ShaderStage::kVertex ?
+          kVertexShaderGuestSize : kPixelShaderGuestSize;
+      if (!resource->owns_guest_memory &&
+          (GuestRangeOverlapsPoolCopy(resource->physical_code_address, resource->physical_code_size, destination) ||
+           GuestRangeOverlapsPoolCopy(resource->guest_address, header_size, destination))) {
+        const size_t stage = StageIndex(resource->stage);
+        if (g_bound_shaders[stage] == resource) rebind[stage] = resource->guest_address;
+        retired.push_back(std::move(it->second));
+        it = g_registry.erase(it);
+      } else ++it;
+    }
+  }
+  const u32 count = u32(retired.size());
+  for (auto& resource : retired) HostDevice::RetireResource(std::move(resource));
+  if (rebind[0]) BindShader(ShaderStage::kVertex, rebind[0]);
+  if (rebind[1]) BindShader(ShaderStage::kPixel, rebind[1]);
+  return count;
+}
+
+u32 BoundShaderTextureMask(ShaderStage stage) {
+  std::lock_guard lock(g_registry_mutex);
+  const auto& resource = g_bound_shaders[StageIndex(stage)];
+  return resource ? resource->texture_mask : 0;
+}
+
+u64 BoundShaderHash(ShaderStage stage) {
+  std::lock_guard lock(g_registry_mutex);
+  const auto& resource = g_bound_shaders[StageIndex(stage)];
+  return resource ? resource->hash : 0;
+}
+
 u32 AddRefNativeShader(u32 guest_address) {
   std::lock_guard lock(g_registry_mutex);
   if (!g_registry.contains(guest_address)) {
@@ -181,6 +480,7 @@ u32 AddRefNativeShader(u32 guest_address) {
 }
 
 u32 ReleaseNativeShader(u32 guest_address) {
+  auto recording = HostDevice::LockRecording();
   std::shared_ptr<ShaderResource> released;
   {
     std::lock_guard lock(g_registry_mutex);
@@ -199,7 +499,9 @@ u32 ReleaseNativeShader(u32 guest_address) {
     g_registry.erase(it);
   }
   HostDevice::RetireResource(released);
-  REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
+  if (released->owns_guest_memory) {
+    REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
+  }
   return 0;
 }
 
@@ -210,7 +512,9 @@ void ResetShaderResources() {
     g_bound_shaders = {};
     guest_addresses.reserve(g_registry.size());
     for (const auto& [guest_address, resource] : g_registry) {
-      guest_addresses.push_back(guest_address);
+      if (resource->owns_guest_memory) {
+        guest_addresses.push_back(guest_address);
+      }
     }
     g_registry.clear();
   }
diff --git a/rexlego/src/gpu_native/shaders.h b/rexlego/src/gpu_native/shaders.h
index 10ee448..1ae2e84 100644
--- a/rexlego/src/gpu_native/shaders.h
+++ b/rexlego/src/gpu_native/shaders.h
@@ -4,6 +4,7 @@
 #include <memory>
 
 #include <rex/types.h>
+#include "gpu_native/pool_copy.h"
 
 namespace plume {
 struct RenderShader;
@@ -27,10 +28,18 @@ bool BindShader(ShaderStage stage, u32 guest_address);
 // A null result marks the containing render pass as unsupported by native GPU.
 plume::RenderShader* ResolveBoundShader(ShaderStage stage, u32 spec_constants);
 u32 BoundShaderAddress(ShaderStage stage);
+u32 BoundShaderTextureMask(ShaderStage stage);
+u64 BoundShaderHash(ShaderStage stage);
+// Same microcode skip list used by the existing Graphics/DoF setting.
+bool ShouldSkipBoundPixelShader();
 
 bool IsNativeShader(u32 guest_address);
 u32 AddRefNativeShader(u32 guest_address);
 u32 ReleaseNativeShader(u32 guest_address);
+// XDK placement storage can be re-registered at the same guest address.
+// The next bind must resolve its new physical code rather than a stale record.
+void InvalidatePlacementShader(u32 guest_address);
+u32 InvalidatePoolCopyShaders(PhysicalCopyRange destination);
 
 // Must run before the Plume device is destroyed.
 void ResetShaderResources();
diff --git a/rexlego/src/gpu_native/shaders/depth_pack_ps.hlsl b/rexlego/src/gpu_native/shaders/depth_pack_ps.hlsl
new file mode 100644
index 0000000..9550024
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders/depth_pack_ps.hlsl
@@ -0,0 +1,12 @@
+#include "copy_common.hlsli"
+Texture2D<float4> g_Texture2DDescriptorHeap[] : register(t0, space0);
+
+float4 main(float4 position : SV_Position) : SV_Target {
+  float depth = g_Texture2DDescriptorHeap[g_PushConstants.ResourceDescriptorIndex]
+      .Load(int3(int2(position.xy), 0)).x;
+  uint packed = uint(round(saturate(depth) * 16777215.0)) << 8;
+  // Raw k_24_8 memory viewed as k_8_8_8_8. Both views have the same endian.
+  // The current depth sampling mirror has no stencil plane; preserve depth.
+  return float4(packed & 255, (packed >> 8) & 255,
+      (packed >> 16) & 255, packed >> 24) / 255.0;
+}
diff --git a/rexlego/src/gpu_native/shaders/depth_restore_ps.hlsl b/rexlego/src/gpu_native/shaders/depth_restore_ps.hlsl
new file mode 100644
index 0000000..a523cbd
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders/depth_restore_ps.hlsl
@@ -0,0 +1,13 @@
+#include "copy_common.hlsli"
+Texture2D<float4> g_Texture2DDescriptorHeap[] : register(t0, space0);
+
+float main(float4 position : SV_Position) : SV_Depth {
+  uint2 pixel = uint2(position.xy);
+  // Color/depth EDRAM bank ordering differs by a 40-sample half-tile.
+  pixel.x = pixel.x % 80 < 40 ? pixel.x + 40 : pixel.x - 40;
+  uint4 bytes = uint4(round(saturate(
+      g_Texture2DDescriptorHeap[g_PushConstants.ResourceDescriptorIndex]
+          .Load(int3(pixel, 0))) * 255.0));
+  uint depth = bytes.y | (bytes.z << 8) | (bytes.w << 16);
+  return float(depth + (depth >> 23)) * (1.0 / 16777216.0);
+}
diff --git a/rexlego/src/gpu_native/shaders/resolve_color_ps.hlsl b/rexlego/src/gpu_native/shaders/resolve_color_ps.hlsl
new file mode 100644
index 0000000..5a34a90
--- /dev/null
+++ b/rexlego/src/gpu_native/shaders/resolve_color_ps.hlsl
@@ -0,0 +1,12 @@
+#include "copy_common.hlsli"
+
+Texture2D<float4> g_Texture2DDescriptorHeap[] : register(t0, space0);
+
+float4 main(float4 position : SV_Position) : SV_Target {
+  // Exact texel resolve, not filtered presentation. Render-target conversion
+  // performs the destination UNORM clamping/quantization.
+  return g_Texture2DDescriptorHeap[g_PushConstants.ResourceDescriptorIndex]
+      .Load(int3(int2(position.xy) + int2(
+          int(g_PushConstants.ResourceDescriptorIndex2 << 16) >> 16,
+          int(g_PushConstants.ResourceDescriptorIndex2) >> 16), 0)) * g_PushConstants.Param0;
+}
diff --git a/rexlego/src/gpu_native/state.cpp b/rexlego/src/gpu_native/state.cpp
index 2af61fd..91a776d 100644
--- a/rexlego/src/gpu_native/state.cpp
+++ b/rexlego/src/gpu_native/state.cpp
@@ -52,6 +52,24 @@ void BindDepthStencil(u32 surface) {
   g_bindings.depth_stencil = surface;
 }
 
+void BeginExportBinding(u32 index, u32 resource, u32 format) {
+  std::lock_guard lock(g_bindings_mutex);
+  g_bindings.export_index = index;
+  g_bindings.export_resource = resource;
+  g_bindings.export_format = format;
+}
+
+void EndExportBinding(u32 index, u32 resource, u32 format) {
+  std::lock_guard lock(g_bindings_mutex);
+  if (g_bindings.export_index == index &&
+      g_bindings.export_resource == resource &&
+      g_bindings.export_format == format) {
+    g_bindings.export_index = 0;
+    g_bindings.export_resource = 0;
+    g_bindings.export_format = 0;
+  }
+}
+
 u32 BoundRenderTarget(u32 index) {
   if (index >= kNativeRenderTargets) {
     return 0;
diff --git a/rexlego/src/gpu_native/state.h b/rexlego/src/gpu_native/state.h
index 6385550..c2f5d81 100644
--- a/rexlego/src/gpu_native/state.h
+++ b/rexlego/src/gpu_native/state.h
@@ -8,7 +8,9 @@
 
 namespace legodimensions::gpu_native {
 
-constexpr u32 kNativeTextureSlots = 16;
+// TU23 samplers map directly to Xenos tfetch indices. Captured VS code uses
+// slot 16, above the sixteen pixel slots; slots 26..31 are vertex fetches.
+constexpr u32 kNativeTextureSlots = 26;
 constexpr u32 kNativeVertexStreams = 16;
 constexpr u32 kNativeRenderTargets = 4;
 
@@ -25,6 +27,9 @@ struct DrawBindings {
   u32 depth_stencil = 0;
   u32 index_buffer = 0;
   u32 vertex_declaration = 0;
+  u32 export_resource = 0;
+  u32 export_index = 0;
+  u32 export_format = 0;
 };
 
 void ResetDrawBindings();
@@ -34,6 +39,8 @@ void BindIndexBuffer(u32 buffer);
 void BindVertexDeclaration(u32 declaration);
 void BindRenderTarget(u32 index, u32 surface);
 void BindDepthStencil(u32 surface);
+void BeginExportBinding(u32 index, u32 resource, u32 format);
+void EndExportBinding(u32 index, u32 resource, u32 format);
 u32 BoundRenderTarget(u32 index);
 u32 BoundDepthStencil();
 DrawBindings SnapshotDrawBindings();
diff --git a/rexlego/src/gpu_native/stencil_state.h b/rexlego/src/gpu_native/stencil_state.h
new file mode 100644
index 0000000..182445b
--- /dev/null
+++ b/rexlego/src/gpu_native/stencil_state.h
@@ -0,0 +1,30 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct StencilFaceState {
+  uint32_t comparison = 7, fail = 0, pass = 0, depth_fail = 0;
+  bool operator==(const StencilFaceState&) const = default;
+};
+struct StencilState {
+  bool enabled = false;
+  StencilFaceState front, back;
+  uint32_t reference = 0, read_mask = 255, write_mask = 255;
+};
+constexpr StencilState DecodeStencilState(uint32_t control, uint32_t ref_mask,
+                                         bool has_stencil, bool polygonal) {
+  StencilState result;
+  result.enabled = has_stencil && (control & 1u);
+  if (!result.enabled) return result;
+  result.front = {(control >> 8) & 7u, (control >> 11) & 7u,
+                  (control >> 14) & 7u, (control >> 17) & 7u};
+  result.back = polygonal && (control & 128u)
+      ? StencilFaceState{(control >> 20) & 7u, (control >> 23) & 7u,
+                         (control >> 26) & 7u, (control >> 29) & 7u}
+      : result.front;
+  result.reference = ref_mask & 255u;
+  result.read_mask = (ref_mask >> 8) & 255u;
+  result.write_mask = (ref_mask >> 16) & 255u;
+  return result;
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/texture_number.h b/rexlego/src/gpu_native/texture_number.h
new file mode 100644
index 0000000..e4a775d
--- /dev/null
+++ b/rexlego/src/gpu_native/texture_number.h
@@ -0,0 +1,22 @@
+#pragma once
+#include <cmath>
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+// Host storage is UNORM16 even when the guest resolves integer values into it.
+inline float Unsigned16ResolveScale(int exponent, bool integer) {
+  return std::ldexp(integer ? (1.0f / 65535.0f) : 1.0f, exponent);
+}
+inline float Unsigned16SampleScale(int exponent, bool integer) {
+  return std::ldexp(integer ? 65535.0f : 1.0f, exponent);
+}
+inline int ColorExponentBias(uint32_t color_info) {
+  const uint32_t value = (color_info >> 20) & 63;
+  return value < 32 ? int(value) : int(value) - 64;
+}
+inline float ColorOutputScale(uint32_t color_info) {
+  const uint32_t format = (color_info >> 16) & 15;
+  return std::ldexp(1.0f, ColorExponentBias(color_info) -
+      ((format == 4 || format == 5) ? 5 : 0));
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/texture_upload_cache.h b/rexlego/src/gpu_native/texture_upload_cache.h
new file mode 100644
index 0000000..550e000
--- /dev/null
+++ b/rexlego/src/gpu_native/texture_upload_cache.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <cstdint>
+#include <span>
+#include <xxhash.h>
+
+namespace legodimensions::gpu_native {
+
+struct TextureUploadSourceKey {
+  uint64_t base_hash;
+  uint64_t mip_hash;
+  uint64_t fetch_hash;
+  uint64_t base_size;
+  uint64_t mip_size;
+  bool operator==(const TextureUploadSourceKey&) const = default;
+};
+
+// The caller checks residency BEFORE constructing these spans. Hash all base
+// faces and the whole mip tail on every use, not only once per frame: CPU-side
+// texture changes must remain visible without an explicit dirty notification.
+inline TextureUploadSourceKey MakeTextureUploadSourceKey(
+    std::span<const uint8_t> base, std::span<const uint8_t> mips,
+    std::span<const uint8_t> fetch) {
+  return {XXH3_64bits(base.data(), base.size()),
+          XXH3_64bits(mips.data(), mips.size()),
+          XXH3_64bits(fetch.data(), fetch.size()), base.size(), mips.size()};
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/texture_volume_upload.h b/rexlego/src/gpu_native/texture_volume_upload.h
new file mode 100644
index 0000000..6ff4827
--- /dev/null
+++ b/rexlego/src/gpu_native/texture_volume_upload.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <span>
+
+namespace legodimensions::gpu_native {
+
+// Source addressing comes from the guest layout (including 3D tiling and mip
+// tail offsets). D3D12 upload slices use the actual block-row count, not the
+// guest's tile-aligned Z stride. Keep decoding in cacheable CPU memory.
+template <typename SourceOffset>
+bool CopyTextureVolumeBlocks(std::span<const uint8_t> source,
+                             std::span<uint8_t> destination,
+                             uint32_t width_blocks, uint32_t height_blocks,
+                             uint32_t depth, uint32_t block_bytes,
+                             uint32_t row_pitch, uint32_t endian_xor,
+                             SourceOffset source_offset) {
+  if (!width_blocks || !height_blocks || !depth || !block_bytes || endian_xor > 3 ||
+      uint64_t(width_blocks) * block_bytes > row_pitch) return false;
+  const uint64_t slice_pitch = uint64_t(row_pitch) * height_blocks;
+  if (slice_pitch > destination.size() / depth) return false;
+  for (uint32_t z = 0; z < depth; ++z) {
+    for (uint32_t y = 0; y < height_blocks; ++y) {
+      for (uint32_t x = 0; x < width_blocks; ++x) {
+        const int64_t offset = source_offset(x, y, z);
+        if (offset < 0) return false;
+        const uint64_t dst = z * slice_pitch + uint64_t(y) * row_pitch +
+                             uint64_t(x) * block_bytes;
+        for (uint32_t byte = 0; byte < block_bytes; ++byte) {
+          const uint64_t src = (uint64_t(offset) + byte) ^ endian_xor;
+          if (src >= source.size()) return false;
+          destination[size_t(dst + byte)] = source[size_t(src)];
+        }
+      }
+    }
+  }
+  return true;
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index 4fbcccc..8a80655 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -1,33 +1,74 @@
 #include "gpu_native/textures.h"
 
 #include <algorithm>
+#include <array>
+#include <atomic>
+#include <chrono>
 #include <cstring>
+#include <cmath>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
 #include <memory>
 #include <mutex>
 #include <unordered_map>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
 #include <plume_render_interface.h>
+#include <plume_d3d12.h>
+// Same XXH3 result, compiled for the game's ISA rather than the SDK library.
+#define XXH_INLINE_ALL
+#include <xxhash.h>
 #include <rex/logging.h>
+#include <rex/graphics/pipeline/texture/util.h>
 #include <rex/system/kernel_state.h>
 #include <rex/system/xmemory.h>
 
 #include "gpu_native/d3d.h"
 #include "gpu_native/device.h"
+#include "gpu_native/shaders.h"
 #include "gpu_native/format.h"
+#include "gpu_native/texture_number.h"
+#include "gpu_native/texture_upload_cache.h"
+#include "gpu_native/texture_volume_upload.h"
+#include "gpu_native/long_probe.h"
+#include "gpu_native/memory_watch.h"
+#include "gpu_native/depth_alias.h"
 
 namespace legodimensions::gpu_native {
 namespace {
 
 constexpr u32 kPitchAlignment = 0x100;
 
+struct FramebufferKey {
+  std::array<u32, 4> colors{};
+  u32 depth = 0;
+  bool operator==(const FramebufferKey&) const = default;
+};
+struct FramebufferKeyHash {
+  size_t operator()(const FramebufferKey& key) const {
+    size_t hash = key.depth;
+    for (u32 color : key.colors) hash = hash * 16777619u ^ color;
+    return hash;
+  }
+};
+struct SurfaceStorage {
+  std::unique_ptr<plume::RenderTexture> texture;
+  std::unique_ptr<plume::RenderTextureView> view;
+};
+
 struct TextureResource {
   u32 guest_address = 0;
   u32 mirror_address = 0;
   u32 mirror_size = 0;
   u32 width = 0;
   u32 height = 0;
+  u32 host_width = 0;
+  u32 host_height = 0;
+  u32 HostWidth() const { return host_width ? host_width : width; }
+  u32 HostHeight() const { return host_height ? host_height : height; }
   u32 depth = 1;
   u32 levels = 1;
   u32 guest_format = 0;
@@ -37,18 +78,103 @@ struct TextureResource {
   std::mutex mutex;
   std::unique_ptr<plume::RenderTexture> texture;
   std::unique_ptr<plume::RenderTextureView> view;
-  std::unordered_map<u32, std::unique_ptr<plume::RenderFramebuffer>>
+  std::unique_ptr<plume::RenderTextureView> resolved_view;
+  std::unique_ptr<plume::RenderTexture> sampled_texture;
+  std::unique_ptr<plume::RenderTextureView> sampled_view;
+  u32 sampled_descriptor_index = ~0u;
+  bool sampled_valid = false;
+  std::unordered_map<FramebufferKey, std::shared_ptr<plume::RenderFramebuffer>, FramebufferKeyHash>
       framebuffers;
+  // Old immutable SRV slots stay reserved until release, even after their
+  // texture storage retires. Never overwrite a descriptor used by an old list.
+  std::vector<u32> previous_surface_descriptors;
   u32 descriptor_index = ~u32{0};
+  u32 resolved_descriptor_index = ~u32{0};
+  bool owns_guest_memory = true;
+  u32 adopted_header_type = 0;
+  rex::graphics::xenos::xe_gpu_texture_fetch_t guest_fetch{};
+  bool guest_uploaded = false;
+  u64 guest_content_hash = 0;
+  TextureUploadSourceKey upload_source_key{};
+  bool upload_source_key_valid = false;
+  CpuMemoryStamp cpu_stamp;
+  bool resolved_on_host = false;
+  u32 edram_base = ~0u;
+  u32 guest_msaa = 0;
+  u64 write_generation = 0;
+  u64 depth_resolve_generation = 0;
+  u64 depth_alias_generation = 0;
 };
 
+std::unique_ptr<plume::RenderTexture> CreateHostTexture(
+    const TextureResource& resource, u32 usage, u32 multi_sample);
+
 std::mutex g_textures_mutex;
 std::unordered_map<u32, std::shared_ptr<TextureResource>> g_textures;
+std::atomic<u32> g_texture_lifecycle_logs{0};
+TextureUploadTiming g_upload_timing;  // Serialized by LockRecording.
+u64 g_depth_alias_generation = 0;  // Serialized by LockRecording.
+std::unordered_map<u32, std::weak_ptr<TextureResource>> g_depth_resolves;
+std::unordered_map<u32, std::weak_ptr<TextureResource>> g_edram_colors;
+
+bool DepthAliasesEnabled() {
+  // Validated native correctness path. Explicit 0 remains available for A/B.
+  static const bool enabled = [] {
+    const char* value = std::getenv("LEGO_NATIVE_DEPTH_ALIAS");
+    return !value || std::strcmp(value, "0") != 0;
+  }();
+  return enabled;
+}
+
+DepthAliasLayout AliasLayout(const TextureResource& resource) {
+  const auto& fetch = resource.guest_fetch;
+  const u32 address = fetch.base_address << 12;
+  return {address < 0x20000000 ? address : REX_KERNEL_MEMORY()->GetPhysicalAddress(address),
+      resource.width, resource.height, u32(fetch.pitch), u32(fetch.tiled),
+      u32(fetch.endianness), resource.levels, u32(fetch.dimension)};
+}
+
+struct UploadTimer {
+  bool enabled = NativeTextureTimingEnabled();
+  std::chrono::steady_clock::time_point start;
+  UploadTimer() {
+    if (enabled) {
+      ++g_upload_timing.calls;
+      start = std::chrono::steady_clock::now();
+    }
+  }
+  ~UploadTimer() {
+    if (enabled) g_upload_timing.cpu_ms +=
+        std::chrono::duration<double, std::milli>(
+            std::chrono::steady_clock::now() - start).count();
+  }
+};
+
+// Validate against the SDK allocation/protection table, not VirtualQuery's
+// potentially very large host mapped-region scan on every texture binding.
+// Includes commitment and allocation holes, not just the starting page.
+bool ReadableUploadSpan(u32 address, u32 size) {
+  if (!size) return true;
+  if (!address || size > 128u * 1024 * 1024) return false;
+  const u64 end = u64(address) + size;
+  if (end > 0x100000000ull) return false;
+  auto* memory = REX_KERNEL_MEMORY();
+  const bool physical = address < 0x20000000u;
+  if (physical && end > 0x20000000u) return false;
+  for (u64 at = address; at < end;) {
+    auto* heap = physical ? memory->GetPhysicalHeap() : memory->LookupHeap(u32(at));
+    if (!heap || at < heap->heap_base() ||
+        at >= u64(heap->heap_base()) + heap->heap_size()) return false;
+    const u64 next = std::min(end, u64(heap->heap_base()) + heap->heap_size());
+    if (!heap->IsRangeCommittedReadable(u32(at), u32(next - at))) return false;
+    at = next;
+  }
+  return true;
+}
 
 template <typename T>
 T* GuestAt(u32 guest_address) {
-  return reinterpret_cast<T*>(REX_KERNEL_MEMORY()->virtual_membase() +
-                              guest_address);
+  return REX_KERNEL_MEMORY()->TranslateVirtual<T*>(guest_address);
 }
 
 std::shared_ptr<TextureResource> FindTexture(u32 guest_address) {
@@ -57,6 +183,209 @@ std::shared_ptr<TextureResource> FindTexture(u32 guest_address) {
   return it == g_textures.end() ? nullptr : it->second;
 }
 
+std::shared_ptr<TextureResource> AdoptTexture(u32 guest_address) {
+  if (!guest_address || !HostDevice::IsReady()) {
+    return nullptr;
+  }
+  if (const auto existing = FindTexture(guest_address)) {
+    // Diagnostic only: the authoritative binding is the device fetch shadow,
+    // so don't replace a cached resource solely because its header changed.
+    // Capture both header versions; draw.cpp also traces VS fetch constants.
+    static const bool trace_headers = std::getenv("LEGO_GPU_SNAPSHOT_DIR") != nullptr;
+    if (trace_headers && !existing->owns_guest_memory) {
+      const auto* guest = GuestAt<const D3DTexture>(guest_address);
+      std::array<u32, 6> cached{}, current{};
+      std::memcpy(cached.data(), &existing->guest_fetch, sizeof(cached));
+      for (u32 i = 0; i < current.size(); ++i) current[i] = guest->format.dword[i];
+      if (current != cached) {
+        if (LongProbeEnabled() && LongProbeOnce(
+            (u64(guest_address)<<32) ^ XXH3_64bits(current.data(),sizeof(current))))
+          LongProbeEvent("stale_header_at_use", true, "guest=", guest_address,
+              "cached=", LongProbeHex(cached), "current=", LongProbeHex(current));
+        static std::mutex trace_mutex;
+        static std::unordered_map<u32, std::array<u32, 6>> last_changed;
+        static u32 logs = 0;
+        std::lock_guard trace_lock(trace_mutex);
+        const auto found = last_changed.find(guest_address);
+        if (logs < 32 && (found == last_changed.end() || found->second != current)) {
+          last_changed[guest_address] = current;
+          ++logs;
+          REXLOG_WARN("Native GPU: adopted texture header changed guest={:08X} common={:08X} "
+              "cached={:08X},{:08X},{:08X},{:08X},{:08X},{:08X} "
+              "current={:08X},{:08X},{:08X},{:08X},{:08X},{:08X}",
+              guest_address, u32(guest->resource.common),
+              cached[0], cached[1], cached[2], cached[3], cached[4], cached[5],
+              current[0], current[1], current[2], current[3], current[4], current[5]);
+        }
+      }
+    }
+    return existing;
+  }
+
+  const auto* guest = GuestAt<const D3DTexture>(guest_address);
+  const u32 common = guest->resource.common;
+  const u32 d3d_type = common & 0x1Fu;
+  if (d3d_type != static_cast<u32>(D3DResourceType::kTexture) &&
+      d3d_type != static_cast<u32>(D3DResourceType::kVolumeTexture) &&
+      d3d_type != static_cast<u32>(D3DResourceType::kCubeTexture)) {
+    return nullptr;
+  }
+
+  const u32 word0 = guest->format.dword[0];
+  const u32 word1 = guest->format.dword[1];
+  const u32 word2 = guest->format.dword[2];
+  const u32 word4 = guest->format.dword[4];
+  const u32 word5 = guest->format.dword[5];
+  const u32 dimension = (word5 >> 9) & 3u;
+  auto resource = std::make_shared<TextureResource>();
+  resource->guest_address = guest_address;
+  resource->owns_guest_memory = false;
+  resource->adopted_header_type = d3d_type;
+  u32 fetch_words[6];
+  for (u32 i = 0; i < 6; ++i) fetch_words[i] = guest->format.dword[i];
+  std::memcpy(&resource->guest_fetch, fetch_words, sizeof(fetch_words));
+  resource->guest_format = word1 & 0x3Fu;
+  resource->format = ConvertXenosTextureFormat(resource->guest_format);
+  resource->d3d_type = d3d_type;
+  resource->surface = false;
+  if (dimension == 2) {
+    resource->width = (word2 & 0x7FFu) + 1;
+    resource->height = ((word2 >> 11) & 0x7FFu) + 1;
+    resource->depth = ((word2 >> 22) & 0x3FFu) + 1;
+    resource->d3d_type = static_cast<u32>(D3DResourceType::kVolumeTexture);
+  } else {
+    resource->width = (word2 & 0x1FFFu) + 1;
+    resource->height = ((word2 >> 13) & 0x1FFFu) + 1;
+    resource->depth = dimension == 3 ? 6 : 1;
+    if (dimension == 3) {
+      resource->d3d_type = static_cast<u32>(D3DResourceType::kCubeTexture);
+    }
+  }
+  const u32 mip_page = (word5 >> 12) & 0xFFFFFu;
+  resource->levels = mip_page ? (((word4 >> 6) & 0xFu) + 1) : 1;
+  if (!resource->width || !resource->height ||
+      resource->format == plume::RenderFormat::UNKNOWN) {
+    return nullptr;
+  }
+  resource->texture = CreateHostTexture(*resource, 0, 0);
+  if (!resource->texture) {
+    return nullptr;
+  }
+  resource->texture->setName((std::string("LEGO texture ") + std::to_string(guest_address) +
+      " base " + std::to_string(resource->guest_fetch.base_address << 12)).c_str());
+  {
+    plume::RenderTextureViewDesc view_desc;
+    constexpr plume::RenderSwizzle swizzles[] = {
+        plume::RenderSwizzle::R, plume::RenderSwizzle::G,
+        plume::RenderSwizzle::B, plume::RenderSwizzle::A,
+        plume::RenderSwizzle::ZERO, plume::RenderSwizzle::ONE,
+        plume::RenderSwizzle::ZERO, plume::RenderSwizzle::ZERO};
+    const u32 swizzle = resource->guest_fetch.swizzle;
+    view_desc.componentMapping = plume::RenderComponentMapping(
+        swizzles[swizzle & 7], swizzles[(swizzle >> 3) & 7],
+        swizzles[(swizzle >> 6) & 7], swizzles[(swizzle >> 9) & 7]);
+    view_desc.format = resource->format;
+    view_desc.mipLevels = resource->levels;
+    view_desc.dimension =
+        resource->d3d_type == static_cast<u32>(D3DResourceType::kVolumeTexture)
+            ? plume::RenderTextureViewDimension::TEXTURE_3D
+        : resource->d3d_type == static_cast<u32>(D3DResourceType::kCubeTexture)
+            ? plume::RenderTextureViewDimension::TEXTURE_CUBE
+            : plume::RenderTextureViewDimension::TEXTURE_2D;
+    resource->view = resource->texture->createTextureView(view_desc);
+    if (resource->view) {
+      resource->descriptor_index = HostDevice::RegisterTexture(
+          resource->texture.get(), resource->view.get());
+    }
+    // The CPU upload contains guest BGRA storage, so its SRV needs ZYXW.
+    // A native color resolve instead copies logical RGBA render-target data.
+    // Give that representation its own immutable SRV; applying the guest
+    // storage swizzle a second time swaps red and blue in every resolved image.
+    if (resource->format == plume::RenderFormat::R8G8B8A8_UNORM &&
+        swizzle == 0x60Au) {
+      view_desc.componentMapping = plume::RenderComponentMapping(
+          plume::RenderSwizzle::R, plume::RenderSwizzle::G,
+          plume::RenderSwizzle::B, plume::RenderSwizzle::A);
+      resource->resolved_view = resource->texture->createTextureView(view_desc);
+      if (resource->resolved_view)
+        resource->resolved_descriptor_index = HostDevice::RegisterTexture(
+            resource->texture.get(), resource->resolved_view.get());
+    }
+  }
+  std::shared_ptr<TextureResource> winner;
+  {
+    std::lock_guard lock(g_textures_mutex);
+    const auto [it, inserted] = g_textures.emplace(guest_address, resource);
+    if (!inserted) {
+      winner = it->second;
+    }
+  }
+  if (winner) {
+    HostDevice::UnregisterTexture(resource->descriptor_index);
+    HostDevice::UnregisterTexture(resource->resolved_descriptor_index);
+    return winner;
+  }
+  if (std::getenv("LEGO_NATIVE_RENDERDOC_CAPTURE")) {
+    static std::atomic<u32> capture_logs{0};
+    if (capture_logs.fetch_add(1) < 1024)
+      REXLOG_INFO("Native texture capture: guest={:08X} base={:08X} format={} srv={} resolved_srv={} endian={}",
+          guest_address, u32(resource->guest_fetch.base_address << 12), resource->guest_format,
+          resource->descriptor_index, resource->resolved_descriptor_index, u32(resource->guest_fetch.endianness));
+  }
+  REXLOG_INFO(
+      "Native GPU: adopted guest texture 0x{:08X} {}x{} depth={} dimension={} levels={} "
+      "xenos_format={} pitch={} tiled={} num={} exp={} sign={},{},{},{} swizzle={:03X}",
+      guest_address, resource->width, resource->height, resource->depth, dimension, resource->levels,
+      resource->guest_format, ((word0 >> 22) & 0x1FFu) << 5, word0 >> 31,
+      u32(resource->guest_fetch.num_format), i32(resource->guest_fetch.exp_adjust),
+      u32(resource->guest_fetch.sign_x), u32(resource->guest_fetch.sign_y),
+      u32(resource->guest_fetch.sign_z), u32(resource->guest_fetch.sign_w),
+      u32(resource->guest_fetch.swizzle));
+  return resource;
+}
+
+// Keep the raw UNORM16 storage (including resolve quantization) and expose a
+// separate float SRV with Xenos integer/exp-adjust sampling semantics. This
+// narrow path only covers single-level, unsigned, identity-swizzled RGBA16.
+// It does not redefine CPU uploads or modify the linked game shaders.
+bool UpdateResolvedSampling(TextureResource& resource) {
+  const auto& fetch = resource.guest_fetch;
+  if ((resource.format != plume::RenderFormat::R16G16B16A16_UNORM &&
+       resource.format != plume::RenderFormat::R16G16_UNORM) ||
+      (!fetch.num_format && !fetch.exp_adjust)) return true;
+  if (resource.levels != 1 || resource.d3d_type != u32(D3DResourceType::kTexture) ||
+      fetch.swizzle != 0x688 || u32(fetch.sign_x) || u32(fetch.sign_y) ||
+      u32(fetch.sign_z) || u32(fetch.sign_w)) return false;
+  resource.sampled_valid = false;
+  if (!resource.sampled_texture) {
+    TextureResource description;
+    description.width = resource.width;
+    description.height = resource.height;
+    description.levels = 1;
+    description.format = plume::RenderFormat::R32G32B32A32_FLOAT;
+    resource.sampled_texture = CreateHostTexture(description, 1, 0);
+    if (!resource.sampled_texture) return false;
+    plume::RenderTextureViewDesc view;
+    view.dimension = plume::RenderTextureViewDimension::TEXTURE_2D;
+    view.format = description.format;
+    view.mipLevels = 1;
+    resource.sampled_view = resource.sampled_texture->createTextureView(view);
+    if (!resource.sampled_view) return false;
+    resource.sampled_descriptor_index = HostDevice::RegisterTexture(
+        resource.sampled_texture.get(), resource.sampled_view.get());
+  }
+  if (resource.sampled_descriptor_index == ~0u) return false;
+  const float scale = Unsigned16SampleScale(fetch.exp_adjust, fetch.num_format != 0);
+  if (!HostDevice::ResolveHdrColor(resource.texture.get(), resource.descriptor_index,
+          resource.sampled_texture.get(), resource.width, resource.height, scale,
+          ColorResolveDestination::kFloatRGBA32, false))
+    return false;
+  resource.sampled_valid = true;
+  HostDevice::SnapshotTexture(resource.sampled_texture.get(),
+      "sampled-" + std::to_string(resource.guest_address), false);
+  return true;
+}
+
 void InvalidateFramebufferReferences(u32 guest_address) {
   std::vector<std::shared_ptr<TextureResource>> resources;
   {
@@ -67,8 +396,19 @@ void InvalidateFramebufferReferences(u32 guest_address) {
     }
   }
   for (const auto& resource : resources) {
-    std::lock_guard lock(resource->mutex);
-    resource->framebuffers.erase(guest_address);
+    std::vector<std::shared_ptr<plume::RenderFramebuffer>> retired;
+    {
+      std::lock_guard lock(resource->mutex);
+      for (auto it = resource->framebuffers.begin(); it != resource->framebuffers.end();) {
+        const auto& key = it->first;
+        if (key.depth == guest_address ||
+            std::find(key.colors.begin(), key.colors.end(), guest_address) != key.colors.end()) {
+          retired.push_back(std::move(it->second));
+          it = resource->framebuffers.erase(it);
+        } else ++it;
+      }
+    }
+    for (auto& framebuffer : retired) HostDevice::RetireResource(std::move(framebuffer));
   }
 }
 
@@ -123,6 +463,67 @@ u32 LevelOffset(const TextureResource& resource, u32 level) {
   return offset;
 }
 
+// Opt-in diagnostic: inspect the actual base-level bytes uploaded to D3D12,
+// independently of draw geometry, shaders, swizzle and sampler state.
+void DumpTextureUpload(const TextureResource& resource, const u8* data) {
+  static const std::filesystem::path root = [] {
+    char* value = nullptr;
+    size_t length = 0;
+    _dupenv_s(&value, &length, "LEGO_DUMP_TEXTURE_UPLOADS");
+    std::filesystem::path result(value ? value : "");
+    std::free(value);
+    return result;
+  }();
+  if (root.empty()) return;
+  static const bool logos_only = std::getenv("LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY") != nullptr;
+  if (logos_only) {
+    const u64 vs = BoundShaderHash(ShaderStage::kVertex);
+    const u64 ps = BoundShaderHash(ShaderStage::kPixel);
+    if (!((vs == 0x91007ACB3E640E3Dull && ps == 0xEEE573BE160E037Dull) ||
+          (vs == 0xC3958E2D1B795ED9ull && ps == 0x0C1840BF35E84F2Full))) return;
+  }
+  const bool bc1 = resource.format == plume::RenderFormat::BC1_UNORM;
+  const bool bc3 = resource.format == plume::RenderFormat::BC3_UNORM;
+  const bool compressed = bc1 || bc3;
+  if (!compressed && resource.format != plume::RenderFormat::R8G8B8A8_UNORM) return;
+  static std::mutex dump_mutex;
+  static std::unordered_set<u32> dumped;
+  std::lock_guard lock(dump_mutex);
+  if (dumped.size() >= 8 || !dumped.insert(resource.guest_address).second) return;
+  std::error_code error;
+  std::filesystem::create_directories(root, error);
+  if (error) return;
+  // Legacy DDS, one mip, little-endian host. Strip D3D12 row padding.
+  u32 header[32]{};
+  header[0] = 0x20534444; header[1] = 124;
+  header[2] = 0x1007 | (compressed ? 0x80000 : 0x8);
+  header[3] = resource.height; header[4] = resource.width;
+  const u32 row_bytes = compressed ? ((resource.width + 3) / 4) * (bc1 ? 8 : 16) : resource.width * 4;
+  const u32 rows = RowCount(resource.format, resource.height);
+  header[5] = compressed ? row_bytes * rows : row_bytes;
+  header[19] = 32; header[20] = compressed ? 4 : 0x41;
+  header[21] = bc1 ? 0x31545844 : bc3 ? 0x35545844 : 0;
+  if (!compressed) {
+    header[22] = 32; header[23] = 0xFF; header[24] = 0xFF00;
+    header[25] = 0xFF0000; header[26] = 0xFF000000;
+  }
+  header[27] = 0x1000;
+  const std::string name = std::to_string(resource.guest_address);
+  std::ofstream file(root / (name + ".dds"), std::ios::binary);
+  file.write(reinterpret_cast<const char*>(header), sizeof(header));
+  for (u32 row = 0; row < rows; ++row)
+    file.write(reinterpret_cast<const char*>(data + row * RowPitch(resource.format, resource.width)), row_bytes);
+  std::ofstream metadata(root / (name + ".txt"));
+  const auto& fetch = resource.guest_fetch;
+  metadata << "guest=" << std::hex << resource.guest_address
+           << " base=" << (fetch.base_address << 12) << " mip=" << (fetch.mip_address << 12)
+           << " swizzle=" << fetch.swizzle << std::dec << " endian=" << u32(fetch.endianness)
+           << " pitch=" << (fetch.pitch << 5) << " tiled=" << fetch.tiled
+           << " packed=" << fetch.packed_mips << '\n';
+  REXLOG_INFO("Native GPU: captured texture upload {} ({}x{}, format={})",
+              name, resource.width, resource.height, resource.guest_format);
+}
+
 bool EnsureMirror(TextureResource& resource) {
   if (resource.mirror_address) {
     return true;
@@ -158,8 +559,8 @@ std::unique_ptr<plume::RenderTexture> CreateHostTexture(
   plume::RenderTextureDesc desc;
   desc.dimension = volume ? plume::RenderTextureDimension::TEXTURE_3D
                           : plume::RenderTextureDimension::TEXTURE_2D;
-  desc.width = resource.width;
-  desc.height = resource.height;
+  desc.width = resource.HostWidth();
+  desc.height = resource.HostHeight();
   desc.depth = volume ? resource.depth : 1;
   desc.mipLevels = resource.levels;
   desc.arraySize = cube ? 6 : 1;
@@ -170,13 +571,21 @@ std::unique_ptr<plume::RenderTexture> CreateHostTexture(
   }
   if (IsDepthFormat(resource.format)) {
     desc.flags |= plume::RenderTextureFlag::DEPTH_TARGET;
-  } else if (resource.surface || usage != 0 ||
-             IsRenderTargetFormat(resource.format)) {
+  } else if (IsRenderTargetFormat(resource.format)) {
     desc.flags |= plume::RenderTextureFlag::RENDER_TARGET;
   }
   desc.committed = resource.surface || IsDepthFormat(resource.format) ||
                    usage != 0 || multi_sample != 0;
-  return device->createTexture(desc);
+  auto texture = device->createTexture(desc);
+  // Plume may return a wrapper even when D3D12 resource creation failed.
+  // Do not dereference that wrapper through setName or createTextureView.
+  if (texture && !static_cast<plume::D3D12Texture*>(texture.get())->d3d) {
+    REXLOG_ERROR("Native GPU: texture allocation failed {}x{} depth={} mips={} format={}",
+                 desc.width, desc.height, desc.depth, desc.mipLevels,
+                 static_cast<u32>(desc.format));
+    return nullptr;
+  }
+  return texture;
 }
 
 u32 AllocateResource(const std::shared_ptr<TextureResource>& resource,
@@ -201,7 +610,7 @@ u32 AllocateResource(const std::shared_ptr<TextureResource>& resource,
   }
   resource->texture->setName(resource->surface ? "LEGO guest surface"
                                                : "LEGO guest texture");
-  if (!IsDepthFormat(resource->format)) {
+  {
     plume::RenderTextureViewDesc view_desc;
     view_desc.format = resource->format;
     view_desc.mipLevels = resource->levels;
@@ -237,11 +646,35 @@ u32 AllocateResource(const std::shared_ptr<TextureResource>& resource,
     std::lock_guard lock(g_textures_mutex);
     g_textures.emplace(resource->guest_address, resource);
   }
+  if (g_texture_lifecycle_logs.fetch_add(1, std::memory_order_relaxed) < 64) {
+    REXLOG_INFO(
+        "Native GPU: created {} 0x{:08X} {}x{} levels={} format=0x{:08X}",
+        resource->surface ? "surface" : "texture", resource->guest_address,
+        resource->width, resource->height, resource->levels,
+        resource->guest_format);
+  }
   return resource->guest_address;
 }
 
 }  // namespace
 
+bool NativeTextureTimingEnabled() {
+  static const bool enabled = [] {
+    char* value = nullptr;
+    size_t size = 0;
+    _dupenv_s(&value, &size, "LEGO_NATIVE_TIMING");
+    const bool result = value && *value;
+    std::free(value);
+    return result;
+  }();
+  return enabled;
+}
+
+TextureUploadTiming ConsumeTextureUploadTiming() {
+  auto recording = HostDevice::LockRecording();
+  return std::exchange(g_upload_timing, {});
+}
+
 u32 CreateTextureResource(u32 width, u32 height, u32 depth, u32 levels,
                           u32 usage, u32 guest_format, u32 /*pool*/,
                           u32 d3d_type) {
@@ -261,7 +694,7 @@ u32 CreateTextureResource(u32 width, u32 height, u32 depth, u32 levels,
 }
 
 u32 CreateSurfaceResource(u32 width, u32 height, u32 guest_format,
-                          u32 multi_sample, u32 /*parameters*/) {
+                          u32 multi_sample, u32 parameters) {
   if (!HostDevice::IsReady()) {
     return 0;
   }
@@ -274,7 +707,74 @@ u32 CreateSurfaceResource(u32 width, u32 height, u32 guest_format,
   resource->d3d_type = static_cast<u32>(D3DResourceType::kSurface);
   resource->format = ConvertGuestTextureFormat(guest_format);
   resource->surface = true;
-  return AllocateResource(resource, 1, multi_sample);
+  if (parameters && !ReadableUploadSpan(parameters, 12)) return 0;
+  const u32 address = AllocateResource(resource, 1, multi_sample);
+  if (!address) return 0;
+  u32 texture_format = guest_format & 63;
+  if (texture_format == 54) texture_format = 7;
+  // The TU23 XG surface-header helper uses this read-only XEX format table.
+  const u32 table = *GuestAt<const be_u16>(0x824C9660u + texture_format * 2);
+  u32 render_format = (table >> 8) & 15;
+  const u32 base = parameters ? u32(*GuestAt<const be_u32>(parameters)) & 0xFFF : 0;
+  resource->edram_base = parameters ? base : ~0u;
+  resource->guest_msaa = multi_sample;
+  const bool depth = texture_format == 22 || texture_format == 23;
+  static std::atomic<u32> capture_logs{0};
+  if (std::getenv("LEGO_NATIVE_RENDERDOC_CAPTURE") && capture_logs.fetch_add(1) < 256)
+    REXLOG_INFO("Native surface capture: guest={:08X} extent={}x{} depth={} format={:08X} msaa={} parameters={:08X} base={} hi={:08X}",
+        address, width, height, depth, guest_format, multi_sample, parameters, base,
+        parameters ? u32(*GuestAt<const be_u32>(parameters + 4)) : 0u);
+  resource->texture->setName((std::string("LEGO surface ") + std::to_string(address) +
+      " base " + std::to_string(base)).c_str());
+  const u32 bias = parameters && !depth ? u32(*GuestAt<const be_u32>(parameters + 8)) & 63 : 0;
+  if (!render_format && (guest_format & 0x3FE00) == 0x7E00) render_format = 1;
+  auto* guest = GuestAt<D3DSurface>(address);
+  // +28 is RB_COLOR_INFO for color surfaces, RB_DEPTH_INFO for depth.
+  guest->depth_info = base | ((depth ? (texture_format == 23 ? 1u : 0u) : render_format) << 16) |
+      (depth ? 0u : bias << 20);
+  static std::atomic<u32> logs{0};
+  if (!depth && logs.fetch_add(1) < 24)
+    REXLOG_INFO("Native GPU: surface color info {:08X} guest_fmt={:08X} rt_fmt={} bias={} scale={}",
+        address, guest_format, render_format, ColorExponentBias(guest->depth_info),
+        ColorOutputScale(guest->depth_info));
+  return address;
+}
+
+float SurfaceColorOutputScale(u32 guest_address) {
+  const auto resource = FindTexture(guest_address);
+  return resource && resource->surface && !IsDepthFormat(resource->format)
+      ? ColorOutputScale(GuestAt<const D3DSurface>(guest_address)->depth_info) : 1.0f;
+}
+
+void MarkSurfaceWritten(u32 guest_address) {
+  if (!DepthAliasesEnabled()) return;
+  const auto resource = FindTexture(guest_address);
+  if (!resource || !resource->surface) return;
+  resource->write_generation = ++g_depth_alias_generation;
+  if (resource->edram_base != ~0u && resource->guest_msaa == 0 &&
+      resource->format == plume::RenderFormat::R8G8B8A8_UNORM)
+    g_edram_colors[resource->edram_base] = resource;
+}
+
+bool PrepareSurfaceDepthAlias(u32 guest_address) {
+  if (!DepthAliasesEnabled()) return true;
+  const auto destination = FindTexture(guest_address);
+  if (!destination || !destination->surface || destination->edram_base == ~0u ||
+      destination->guest_msaa != 0 || (destination->guest_format & 63) != 22 ||
+      destination->HostWidth() % 80) return true;
+  const auto found = g_edram_colors.find(destination->edram_base);
+  const auto source = found == g_edram_colors.end() ? nullptr : found->second.lock();
+  if (!source || source->write_generation <= destination->write_generation ||
+      source->HostWidth() != destination->HostWidth() ||
+      source->HostHeight() != destination->HostHeight()) return true;
+  if (!HostDevice::TransferDepthAlias(source->texture.get(), source->descriptor_index,
+      destination->texture.get(), destination->HostWidth(), destination->HostHeight(), true)) return false;
+  destination->write_generation = source->write_generation;
+  static u32 logs = 0;
+  if (logs++ < 24) REXLOG_INFO("Native GPU: EDRAM color->depth alias {:08X}->{:08X} base={} {}x{}",
+      source->guest_address, guest_address, destination->edram_base,
+      destination->HostWidth(), destination->HostHeight());
+  return true;
 }
 
 bool LockTextureResource(u32 guest_address, u32 level, u32 locked_rect,
@@ -342,25 +842,346 @@ bool DescribeSurfaceResource(u32 guest_address, u32 desc_address) {
   return true;
 }
 
-TextureResourceView ResolveTextureResource(u32 guest_address) {
+void RefreshTextureHeader(u32 guest_address) {
+  auto recording = HostDevice::LockRecording();
   const auto resource = FindTexture(guest_address);
+  if (!resource || resource->owns_guest_memory || resource->surface) return;
+  const auto* header = GuestAt<const D3DTexture>(guest_address);
+  std::array<u32, 6> current{}, cached{};
+  for (u32 i = 0; i < current.size(); ++i) current[i] = header->format.dword[i];
+  bool address_only = false;
+  {
+    std::lock_guard lock(resource->mutex);
+    std::memcpy(cached.data(), &resource->guest_fetch, sizeof(cached));
+    const bool same_type = resource->adopted_header_type == (u32(header->resource.common) & 0x1Fu);
+    if (cached == current && same_type) return;
+    // Header addresses are CPU aliases, including the E/F +4 KB mapping.
+    // Keep them raw for UploadTextureResource; SetTexture separately converts
+    // the active device fetch to physical addresses and preserves sampler bits.
+    auto old_layout = cached, new_layout = current;
+    old_layout[1] &= 0xFFFu; new_layout[1] &= 0xFFFu;
+    old_layout[5] &= 0xFFFu; new_layout[5] &= 0xFFFu;
+    // Adding/removing a mip allocation changes the host mip count even when
+    // word4 still advertises the same maximum level.
+    address_only = same_type && old_layout == new_layout &&
+        bool(cached[5] >> 12) == bool(current[5] >> 12);
+    if (address_only) {
+      std::memcpy(&resource->guest_fetch, current.data(), sizeof(current));
+      resource->upload_source_key_valid = false;
+      resource->cpu_stamp = {};
+      resource->depth_alias_generation = 0;
+      // Keep resolved host storage authoritative. Its guest allocation may
+      // move, but uploading CPU memory would erase the native resolve result.
+      if (!resource->resolved_on_host) resource->guest_uploaded = false;
+    }
+  }
+  if (!address_only) {
+    // The same 52-byte header may now describe another size/format/mip chain.
+    // Re-adopt it lazily with fresh storage, retaining old lists at their fences.
+    {
+      std::lock_guard lock(g_textures_mutex);
+      const auto it = g_textures.find(guest_address);
+      if (it == g_textures.end() || it->second != resource) return;
+      g_textures.erase(it);
+    }
+    InvalidateFramebufferReferences(guest_address);
+    HostDevice::UnregisterTexture(resource->descriptor_index);
+    HostDevice::UnregisterTexture(resource->resolved_descriptor_index);
+    HostDevice::UnregisterTexture(resource->sampled_descriptor_index);
+    for (u32 descriptor : resource->previous_surface_descriptors)
+      HostDevice::UnregisterTexture(descriptor);
+    if (resource->mirror_address)
+      REX_KERNEL_MEMORY()->SystemHeapFree(resource->mirror_address);
+    HostDevice::RetireResource(resource);
+  }
+  static std::atomic<u32> refresh_logs{0};
+  if (LongProbeEnabled()) LongProbeEvent("texture_header_refresh", !address_only,
+      "guest=", guest_address, "action=", address_only ? "relocate" : "replace",
+      "host_resolved=", resource->resolved_on_host,
+      "cached=", LongProbeHex(cached), "current=", LongProbeHex(current));
+  if (refresh_logs.fetch_add(1, std::memory_order_relaxed) < 64)
+    REXLOG_INFO("Native GPU: refreshed borrowed texture {:08X} action={} host_resolved={} "
+        "base={:08X}->{:08X} mip={:08X}->{:08X} size={:08X}->{:08X}",
+        guest_address, address_only ? "relocate" : "replace", resource->resolved_on_host,
+        cached[1], current[1], cached[5], current[5], cached[2], current[2]);
+}
+
+TextureResourceView ResolveTextureResource(u32 guest_address) {
+  const auto resource = AdoptTexture(guest_address);
   if (!resource) {
     return {};
   }
+  if (resource->resolved_on_host && resource->sampled_valid)
+    return {resource->sampled_texture.get(), resource->sampled_view.get(),
+            IsDepthFormat(resource->format) ? plume::RenderFormat::R32_FLOAT
+                                          : plume::RenderFormat::R32G32B32A32_FLOAT,
+            resource->sampled_descriptor_index,
+            resource->width, resource->height, resource->d3d_type, false, false};
+  const bool native_color = resource->resolved_on_host &&
+      resource->resolved_descriptor_index != ~u32{0};
   return {resource->texture.get(),
-          resource->view.get(),
+          native_color ? resource->resolved_view.get() : resource->view.get(),
           resource->format,
-          resource->descriptor_index,
-          resource->width,
-          resource->height,
+          native_color ? resource->resolved_descriptor_index : resource->descriptor_index,
+          resource->HostWidth(),
+          resource->HostHeight(),
           resource->d3d_type,
           resource->surface,
           IsDepthFormat(resource->format)};
 }
 
 bool UploadTextureResource(u32 guest_address,
-                           plume::RenderCommandList* commands) {
+                           plume::RenderCommandList* commands,
+                           bool require_content_hash) {
+  auto recording = HostDevice::LockRecording();
+  UploadTimer timer;
   const auto resource = FindTexture(guest_address);
+  if (DepthAliasesEnabled() && resource && commands && !resource->surface &&
+      !resource->owns_guest_memory && !resource->resolved_on_host && resource->guest_format == 6 &&
+      resource->guest_fetch.num_format == 0 && resource->guest_fetch.exp_adjust == 0 &&
+      !(u32(resource->guest_fetch.sign_x) | u32(resource->guest_fetch.sign_y) |
+        u32(resource->guest_fetch.sign_z) | u32(resource->guest_fetch.sign_w))) {
+    const auto layout = AliasLayout(*resource);
+    const auto found = g_depth_resolves.find(layout.physical_base);
+    const auto depth = found == g_depth_resolves.end() ? nullptr : found->second.lock();
+    if (depth && depth->sampled_valid && depth->depth_resolve_generation &&
+        CompatibleDepthAlias(layout, AliasLayout(*depth))) {
+      if (resource->depth_alias_generation != depth->depth_resolve_generation) {
+        if (!HostDevice::TransferDepthAlias(depth->sampled_texture.get(), depth->sampled_descriptor_index,
+            resource->texture.get(), resource->width, resource->height, false)) return false;
+        resource->depth_alias_generation = depth->depth_resolve_generation;
+        // Keep the raw-storage SRV swizzle: this conversion writes packed bytes,
+        // unlike a color resolve, which writes logical RGBA values.
+        static u32 logs = 0;
+        if (logs++ < 24) REXLOG_INFO("Native GPU: depth raw texture alias {:08X}->{:08X} physical={:08X}",
+            depth->guest_address, guest_address, layout.physical_base);
+      }
+      return true;
+    }
+    if (resource->depth_alias_generation) {
+      resource->depth_alias_generation = 0;
+      resource->guest_uploaded = false;
+    }
+  }
+  if (resource && commands && !resource->owns_guest_memory &&
+      !resource->surface && !IsDepthFormat(resource->format)) {
+    namespace tu = rex::graphics::texture_util;
+    using rex::graphics::xenos::DataDimension;
+    std::lock_guard lock(resource->mutex);
+    if (resource->resolved_on_host) return true;
+    static const bool audit_texture_watch = std::getenv("LEGO_NATIVE_AUDIT_TEXTURE_WATCH") != nullptr;
+    const bool watch_hit = !require_content_hash && resource->guest_uploaded &&
+        CpuMemoryUnchanged(resource->cpu_stamp);
+    if (watch_hit && !audit_texture_watch) {
+      if (timer.enabled) ++g_upload_timing.source_hits;
+      return true;
+    }
+    const auto& fetch = resource->guest_fetch;
+    // Cube faces are array slices within each guest mip. A volume mip instead
+    // includes all Z slices in one subresource. Stacked 2D arrays remain separate.
+    const bool cube = fetch.dimension == DataDimension::kCube;
+    const bool volume = fetch.dimension == DataDimension::k3D;
+    if ((!cube && !volume && fetch.dimension != DataDimension::k2DOrStacked) ||
+        fetch.stacked || !fetch.base_address)
+      return false;
+    const u32 faces = cube ? 6 : 1;
+    const u32 block_bytes = plume::RenderFormatSize(resource->format);
+    const u32 block_width = plume::RenderFormatBlockWidth(resource->format);
+    const auto* format_info = rex::graphics::FormatInfo::Get(fetch.format);
+    if (!block_bytes || !block_width ||
+        format_info->bits_per_pixel * format_info->block_width * format_info->block_height != block_bytes * 8)
+      return false;
+    const auto layout = tu::GetGuestTextureLayout(fetch.dimension, fetch.pitch,
+        resource->width, resource->height, volume ? resource->depth : 1, fetch.tiled, fetch.format,
+        fetch.packed_mips, true, resource->levels - 1);
+    const auto guest_bytes = [](u32 address) {
+      return address < 0x20000000u
+          ? REX_KERNEL_MEMORY()->TranslatePhysical<const u8*>(address)
+          : REX_KERNEL_MEMORY()->TranslateVirtual<const u8*>(address);
+    };
+    const auto* base_bytes = guest_bytes(fetch.base_address << 12);
+    const u32 mip_size = resource->levels > 1 ? layout.mips_total_extent_bytes : 0;
+    const auto* mip_bytes = mip_size ? guest_bytes(fetch.mip_address << 12) : nullptr;
+    TextureUploadSourceKey source_key{};
+    const CpuMemorySpan source_spans[] = {
+      {u32(fetch.base_address << 12), layout.base.level_data_extent_bytes},
+      {u32(fetch.mip_address << 12), mip_size}};
+    auto cpu_stamp = WatchCpuMemory(source_spans);
+    const auto source_start = timer.enabled ? std::chrono::steady_clock::now()
+                                            : std::chrono::steady_clock::time_point{};
+    const bool source_key_valid =
+        ReadableUploadSpan(fetch.base_address << 12, layout.base.level_data_extent_bytes) &&
+        (!mip_size || (fetch.mip_address && ReadableUploadSpan(fetch.mip_address << 12, mip_size)));
+    if (LongProbeEnabled() && !source_key_valid && LongProbeOnce(0xBAD0000000000000ull | guest_address))
+      LongProbeEvent("unreadable_texture_source", true, "guest=", guest_address,
+          "base=", u32(fetch.base_address<<12), "mip=", u32(fetch.mip_address<<12));
+    // Never read an unvalidated volume extent. Volumes may have padding
+    // between Z slices, so validating just width*height*depth isn't enough.
+    if (volume && !source_key_valid) return false;
+    if (source_key_valid) {
+      const auto hash_start = timer.enabled ? std::chrono::steady_clock::now()
+                                            : std::chrono::steady_clock::time_point{};
+      source_key = MakeTextureUploadSourceKey(
+          {base_bytes, layout.base.level_data_extent_bytes}, {mip_bytes, mip_size},
+          {reinterpret_cast<const u8*>(&fetch), sizeof(fetch)});
+      if (watch_hit && audit_texture_watch) {
+        static u64 verified = 0, mismatches = 0;
+        ++verified;
+        if (!resource->upload_source_key_valid || resource->upload_source_key != source_key) {
+          ++mismatches;
+          if (mismatches <= 32)
+            REXLOG_ERROR("Native texture watch stale: guest={:08X} base={:08X} mip={:08X}",
+                guest_address, u32(fetch.base_address << 12), u32(fetch.mip_address << 12));
+        }
+        if (verified % 100000 == 0)
+          REXLOG_INFO("Native texture watch audit: verified={} mismatches={}", verified, mismatches);
+      }
+      if (timer.enabled) {
+        g_upload_timing.hash_ms += std::chrono::duration<double, std::milli>(
+            std::chrono::steady_clock::now() - hash_start).count();
+        g_upload_timing.hashed_bytes += u64(layout.base.level_data_extent_bytes) + mip_size;
+        g_upload_timing.source_ms += std::chrono::duration<double, std::milli>(
+            std::chrono::steady_clock::now() - source_start).count();
+      }
+      if (resource->guest_uploaded && resource->upload_source_key_valid &&
+          resource->upload_source_key == source_key) {
+        if (timer.enabled) ++g_upload_timing.source_hits;
+        resource->cpu_stamp = std::move(cpu_stamp);
+        return true;
+      }
+    }
+    resource->upload_source_key_valid = false;
+    u64 upload_size_wide = 0;
+    for (u32 face = 0; face < faces; ++face) {
+      for (u32 level = 0; level < resource->levels; ++level)
+        upload_size_wide = ((upload_size_wide + 511) & ~u64{511}) +
+                           LevelSize(*resource, level);
+    }
+    if (!upload_size_wide || upload_size_wide > UINT32_MAX) return false;
+    const u32 upload_size = static_cast<u32>(upload_size_wide);
+    // Untiling, diagnostics and the decoded-content hash need CPU reads.
+    // Keep those in normal RAM; UPLOAD memory must be a write-only destination.
+    std::vector<u8> decoded(upload_size);
+    auto* mapped = decoded.data();
+    u32 destination_offset = 0;
+    const u32 endian_xor = static_cast<u32>(fetch.endianness) == 1 ? 1
+        : static_cast<u32>(fetch.endianness) == 2 ? 3
+        : static_cast<u32>(fetch.endianness) == 3 ? 2 : 0;
+    for (u32 subresource = 0; subresource < faces * resource->levels; ++subresource) {
+      const u32 face = subresource / resource->levels;
+      const u32 level = subresource % resource->levels;
+      destination_offset = AlignUp(destination_offset, 512);
+      const u32 packed_level = std::min(level, layout.packed_level);
+      const auto& guest_level = level == 0 ? layout.base : layout.mips[packed_level];
+      const u32 base = (level == 0 ? fetch.base_address << 12
+          : (fetch.mip_address << 12) + layout.mip_offsets_bytes[packed_level]) +
+          face * guest_level.array_slice_stride_bytes;
+      // D3D resource headers retain the CPU allocation address, unlike PM4
+      // fetch constants. In particular, the E/F physical aliases include a
+      // 4 KB bias in Memory::TranslateVirtual/GetPhysicalAddress. Masking them
+      // with TranslatePhysical reads the preceding page and scrambles tiles.
+      // Keep support for headers that already contain a physical address.
+      const auto* source = base < 0x20000000u
+          ? REX_KERNEL_MEMORY()->TranslatePhysical<const u8*>(base)
+          : REX_KERNEL_MEMORY()->TranslateVirtual<const u8*>(base);
+      u32 offset_x = 0, offset_y = 0, offset_z = 0;
+      if (fetch.packed_mips)
+        tu::GetPackedMipOffset(resource->width, resource->height,
+                               volume ? resource->depth : 1, fetch.format,
+                               level, offset_x, offset_y, offset_z);
+      const u32 width = MipDimension(resource->width, level);
+      const u32 height = MipDimension(resource->height, level);
+      const u32 pitch = RowPitch(resource->format, width);
+      if (volume) {
+        const u32 block_height = format_info->block_height;
+        const bool copied = CopyTextureVolumeBlocks(
+            {source, guest_level.array_slice_data_extent_bytes},
+            {mapped + destination_offset, LevelSize(*resource, level)},
+            (width + block_width - 1) / block_width,
+            (height + block_height - 1) / block_height,
+            MipDimension(resource->depth, level), block_bytes, pitch, endian_xor,
+            [&](u32 x, u32 y, u32 z) -> int64_t {
+              if (fetch.tiled)
+                return tu::GetTiledOffset3D(x + offset_x, y + offset_y, z + offset_z,
+                    guest_level.row_pitch_bytes / block_bytes,
+                    guest_level.z_slice_stride_block_rows, rex::log2_floor(block_bytes));
+              return (u64(z + offset_z) * guest_level.z_slice_stride_block_rows +
+                      y + offset_y) * guest_level.row_pitch_bytes +
+                     u64(x + offset_x) * block_bytes;
+            });
+        if (!copied) return false;
+        destination_offset += LevelSize(*resource, level);
+        continue;
+      }
+      for (u32 y = 0; y < (height + block_width - 1) / block_width; ++y) {
+        for (u32 x = 0; x < (width + block_width - 1) / block_width; ++x) {
+          const u32 source_offset = fetch.tiled
+              ? tu::GetTiledOffset2D(x + offset_x, y + offset_y,
+                    guest_level.row_pitch_bytes / block_bytes, rex::log2_floor(block_bytes))
+              : (y + offset_y) * guest_level.row_pitch_bytes + (x + offset_x) * block_bytes;
+          for (u32 byte = 0; byte < block_bytes; ++byte)
+            mapped[destination_offset + y * pitch + x * block_bytes + byte] =
+                source[(source_offset + byte) ^ endian_xor];
+        }
+      }
+      destination_offset += LevelSize(*resource, level);
+    }
+    DumpTextureUpload(*resource, mapped);
+    if (timer.enabled) g_upload_timing.converted_bytes += upload_size;
+    const u64 content_hash = XXH3_64bits(mapped, upload_size);
+    if (LongProbeEnabled() && (!resource->guest_uploaded || resource->guest_content_hash != content_hash))
+      LongProbeEvent("texture_upload_changed", false, "guest=", guest_address,
+          "width=", resource->width, "height=", resource->height,
+          "hash=", content_hash, "previous_hash=", resource->guest_content_hash,
+          "bytes=", upload_size, "base=", u32(fetch.base_address<<12),
+          "mip=", u32(fetch.mip_address<<12));
+    if (resource->guest_uploaded && resource->guest_content_hash == content_hash) {
+      resource->upload_source_key = source_key;
+      resource->upload_source_key_valid = source_key_valid;
+      resource->cpu_stamp = std::move(cpu_stamp);
+      return true;
+    }
+    auto upload = std::shared_ptr<plume::RenderBuffer>(HostDevice::Device()->createBuffer(
+        plume::RenderBufferDesc::UploadBuffer(upload_size)).release());
+    if (!upload) return false;
+    auto* upload_bytes = upload->map();
+    if (!upload_bytes) return false;
+    std::memcpy(upload_bytes, decoded.data(), decoded.size());
+    upload->unmap();
+    commands->barriers(plume::RenderBarrierStage::COPY,
+        plume::RenderTextureBarrier(resource->texture.get(), plume::RenderTextureLayout::COPY_DEST));
+    destination_offset = 0;
+    for (u32 subresource = 0; subresource < faces * resource->levels; ++subresource) {
+      const u32 face = subresource / resource->levels;
+      const u32 level = subresource % resource->levels;
+      destination_offset = AlignUp(destination_offset, 512);
+      const u32 width = MipDimension(resource->width, level);
+      const u32 height = MipDimension(resource->height, level);
+      commands->copyTextureRegion(
+          plume::RenderTextureCopyLocation::Subresource(resource->texture.get(), level, face),
+          plume::RenderTextureCopyLocation::PlacedFootprint(upload.get(), resource->format,
+              width, height, volume ? MipDimension(resource->depth, level) : 1,
+              RowPitch(resource->format, width) / block_bytes * block_width,
+              destination_offset));
+      destination_offset += LevelSize(*resource, level);
+    }
+    commands->barriers(plume::RenderBarrierStage::GRAPHICS,
+        plume::RenderTextureBarrier(resource->texture.get(), plume::RenderTextureLayout::SHADER_READ));
+    HostDevice::RetireResource(std::move(upload));
+    resource->guest_uploaded = true;
+    resource->guest_content_hash = content_hash;
+    resource->upload_source_key = source_key;
+    resource->upload_source_key_valid = source_key_valid;
+    resource->cpu_stamp = std::move(cpu_stamp);
+    if (volume) {
+      static std::atomic<u32> volume_uploads{0};
+      if (volume_uploads.fetch_add(1) < 8)
+        REXLOG_INFO("Native GPU: uploaded volume 0x{:08X} {}x{}x{} levels={} bytes={} hash={:016X}",
+                    guest_address, resource->width, resource->height, resource->depth,
+                    resource->levels, upload_size, content_hash);
+    }
+    return true;
+  }
   if (!resource || !commands || resource->surface ||
       !resource->mirror_address || !resource->mirror_size ||
       !resource->texture || IsDepthFormat(resource->format)) {
@@ -446,16 +1267,232 @@ bool UploadTextureResource(u32 guest_address,
 
 bool ResolveTextureFromSurface(u32 destination_texture, u32 source_surface,
                                u32 destination_level,
-                               u32 destination_slice) {
+                               u32 destination_slice, u32 resolve_flags,
+                               const ResolveRect* rectangle, const ResolvePoint* point) {
+  auto recording = HostDevice::LockRecording();
+  if (!PrepareSurfaceDepthAlias(source_surface)) return false;
+  RefreshTextureHeader(destination_texture);
   const auto destination = FindTexture(destination_texture);
+  const auto adopted_destination =
+      destination ? destination : AdoptTexture(destination_texture);
   const auto source = FindTexture(source_surface);
-  if (!destination || !source || !destination->texture || !source->texture ||
-      !source->surface || destination->surface ||
-      destination_level >= destination->levels ||
-      destination->format != source->format) {
+  if (source && adopted_destination && std::getenv("LEGO_NATIVE_RENDERDOC_CAPTURE")) {
+    static std::unordered_set<u64> captured;
+    if (captured.size() < 128 && captured.insert((u64(source_surface) << 32) | destination_texture).second)
+      REXLOG_INFO("Native resolve capture: source={:08X} destination={:08X} base={:08X} format={} flags={:08X}",
+          source_surface, destination_texture, u32(adopted_destination->guest_fetch.base_address << 12),
+          adopted_destination->guest_format, resolve_flags);
+  }
+  const auto region = source && adopted_destination &&
+      destination_level < adopted_destination->levels ? CheckedResolveRegion(
+      source->HostWidth(), source->HostHeight(),
+      MipDimension(adopted_destination->width, destination_level),
+      MipDimension(adopted_destination->height, destination_level), rectangle, point)
+      : std::nullopt;
+  if (LongProbeEnabled()) {
+    const std::array<u32,12> signature{source_surface,destination_texture,resolve_flags,
+        destination_level,destination_slice,region ? region->left : 0,
+        region ? region->top : 0,region ? region->width : 0,region ? region->height : 0,
+        region ? region->x : 0,region ? region->y : 0,source ? source->guest_format : 0};
+    if (HostDevice::LongProbeSnapshotActive() || LongProbeOnce(XXH3_64bits(signature.data(),sizeof(signature))))
+      LongProbeEvent("resolve", !region || !source || !adopted_destination,
+          "signature_src_dst_flags_level_slice_rect_point_format=", LongProbeHex(signature),
+          "dst_width=", adopted_destination ? adopted_destination->width : 0,
+          "dst_height=", adopted_destination ? adopted_destination->height : 0);
+  }
+  if (source && adopted_destination && !region) {
+    static std::atomic<u32> bad_regions{0};
+    if (bad_regions.fetch_add(1) < 16)
+      REXLOG_WARN("Native GPU: rejected invalid resolve region src={:08X} dst={:08X}",
+                  source_surface, destination_texture);
+    return false;
+  }
+  if (!adopted_destination) {
+    static std::atomic<u32> unknown_resolves{0};
+    if (unknown_resolves.fetch_add(1) < 4)
+      LogUnknownTexture(destination_texture, "resolve destination");
+  }
+  if (source && source->texture && !IsDepthFormat(source->format)) {
+    HostDevice::SnapshotTexture(source->texture.get(),
+        "resolve-" + std::to_string(source_surface) + "-to-" +
+        std::to_string(destination_texture), true);
+  }
+  // Xenos k_24_8 sampling returns normalized depth in X. Keep each atlas tile
+  // in a float sampling mirror. Stencil and CPU readback remain unsupported.
+  if (source && adopted_destination && region && source->surface &&
+      !adopted_destination->surface && source->texture &&
+      IsDepthFormat(source->format) && IsDepthFormat(adopted_destination->format) &&
+      (adopted_destination->guest_format & 0x3Fu) == 22 &&
+      destination_level == 0 && destination_slice == 0 &&
+      adopted_destination->levels == 1 &&
+      adopted_destination->d3d_type == u32(D3DResourceType::kTexture) &&
+      !(resolve_flags & 0xFC000000u) &&
+      (adopted_destination->guest_fetch.swizzle == 0xB48 ||
+       adopted_destination->guest_fetch.swizzle == 0x688)) {
+    auto& dst = *adopted_destination;
+    if (!dst.sampled_texture) {
+      auto* device = HostDevice::Device();
+      plume::RenderTextureDesc desc;
+      desc.dimension = plume::RenderTextureDimension::TEXTURE_2D;
+      desc.width = dst.width; desc.height = dst.height;
+      desc.depth = 1; desc.mipLevels = 1; desc.arraySize = 1;
+      desc.multisampling.sampleCount = plume::RenderSampleCount::COUNT_1;
+      desc.format = plume::RenderFormat::R32_FLOAT;
+      desc.flags |= plume::RenderTextureFlag::RENDER_TARGET;
+      desc.committed = true;
+      dst.sampled_texture = device->createTexture(desc);
+      if (!dst.sampled_texture ||
+          !static_cast<plume::D3D12Texture*>(dst.sampled_texture.get())->d3d) return false;
+      plume::RenderTextureViewDesc view;
+      view.dimension = plume::RenderTextureViewDimension::TEXTURE_2D;
+      view.format = plume::RenderFormat::R32_FLOAT;
+      view.mipLevels = 1;
+      if (dst.guest_fetch.swizzle == 0xB48)
+        view.componentMapping = plume::RenderComponentMapping(plume::RenderSwizzle::R,
+            plume::RenderSwizzle::G, plume::RenderSwizzle::ONE, plume::RenderSwizzle::ONE);
+      dst.sampled_view = dst.sampled_texture->createTextureView(view);
+      if (!dst.sampled_view) return false;
+      dst.sampled_descriptor_index = HostDevice::RegisterTexture(
+          dst.sampled_texture.get(), dst.sampled_view.get());
+    }
+    if (!HostDevice::ResolveHdrColor(source->texture.get(), source->descriptor_index,
+        dst.sampled_texture.get(), dst.width, dst.height, 1.0f,
+        ColorResolveDestination::kDepthFloat32, true, &*region)) return false;
+    dst.resolved_on_host = true; dst.sampled_valid = true;
+    if (DepthAliasesEnabled()) {
+      dst.depth_resolve_generation = ++g_depth_alias_generation;
+      g_depth_resolves[AliasLayout(dst).physical_base] = adopted_destination;
+    }
+    HostDevice::SnapshotTexture(dst.sampled_texture.get(),
+        "depth-atlas-" + std::to_string(destination_texture), false);
+    static std::atomic<u32> logs{0};
+    if (logs.fetch_add(1) < 16)
+      REXLOG_INFO("Native GPU: depth atlas src=({},{}) {}x{} dst=({},{}) texture={:08X}",
+          region->left, region->top, region->width, region->height,
+          region->x, region->y, destination_texture);
+    return true;
+  }
+  // Preserve both host depth and stencil planes. Unlike a color subrect copy,
+  // CopyResource requires identical full resource descriptions. It does NOT
+  // implement Xenos 24-bit quantization or guest-memory writeback.
+  if (source && adopted_destination && source->surface &&
+      !adopted_destination->surface && source->texture && adopted_destination->texture &&
+      source->texture.get() != adopted_destination->texture.get() &&
+      IsDepthFormat(source->format) && source->format == adopted_destination->format &&
+      destination_level == 0 && destination_slice == 0 &&
+      !(resolve_flags & 0xFC000000u) && !rectangle && !point) {
+    auto* src = static_cast<plume::D3D12Texture*>(source->texture.get())->d3d;
+    auto* dst = static_cast<plume::D3D12Texture*>(adopted_destination->texture.get())->d3d;
+    if (src && dst) {
+      const auto a = src->GetDesc();
+      const auto b = dst->GetDesc();
+      if (a.Dimension == b.Dimension && a.Width == b.Width && a.Height == b.Height &&
+          a.DepthOrArraySize == 1 && b.DepthOrArraySize == 1 &&
+          a.MipLevels == 1 && b.MipLevels == 1 && a.Format == b.Format &&
+          a.SampleDesc.Count == 1 && b.SampleDesc.Count == 1 &&
+          a.SampleDesc.Quality == b.SampleDesc.Quality) {
+        auto* commands = HostDevice::BeginFrameCommands();
+        if (!commands) return false;
+        commands->setFramebuffer(nullptr);
+        const plume::RenderTextureBarrier before[] = {
+          {source->texture.get(), plume::RenderTextureLayout::COPY_SOURCE},
+          {adopted_destination->texture.get(), plume::RenderTextureLayout::COPY_DEST}};
+        commands->barriers(plume::RenderBarrierStage::COPY, before, 2);
+        static_cast<plume::D3D12CommandList*>(commands)->d3d->CopyResource(dst, src);
+        const plume::RenderTextureBarrier after[] = {
+          {source->texture.get(), plume::RenderTextureLayout::DEPTH_WRITE},
+          {adopted_destination->texture.get(), plume::RenderTextureLayout::SHADER_READ}};
+        commands->barriers(plume::RenderBarrierStage::GRAPHICS, after, 2);
+        adopted_destination->resolved_on_host = true;
+        static std::atomic<u32> depth_copies{0};
+        if (depth_copies.fetch_add(1) < 8)
+          REXLOG_INFO("Native GPU: copied host depth+stencil {}x{} dst=0x{:08X} srv={}",
+                      source->HostWidth(), source->HostHeight(), destination_texture,
+                      adopted_destination->descriptor_index);
+        return true;
+      }
+    }
+  }
+  if (source && adopted_destination && source->surface &&
+      !adopted_destination->surface && destination_level == 0 && destination_slice == 0 &&
+      source->texture && adopted_destination->texture &&
+      source->format == plume::RenderFormat::R16G16B16A16_SNORM &&
+      adopted_destination->format == plume::RenderFormat::R16G16B16A16_UNORM) {
+    // Host fixed16 storage is value/32 in SNORM. Resolve converts values,
+    // including its signed 6-bit exponent, into the destination number format.
+    // Copying SNORM bits into UNORM would also corrupt negative values.
+    const auto& fetch = adopted_destination->guest_fetch;
+    const float scale = 32.0f * Unsigned16ResolveScale(
+        static_cast<i32>(resolve_flags) >> 26, fetch.num_format != 0);
+    if (!HostDevice::ResolveHdrColor(source->texture.get(), source->descriptor_index,
+            adopted_destination->texture.get(), adopted_destination->width,
+            adopted_destination->height, scale, ColorResolveDestination::kUnormRGBA16,
+            true, &*region))
+      return false;
+    adopted_destination->resolved_on_host = true;
+    if (!UpdateResolvedSampling(*adopted_destination)) return false;
+    static std::atomic<u32> logs{0};
+    if (logs.fetch_add(1) < 8)
+      REXLOG_INFO("Native GPU: fixed16 resolve SNORM->UNORM flags={:08X} scale={} dst={:08X}",
+          resolve_flags, scale, destination_texture);
+    return true;
+  }
+  if (source && adopted_destination && source->surface &&
+      !adopted_destination->surface && destination_level == 0 &&
+      destination_slice == 0 && source->texture && adopted_destination->texture &&
+      adopted_destination->d3d_type == static_cast<u32>(D3DResourceType::kTexture) &&
+      ((source->format == plume::RenderFormat::R16G16B16A16_FLOAT &&
+        adopted_destination->format == plume::RenderFormat::R16G16B16A16_UNORM) ||
+       (source->format == plume::RenderFormat::R16G16_FLOAT &&
+        adopted_destination->format == plume::RenderFormat::R16G16_UNORM))) {
+    const auto& fetch = adopted_destination->guest_fetch;
+    // UINT16 resolve writes a clamped integer, not a normalized fraction.
+    // Our raw UNORM16 allocation represents that integer divided by 65535.
+    const float scale = Unsigned16ResolveScale(static_cast<i32>(resolve_flags) >> 26,
+                                              fetch.num_format != 0);
+    if (!HostDevice::ResolveHdrColor(source->texture.get(), source->descriptor_index,
+                                    adopted_destination->texture.get(),
+                                    adopted_destination->width, adopted_destination->height, scale,
+                                    adopted_destination->format == plume::RenderFormat::R16G16_UNORM
+                                        ? ColorResolveDestination::kUnormRG16
+                                        : ColorResolveDestination::kUnormRGBA16,
+                                    true, &*region)) return false;
+    adopted_destination->resolved_on_host = true;
+    if (!UpdateResolvedSampling(*adopted_destination)) return false;
+    static std::atomic<u32> conversions{0};
+    if (conversions.fetch_add(1) < 8)
+      REXLOG_INFO("Native GPU: HDR resolve FLOAT->UNORM {}x{} scale={}",
+                  source->HostWidth(), source->HostHeight(), scale);
+    return true;
+  }
+  const char* failure = nullptr;
+  if (!adopted_destination || !source) failure = "unregistered resource";
+  else if (!adopted_destination->texture || !source->texture)
+    failure = "missing host texture";
+  else if (!source->surface || adopted_destination->surface)
+    failure = "invalid resource kind";
+  else if (destination_level >= adopted_destination->levels)
+    failure = "mip out of range";
+  else if (IsDepthFormat(source->format) || IsDepthFormat(adopted_destination->format))
+    failure = "depth conversion not implemented";
+  else if (adopted_destination->format != source->format)
+    failure = "format conversion not implemented";
+  if (failure) {
+    static std::atomic<u32> failures{0};
+    if (failures.fetch_add(1) < 32) {
+      REXLOG_WARN("Native GPU: resolve rejected ({}) src=0x{:08X} {}x{} fmt={} "
+                  "dst=0x{:08X} {}x{} fmt={} mip={} slice={}", failure,
+                  source_surface, source ? source->HostWidth() : 0,
+                  source ? source->HostHeight() : 0, source ? u32(source->format) : 0,
+                  destination_texture, adopted_destination ? adopted_destination->width : 0,
+                  adopted_destination ? adopted_destination->height : 0,
+                  adopted_destination ? u32(adopted_destination->format) : 0,
+                  destination_level, destination_slice);
+    }
     return false;
   }
-  const bool cube = destination->d3d_type ==
+  const auto& resolved_destination = adopted_destination;
+  const bool cube = resolved_destination->d3d_type ==
                     static_cast<u32>(D3DResourceType::kCubeTexture);
   if ((!cube && destination_slice != 0) || (cube && destination_slice >= 6)) {
     return false;
@@ -467,40 +1504,97 @@ bool ResolveTextureFromSurface(u32 destination_texture, u32 source_surface,
   const plume::RenderTextureBarrier barriers[] = {
       plume::RenderTextureBarrier(source->texture.get(),
                                   plume::RenderTextureLayout::COPY_SOURCE),
-      plume::RenderTextureBarrier(destination->texture.get(),
+      plume::RenderTextureBarrier(resolved_destination->texture.get(),
                                   plume::RenderTextureLayout::COPY_DEST),
   };
   commands->barriers(plume::RenderBarrierStage::COPY, barriers, 2);
+  // EDRAM surfaces are padded (for example 640x368 -> 640x360). A whole
+  // subresource copy exceeds the destination and can remove the D3D12 device.
+  const plume::RenderBox source_box(region->left, region->top,
+      region->left + region->width, region->top + region->height, 0, 1);
   commands->copyTextureRegion(
       plume::RenderTextureCopyLocation::Subresource(
-          destination->texture.get(), destination_level, destination_slice),
+          resolved_destination->texture.get(), destination_level,
+          destination_slice),
       plume::RenderTextureCopyLocation::Subresource(source->texture.get(), 0,
-                                                    0));
+                                                    0), region->x, region->y, 0, &source_box);
   const plume::RenderTextureBarrier final_barriers[] = {
       plume::RenderTextureBarrier(source->texture.get(),
                                   plume::RenderTextureLayout::COLOR_WRITE),
-      plume::RenderTextureBarrier(destination->texture.get(),
+      plume::RenderTextureBarrier(resolved_destination->texture.get(),
                                   plume::RenderTextureLayout::SHADER_READ),
   };
   commands->barriers(plume::RenderBarrierStage::GRAPHICS, final_barriers, 2);
+  resolved_destination->resolved_on_host = true;
+  return UpdateResolvedSampling(*resolved_destination);
+}
+
+bool PromoteTiledSurface(u32 guest_address, u32 width, u32 height) {
+  auto recording = HostDevice::LockRecording();
+  if (!guest_address) return true;
+  const auto resource = FindTexture(guest_address);
+  if (!resource || !resource->surface || resource->width != width ||
+      resource->height > height || !width || !height) return false;
+  if (resource->HostWidth() == width && resource->HostHeight() == height) return true;
+  TextureResource description;
+  description.width = width; description.height = height;
+  description.surface = true; description.format = resource->format;
+  description.d3d_type = resource->d3d_type;
+  auto texture = CreateHostTexture(description, 1, 0);
+  if (!texture) return false;
+  texture->setName("LEGO full logical tiled surface");
+  plume::RenderTextureViewDesc view_desc;
+  view_desc.format = resource->format;
+  view_desc.dimension = plume::RenderTextureViewDimension::TEXTURE_2D;
+  view_desc.mipLevels = 1;
+  auto view = texture->createTextureView(view_desc);
+  const u32 descriptor = HostDevice::RegisterTexture(texture.get(), view.get());
+  if (descriptor == ~u32{0}) return false;
+  // Cached framebuffers contain physical texture pointers, not guest addresses.
+  InvalidateFramebufferReferences(guest_address);
+  auto retired = std::make_shared<SurfaceStorage>();
+  {
+    std::lock_guard lock(resource->mutex);
+    retired->texture = std::move(resource->texture);
+    retired->view = std::move(resource->view);
+    resource->previous_surface_descriptors.push_back(resource->descriptor_index);
+    resource->texture = std::move(texture);
+    resource->view = std::move(view);
+    resource->descriptor_index = descriptor;
+    resource->host_width = width; resource->host_height = height;
+  }
+  HostDevice::RetireResource(std::move(retired));
+  REXLOG_INFO("Native GPU: promoted tiled surface {:08X} guest={}x{} host={}x{} depth={}",
+      guest_address, resource->width, resource->height, width, height, IsDepthFormat(resource->format));
   return true;
 }
 
-plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
-                                             u32 depth_stencil) {
-  const auto color = FindTexture(render_target);
-  const auto depth = FindTexture(depth_stencil);
-  if ((!color || !color->texture) && (!depth || !depth->texture)) {
-    return nullptr;
+plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target, u32 depth_stencil) {
+  return ResolveFramebuffer(std::array<u32, 4>{render_target, 0, 0, 0}, depth_stencil);
+}
+
+plume::RenderFramebuffer* ResolveFramebuffer(
+    const std::array<u32, 4>& render_targets, u32 depth_stencil) {
+  std::array<std::shared_ptr<TextureResource>, 4> colors;
+  u32 color_count = 0;
+  for (u32 i = 0; i < colors.size(); ++i) {
+    if (!render_targets[i]) continue;
+    colors[i] = FindTexture(render_targets[i]);
+    if (!colors[i] || !colors[i]->texture || IsDepthFormat(colors[i]->format)) return nullptr;
+    color_count = i + 1;
   }
-  if (color && IsDepthFormat(color->format)) {
+  // Plume requires contiguous color attachments. Reject a gap rather than
+  // renumbering shader outputs or dereferencing a null texture in Plume.
+  for (u32 i = 0; i < color_count; ++i) if (!colors[i]) return nullptr;
+  const auto depth = FindTexture(depth_stencil);
+  if (!color_count && (!depth || !depth->texture)) {
     return nullptr;
   }
   if (depth && !IsDepthFormat(depth->format)) {
     return nullptr;
   }
-  const auto owner = depth ? depth : color;
-  const u32 key = depth ? render_target : depth_stencil;
+  const auto owner = depth ? depth : colors[0];
+  const FramebufferKey key{render_targets, depth_stencil};
   std::lock_guard lock(owner->mutex);
   const auto existing = owner->framebuffers.find(key);
   if (existing != owner->framebuffers.end()) {
@@ -508,11 +1602,11 @@ plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
   }
 
   plume::RenderFramebufferDesc desc;
-  const plume::RenderTexture* color_attachments[1];
-  if (color && color->texture) {
-    color_attachments[0] = color->texture.get();
-    desc.colorAttachments = color_attachments;
-    desc.colorAttachmentsCount = 1;
+  std::array<const plume::RenderTexture*, 4> color_attachments{};
+  if (color_count) {
+    for (u32 i = 0; i < color_count; ++i) color_attachments[i] = colors[i]->texture.get();
+    desc.colorAttachments = color_attachments.data();
+    desc.colorAttachmentsCount = color_count;
   }
   if (depth && depth->texture) {
     desc.depthAttachment = depth->texture.get();
@@ -524,7 +1618,7 @@ plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
   auto framebuffer = device->createFramebuffer(desc);
   if (!framebuffer) {
     REXLOG_ERROR("Native GPU: failed to create framebuffer (rt=0x{:08X}, "
-                 "ds=0x{:08X})", render_target, depth_stencil);
+                 "ds=0x{:08X})", render_targets[0], depth_stencil);
     return nullptr;
   }
   auto* result = framebuffer.get();
@@ -537,9 +1631,39 @@ bool IsNativeTexture(u32 guest_address) {
   return g_textures.contains(guest_address);
 }
 
+void LogUnknownTexture(u32 guest_address, const char* operation) {
+  if (!guest_address) {
+    REXLOG_WARN("Native GPU: {} received a null texture", operation);
+    return;
+  }
+  const auto* words = GuestAt<const be_u32>(guest_address);
+  REXLOG_WARN(
+      "Native GPU: {} unknown texture 0x{:08X}, header "
+      "{:08X} {:08X} {:08X} {:08X} {:08X} {:08X} {:08X} {:08X} "
+      "{:08X} {:08X} {:08X} {:08X} {:08X}",
+      operation, guest_address, static_cast<u32>(words[0]),
+      static_cast<u32>(words[1]), static_cast<u32>(words[2]),
+      static_cast<u32>(words[3]), static_cast<u32>(words[4]),
+      static_cast<u32>(words[5]), static_cast<u32>(words[6]),
+      static_cast<u32>(words[7]), static_cast<u32>(words[8]),
+      static_cast<u32>(words[9]), static_cast<u32>(words[10]),
+      static_cast<u32>(words[11]), static_cast<u32>(words[12]));
+}
+
 u32 NativeTextureType(u32 guest_address) {
   const auto resource = FindTexture(guest_address);
-  return resource ? resource->d3d_type : 0;
+  // Borrowed headers must use the original GetResourceType, including its
+  // dimension/stacked distinctions. Registry metadata is a host view.
+  return resource && resource->owns_guest_memory ? resource->d3d_type : 0;
+}
+
+bool IsCpuUploadedTexture(u32 guest_address) {
+  auto recording = HostDevice::LockRecording();
+  const auto resource = FindTexture(guest_address);
+  if (!resource) return false;
+  std::lock_guard lock(resource->mutex);
+  return !resource->surface && !resource->resolved_on_host &&
+      !resource->depth_alias_generation && !resource->owns_guest_memory;
 }
 
 u32 AddRefNativeTexture(u32 guest_address) {
@@ -554,6 +1678,7 @@ u32 AddRefNativeTexture(u32 guest_address) {
 }
 
 u32 ReleaseNativeTexture(u32 guest_address) {
+  auto recording = HostDevice::LockRecording();
   std::shared_ptr<TextureResource> released;
   {
     std::lock_guard lock(g_textures_mutex);
@@ -571,24 +1696,41 @@ u32 ReleaseNativeTexture(u32 guest_address) {
     released = std::move(it->second);
     g_textures.erase(it);
   }
+  if (g_texture_lifecycle_logs.fetch_add(1, std::memory_order_relaxed) < 64) {
+    REXLOG_INFO("Native GPU: released {} 0x{:08X}",
+                released->surface ? "surface" : "texture", guest_address);
+  }
   auto* memory = REX_KERNEL_MEMORY();
   InvalidateFramebufferReferences(guest_address);
   HostDevice::UnregisterTexture(released->descriptor_index);
+  HostDevice::UnregisterTexture(released->resolved_descriptor_index);
+  HostDevice::UnregisterTexture(released->sampled_descriptor_index);
+  for (u32 descriptor : released->previous_surface_descriptors)
+    HostDevice::UnregisterTexture(descriptor);
   HostDevice::RetireResource(released);
   if (released->mirror_address) {
     memory->SystemHeapFree(released->mirror_address);
   }
-  memory->SystemHeapFree(guest_address);
+  if (released->owns_guest_memory) {
+    memory->SystemHeapFree(guest_address);
+  }
   return 0;
 }
 
 void ResetTextureResources() {
+  g_depth_resolves.clear();
+  g_edram_colors.clear();
+  g_depth_alias_generation = 0;
   std::vector<std::pair<u32, u32>> allocations;
   {
     std::lock_guard lock(g_textures_mutex);
     allocations.reserve(g_textures.size());
     for (const auto& [guest_address, resource] : g_textures) {
-      allocations.emplace_back(guest_address, resource->mirror_address);
+      if (resource->owns_guest_memory) {
+        allocations.emplace_back(guest_address, resource->mirror_address);
+      } else if (resource->mirror_address) {
+        allocations.emplace_back(0, resource->mirror_address);
+      }
     }
     g_textures.clear();
   }
@@ -597,8 +1739,79 @@ void ResetTextureResources() {
     if (mirror_address) {
       memory->SystemHeapFree(mirror_address);
     }
-    memory->SystemHeapFree(guest_address);
+    if (guest_address) {
+      memory->SystemHeapFree(guest_address);
+    }
   }
 }
 
+namespace {
+bool TextureDataOverlapsPoolCopy(const TextureResource& resource, PhysicalCopyRange range) {
+  if (resource.surface) return false;  // EDRAM has no physical-memory backing.
+  if (resource.owns_guest_memory)
+    return GuestRangeOverlapsPoolCopy(resource.mirror_address, resource.mirror_size, range);
+  namespace tu = rex::graphics::texture_util;
+  const auto& fetch = resource.guest_fetch;
+  if (!fetch.base_address || !resource.levels) return false;
+  const auto layout = tu::GetGuestTextureLayout(fetch.dimension, fetch.pitch,
+      resource.width, resource.height,
+      fetch.dimension == rex::graphics::xenos::DataDimension::k3D ? resource.depth : 1,
+      fetch.tiled, fetch.format, fetch.packed_mips, true, resource.levels - 1);
+  return GuestRangeOverlapsPoolCopy(fetch.base_address << 12, layout.base.level_data_extent_bytes, range) ||
+      (resource.levels > 1 && fetch.mip_address &&
+       GuestRangeOverlapsPoolCopy(fetch.mip_address << 12, layout.mips_total_extent_bytes, range));
+}
+}  // namespace
+
+bool PoolCopyHasHostTextureSource(PhysicalCopyRange source) {
+  auto recording = HostDevice::LockRecording();
+  std::lock_guard lock(g_textures_mutex);
+  for (const auto& [address, resource] : g_textures) {
+    std::lock_guard texture_lock(resource->mutex);
+    if ((resource->resolved_on_host || resource->depth_alias_generation) &&
+        (TextureDataOverlapsPoolCopy(*resource, source) ||
+         GuestRangeOverlapsPoolCopy(address, sizeof(D3DTexture), source)))
+      return true;
+  }
+  return false;
+}
+
+u32 InvalidatePoolCopyTextures(PhysicalCopyRange destination) {
+  auto recording = HostDevice::LockRecording();
+  std::vector<std::shared_ptr<TextureResource>> retired;
+  u32 count = 0;
+  {
+    std::lock_guard lock(g_textures_mutex);
+    for (auto it = g_textures.begin(); it != g_textures.end();) {
+      const auto& resource = it->second;
+      std::lock_guard texture_lock(resource->mutex);
+      if (!resource->owns_guest_memory &&
+          GuestRangeOverlapsPoolCopy(resource->guest_address, sizeof(D3DTexture), destination)) {
+        retired.push_back(std::move(it->second));
+        it = g_textures.erase(it);
+        ++count;
+      } else {
+        if (TextureDataOverlapsPoolCopy(*resource, destination)) {
+          resource->guest_uploaded = false;
+          resource->upload_source_key_valid = false;
+          resource->depth_alias_generation = 0;
+          ++count;
+        }
+        ++it;
+      }
+    }
+  }
+  for (auto& resource : retired) {
+    InvalidateFramebufferReferences(resource->guest_address);
+    HostDevice::UnregisterTexture(resource->descriptor_index);
+    HostDevice::UnregisterTexture(resource->resolved_descriptor_index);
+    HostDevice::UnregisterTexture(resource->sampled_descriptor_index);
+    for (u32 descriptor : resource->previous_surface_descriptors)
+      HostDevice::UnregisterTexture(descriptor);
+    if (resource->mirror_address) REX_KERNEL_MEMORY()->SystemHeapFree(resource->mirror_address);
+    HostDevice::RetireResource(std::move(resource));
+  }
+  return count;
+}
+
 }  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/textures.h b/rexlego/src/gpu_native/textures.h
index 66b4086..4ef055b 100644
--- a/rexlego/src/gpu_native/textures.h
+++ b/rexlego/src/gpu_native/textures.h
@@ -1,8 +1,12 @@
 // Guest-visible texture/surface resources and their CPU mirrors.
 #pragma once
 
+#include <array>
+
 #include <plume_render_interface_types.h>
 #include <rex/types.h>
+#include "gpu_native/resolve_region.h"
+#include "gpu_native/pool_copy.h"
 
 namespace plume {
 struct RenderFramebuffer;
@@ -36,20 +40,52 @@ struct TextureResourceView {
 };
 
 TextureResourceView ResolveTextureResource(u32 guest_address);
+// Call at SetTexture and Resolve, where the header supplies the new binding.
+// A draw already in flight must retain its old texture and immutable SRV.
+void RefreshTextureHeader(u32 guest_address);
 // Copies the complete CPU mirror into the host texture and leaves it in
 // SHADER_READ. The CPU mirror remains authoritative for readback/correctness.
 bool UploadTextureResource(u32 guest_address,
-                           plume::RenderCommandList* commands);
+                           plume::RenderCommandList* commands,
+                           bool require_content_hash = false);
+struct TextureUploadTiming {
+  u64 calls = 0;
+  u64 source_hits = 0;
+  u64 converted_bytes = 0;
+  u64 hashed_bytes = 0;
+  double source_ms = 0;
+  double hash_ms = 0;
+  double cpu_ms = 0;
+};
+bool NativeTextureTimingEnabled();
+TextureUploadTiming ConsumeTextureUploadTiming();
 bool ResolveTextureFromSurface(u32 destination_texture, u32 source_surface,
                                u32 destination_level,
-                               u32 destination_slice);
+                               u32 destination_slice, u32 resolve_flags = 0,
+                               const ResolveRect* rectangle = nullptr,
+                               const ResolvePoint* point = nullptr);
 plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target,
                                              u32 depth_stencil);
+plume::RenderFramebuffer* ResolveFramebuffer(
+    const std::array<u32, 4>& render_targets, u32 depth_stencil);
+// Synchronous BeginTiling allocation, before the pass's clear and geometry.
+// Guest surface dimensions and Xbox tiling/command-queue fields are unchanged.
+bool PromoteTiledSurface(u32 guest_address, u32 width, u32 height);
+float SurfaceColorOutputScale(u32 guest_address);
+bool PrepareSurfaceDepthAlias(u32 guest_address);
+void MarkSurfaceWritten(u32 guest_address);
 
 bool IsNativeTexture(u32 guest_address);
+// CPU texels are copied into retained UPLOAD storage; the GPU never reads
+// their guest allocation. Render targets and resolved destinations excluded.
+bool IsCpuUploadedTexture(u32 guest_address);
+void LogUnknownTexture(u32 guest_address, const char* operation);
 u32 NativeTextureType(u32 guest_address);
 u32 AddRefNativeTexture(u32 guest_address);
 u32 ReleaseNativeTexture(u32 guest_address);
 void ResetTextureResources();
+// A CPU copy must not read stale guest bytes for a GPU-resolved source.
+bool PoolCopyHasHostTextureSource(PhysicalCopyRange source);
+u32 InvalidatePoolCopyTextures(PhysicalCopyRange destination);
 
 }  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/tile_extent.h b/rexlego/src/gpu_native/tile_extent.h
new file mode 100644
index 0000000..7c82ad1
--- /dev/null
+++ b/rexlego/src/gpu_native/tile_extent.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <span>
+
+namespace legodimensions::gpu_native {
+
+struct NativeTileRect { int32_t left, top, right, bottom; };
+struct NativeTileExtent { uint32_t width = 0, height = 0; };
+
+// Support a complete, non-overlapping rectangle partition. Reject holes and
+// overlaps rather than inventing pixels outside the guest's tiling contract.
+inline NativeTileExtent LogicalTileExtent(std::span<const NativeTileRect> tiles) {
+  if (tiles.empty() || tiles.size() > 16) return {};
+  NativeTileExtent result;
+  uint64_t area = 0;
+  for (size_t i = 0; i < tiles.size(); ++i) {
+    const auto& r = tiles[i];
+    if (r.left < 0 || r.top < 0 || r.right <= r.left || r.bottom <= r.top ||
+        r.right > 16384 || r.bottom > 16384) return {};
+    result.width = std::max(result.width, uint32_t(r.right));
+    result.height = std::max(result.height, uint32_t(r.bottom));
+    area += uint64_t(r.right - r.left) * (r.bottom - r.top);
+    for (size_t j = 0; j < i; ++j) {
+      const auto& q = tiles[j];
+      if (r.left < q.right && q.left < r.right &&
+          r.top < q.bottom && q.top < r.bottom) return {};
+    }
+  }
+  return area == uint64_t(result.width) * result.height ? result : NativeTileExtent{};
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/tonemap_dof.h b/rexlego/src/gpu_native/tonemap_dof.h
new file mode 100644
index 0000000..fa0cc7a
--- /dev/null
+++ b/rexlego/src/gpu_native/tonemap_dof.h
@@ -0,0 +1,21 @@
+// The game's disabled DoF setting must also disable the final sharp/mip mix.
+#pragma once
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+namespace legodimensions::gpu_native {
+inline bool ApplyDisabledTonemapDof(std::uint64_t pixel_shader_hash, bool disabled,
+                                    void* pixel_constants, std::size_t bytes) {
+  constexpr std::size_t kMixOffset = 12 * 16;
+  if (!disabled || pixel_shader_hash != UINT64_C(0x3A47E5DDE66B42C6) ||
+      !pixel_constants || bytes < kMixOffset + 2 * sizeof(float)) return false;
+  // HLSL: w=saturate(2*(mip.a*c12.x+c12.y)-1), col=lerp(full,mip.rgb,w).
+  // Resolved mipColor1 is UNORM (finite alpha); x=y=0 gives w=0.
+  // Preserve c12.z (output alpha threshold), c12.w, exposure, bloom and LUT.
+  const float disabled_mix[2] = {0.0f, 0.0f};
+  std::memcpy(static_cast<std::uint8_t*>(pixel_constants) + kMixOffset,
+              disabled_mix, sizeof(disabled_mix));
+  return true;
+}
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/vertex_byte_order.h b/rexlego/src/gpu_native/vertex_byte_order.h
new file mode 100644
index 0000000..6cb6f8f
--- /dev/null
+++ b/rexlego/src/gpu_native/vertex_byte_order.h
@@ -0,0 +1,45 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <span>
+
+namespace legodimensions::gpu_native {
+
+struct VertexByteOrder {
+  uint32_t stride = 0;
+  uint32_t stream_offset = 0;
+  std::span<const uint32_t> reversed_elements;
+};
+
+// Different stream offsets at the same phase fetch the same converted bytes.
+// Convert all records at that phase once, rather than a fresh suffix per draw.
+constexpr VertexByteOrder CanonicalVertexByteOrder(VertexByteOrder order) {
+  if (order.stride >= 4) order.stream_offset %= order.stride;
+  return order;
+}
+
+// Input bytes have already undergone the common guest DWORD endian swap.
+// The TT UBYTE4[N] declaration swizzle WZYX then reverses the four components.
+// Only the upload copy is changed; floats, other fields and guest RAM stay intact.
+constexpr bool ApplyVertexByteOrder(uint8_t* data, size_t size,
+                                    const VertexByteOrder& order) {
+  if (order.reversed_elements.empty()) return true;
+  if (order.stride < 4 || order.stream_offset > size) return false;
+  for (uint32_t field : order.reversed_elements)
+    if (field > order.stride - 4) return false;
+  for (uint64_t base = order.stream_offset; base < size; base += order.stride) {
+    for (uint32_t field : order.reversed_elements) {
+      const uint64_t offset = base + field;
+      if (offset + 4 > size) continue;
+      const uint8_t a = data[offset], b = data[offset + 1];
+      data[offset] = data[offset + 3];
+      data[offset + 1] = data[offset + 2];
+      data[offset + 2] = b;
+      data[offset + 3] = a;
+    }
+  }
+  return true;
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/vertex_declarations.cpp b/rexlego/src/gpu_native/vertex_declarations.cpp
index 329faa9..002b4a0 100644
--- a/rexlego/src/gpu_native/vertex_declarations.cpp
+++ b/rexlego/src/gpu_native/vertex_declarations.cpp
@@ -5,12 +5,14 @@
 #include <memory>
 #include <mutex>
 #include <unordered_map>
+#include <utility>
 #include <vector>
 
 #include <plume_render_interface.h>
 #include <rex/logging.h>
 #include <rex/system/kernel_state.h>
 #include <rex/system/xmemory.h>
+#include <xxhash.h>
 
 #include "gpu_native/d3d.h"
 #include "gpu_native/device.h"
@@ -51,6 +53,8 @@ static_assert(sizeof(GuestVertexElement) == 12);
 
 struct VertexDeclarationResource {
   u32 guest_address = 0;
+  u64 content_hash = 0;
+  bool owns_guest_memory = true;
   std::vector<plume::RenderInputElement> inputs;
   bool supported = true;
   u32 swapped_texcoords = 0;
@@ -61,6 +65,7 @@ struct VertexDeclarationResource {
   u32 swapped_positions = 0;
   u32 sint_texcoords = 0;
   bool has_r11g11b10_normal = false;
+  u64 reversed_byte_elements = 0;
 };
 
 std::mutex g_declarations_mutex;
@@ -108,8 +113,15 @@ plume::RenderFormat ConvertDeclFormat(u32 type) {
     case 0x001A2386: return RF::R8G8B8A8_UINT;
     case 0x002C2359: return RF::R16G16_SINT;
     case 0x001A235A: return RF::R16G16B16A16_SNORM;
+    case 0x001A225A: return RF::R16G16B16A16_SINT;
     case 0x001A2086:
     case 0x001A2186: return RF::R8G8B8A8_UNORM;
+    // TT UBYTE4[N] with WZYX component swizzle (bits 10..21 = 0x053).
+    // Sign bit 8 is CLEAR; bit 9 selects integer vs normalized conversion.
+    // Apply the component reversal to the host vertex upload, separately
+    // from the DWORD endian swap used for guest floating-point attributes.
+    case 0x00014C86: return RF::R8G8B8A8_UNORM;
+    case 0x00014E86: return RF::R8G8B8A8_UINT;
     case 0x002C2159: return RF::R16G16_SNORM;
     case 0x001A215A: return RF::R16G16B16A16_SNORM;
     case 0x002C2059: return RF::R16G16_UNORM;
@@ -146,29 +158,32 @@ std::shared_ptr<VertexDeclarationResource> FindDeclaration(u32 address) {
 }
 
 D3DResource* GuestHeader(u32 address) {
-  return reinterpret_cast<D3DResource*>(REX_KERNEL_MEMORY()->virtual_membase() +
-                                        address);
+  return REX_KERNEL_MEMORY()->TranslateVirtual<D3DResource*>(address);
 }
 
 }  // namespace
 
-u32 CreateVertexDeclarationResource(u32 elements_address) {
+static u32 BuildVertexDeclaration(u32 elements_address, u32 existing_address = 0,
+                                  u32 existing_count = 0) {
   if (!elements_address) {
     return 0;
   }
-  const auto* elements = reinterpret_cast<const GuestVertexElement*>(
-      REX_KERNEL_MEMORY()->virtual_membase() + elements_address);
-  u32 count = 0;
-  while (count < kMaxElements && static_cast<u16>(elements[count].stream) != 0xFF &&
+  const auto* elements =
+      REX_KERNEL_MEMORY()->TranslateVirtual<const GuestVertexElement*>(
+          elements_address);
+  u32 count = existing_count;
+  while (!existing_address && count < kMaxElements && static_cast<u16>(elements[count].stream) != 0xFF &&
          static_cast<u32>(elements[count].type) != 0xFFFFFFFFu) {
     ++count;
   }
-  if (count == kMaxElements) {
+  if (count >= kMaxElements) {
     REXLOG_ERROR("Native GPU: unterminated vertex declaration");
     return 0;
   }
 
   auto resource = std::make_shared<VertexDeclarationResource>();
+  resource->content_hash = XXH3_64bits(elements, count * sizeof(GuestVertexElement));
+  resource->owns_guest_memory = existing_address == 0;
   resource->inputs.reserve(count);
   for (u32 i = 0; i < count; ++i) {
     const auto& element = elements[i];
@@ -180,6 +195,8 @@ u32 CreateVertexDeclarationResource(u32 elements_address) {
     input.format = ConvertDeclFormat(element.type);
     input.slotIndex = element.stream;
     input.alignedByteOffset = element.offset;
+    if (u32(element.type) == 0x00014C86 || u32(element.type) == 0x00014E86)
+      resource->reversed_byte_elements |= u64{1} << i;
     if (input.format == plume::RenderFormat::UNKNOWN) {
       resource->supported = false;
       REXLOG_WARN("Native GPU: unsupported vertex element usage={} index={} "
@@ -256,30 +273,56 @@ u32 CreateVertexDeclarationResource(u32 elements_address) {
 
   const u32 allocation_size = 52 + (count + 1) * sizeof(GuestVertexElement);
   auto* memory = REX_KERNEL_MEMORY();
-  const u32 guest_address = memory->SystemHeapAlloc(allocation_size, 0x10);
+  const u32 guest_address = existing_address ? existing_address :
+      memory->SystemHeapAlloc(allocation_size, 0x10);
   if (!guest_address) {
     return 0;
   }
-  memory->Zero(guest_address, allocation_size);
-  auto* header = GuestHeader(guest_address);
-  header->common = kDeclarationFlag;
-  header->reference_count = 1;
-  header->base_flush = 0xFFFF0000u;
-  auto* bytes = memory->virtual_membase() + guest_address;
-  *reinterpret_cast<be_u32*>(bytes + 24) = count;
-  std::memcpy(bytes + 52, elements,
-              (count + 1) * sizeof(GuestVertexElement));
+  if (!existing_address) {
+    memory->Zero(guest_address, allocation_size);
+    auto* header = GuestHeader(guest_address);
+    header->common = kDeclarationFlag;
+    header->reference_count = 1;
+    header->base_flush = 0xFFFF0000u;
+    auto* bytes = memory->TranslateVirtual<uint8_t*>(guest_address);
+    *reinterpret_cast<be_u32*>(bytes + 24) = count;
+    std::memcpy(bytes + 52, elements,
+                (count + 1) * sizeof(GuestVertexElement));
+  }
 
   resource->guest_address = guest_address;
+  std::shared_ptr<VertexDeclarationResource> previous;
   {
     std::lock_guard lock(g_declarations_mutex);
-    g_declarations.emplace(guest_address, resource);
+    auto& entry = g_declarations[guest_address];
+    previous = std::exchange(entry, resource);
   }
+  if (previous) HostDevice::RetireResource(std::move(previous));
   return guest_address;
 }
 
+u32 CreateVertexDeclarationResource(u32 elements_address) {
+  return BuildVertexDeclaration(elements_address);
+}
+
 VertexDeclarationView ResolveVertexDeclaration(u32 guest_address) {
-  const auto resource = FindDeclaration(guest_address);
+  auto resource = FindDeclaration(guest_address);
+  if (guest_address && (!resource || !resource->owns_guest_memory)) {
+    const auto* words = REX_KERNEL_MEMORY()->TranslateVirtual<const be_u32*>(guest_address);
+    if ((static_cast<u32>(words[0]) & 0xFu) == 5 &&
+        static_cast<u32>(words[6]) < kMaxElements) {
+      const auto* elements = REX_KERNEL_MEMORY()->TranslateVirtual<const u8*>(guest_address + 52);
+      const u64 hash = XXH3_64bits(elements, u32(words[6]) * sizeof(GuestVertexElement));
+      if (!resource || resource->content_hash != hash) {
+        if (resource)
+          REXLOG_INFO("Native GPU: placement declaration contents changed at {:08X}", guest_address);
+        BuildVertexDeclaration(guest_address + 52, guest_address, words[6]);
+        resource = FindDeclaration(guest_address);
+      }
+    } else {
+      return {};
+    }
+  }
   if (!resource) {
     return {};
   }
@@ -293,7 +336,9 @@ VertexDeclarationView ResolveVertexDeclaration(u32 guest_address) {
           resource->swapped_blend_weights,
           resource->swapped_positions,
           resource->sint_texcoords,
-          resource->has_r11g11b10_normal};
+          resource->has_r11g11b10_normal,
+          resource->reversed_byte_elements,
+          resource->content_hash};
 }
 
 bool IsNativeVertexDeclaration(u32 guest_address) {
@@ -325,7 +370,7 @@ u32 ReleaseNativeVertexDeclaration(u32 guest_address) {
     g_declarations.erase(it);
   }
   HostDevice::RetireResource(released);
-  REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
+  if (released->owns_guest_memory) REX_KERNEL_MEMORY()->SystemHeapFree(guest_address);
   return 0;
 }
 
@@ -335,7 +380,7 @@ void ResetVertexDeclarations() {
     std::lock_guard lock(g_declarations_mutex);
     addresses.reserve(g_declarations.size());
     for (const auto& [address, resource] : g_declarations) {
-      addresses.push_back(address);
+      if (resource->owns_guest_memory) addresses.push_back(address);
     }
     g_declarations.clear();
   }
diff --git a/rexlego/src/gpu_native/vertex_declarations.h b/rexlego/src/gpu_native/vertex_declarations.h
index 07b6479..f62e31b 100644
--- a/rexlego/src/gpu_native/vertex_declarations.h
+++ b/rexlego/src/gpu_native/vertex_declarations.h
@@ -21,6 +21,8 @@ struct VertexDeclarationView {
   u32 swapped_positions = 0;
   u32 sint_texcoords = 0;
   bool has_r11g11b10_normal = false;
+  u64 reversed_byte_elements = 0;
+  u64 content_hash = 0;
 };
 
 u32 CreateVertexDeclarationResource(u32 elements_address);
diff --git a/rexlego/src/legodimensions_app.h b/rexlego/src/legodimensions_app.h
index e486533..41f3d1a 100644
--- a/rexlego/src/legodimensions_app.h
+++ b/rexlego/src/legodimensions_app.h
@@ -33,6 +33,7 @@
 #include "updates.h"
 #ifdef LEGODIMENSIONS_NATIVE_GPU
 #include "gpu_native/device.h"
+#include "gpu_native/renderer_mode.h"
 #endif
 
 class LegodimensionsApp : public rex::ReXApp {
@@ -70,11 +71,16 @@ class LegodimensionsApp : public rex::ReXApp {
   // Detached renderer mode: the SDK still owns the window/input/audio, while
   // the title's D3D calls are handled by gpu_native instead of rexgpu-xenos.
   void OnPreSetup(rex::RuntimeConfig& config) override {
+    if (legodimensions::gpu_native::UsePm4Reference()) {
+      if (config.gpu_plugin.empty()) config.gpu_plugin = "xenos";
+      return;
+    }
     config.graphics.reset();
     config.gpu_plugin.clear();
   }
 
   void OnPreLaunchModule() override {
+    if (legodimensions::gpu_native::UsePm4Reference()) return;
     if (!legodimensions::gpu_native::HostDevice::Create(window())) {
       REXLOG_ERROR("Native GPU initialization failed; stopping before guest launch");
       app_context().QuitFromUIThread();
@@ -99,7 +105,8 @@ class LegodimensionsApp : public rex::ReXApp {
   // Dropping the pipe is what clears the presence; Discord does the rest.
   void OnShutdown() override {
 #ifdef LEGODIMENSIONS_NATIVE_GPU
-    legodimensions::gpu_native::HostDevice::Shutdown();
+    if (!legodimensions::gpu_native::UsePm4Reference())
+      legodimensions::gpu_native::HostDevice::Shutdown();
 #endif
     legodimensions::discord::Stop();
     legodimensions::toypad_app::StopIfStarted();
diff --git a/thirdparty/plume b/thirdparty/plume
index de0f70f..ceaeeee 160000
--- a/thirdparty/plume
+++ b/thirdparty/plume
@@ -1 +1 @@
-Subproject commit de0f70f75087242ce92516dd587ecadff25385db
+Subproject commit ceaeeee0f7e914886542f041a52ecb40b76c8aaf

From c376d4061ebbd8e16fa722cfe747fc42ff5a2813 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 20:40:57 +0300
Subject: [PATCH 18/44] Update resource regression harnesses for depth aliases
 and portable Python

---
 tools/gpu-aot/test_alpha_test.py             | 3 +++
 tools/gpu-aot/test_cpu_resource_wait.py      | 6 +++---
 tools/gpu-aot/test_texture_header_refresh.py | 4 ++++
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tools/gpu-aot/test_alpha_test.py b/tools/gpu-aot/test_alpha_test.py
index 5a2d39a..e3732e2 100644
--- a/tools/gpu-aot/test_alpha_test.py
+++ b/tools/gpu-aot/test_alpha_test.py
@@ -9,6 +9,9 @@
 import re
 import struct
 import subprocess
+import sys
+# The portable embedded Python does not add the script directory to sys.path.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
 from test_depth_export import fixture
 
 p = argparse.ArgumentParser(description=__doc__)
diff --git a/tools/gpu-aot/test_cpu_resource_wait.py b/tools/gpu-aot/test_cpu_resource_wait.py
index 665c223..5c18f23 100644
--- a/tools/gpu-aot/test_cpu_resource_wait.py
+++ b/tools/gpu-aot/test_cpu_resource_wait.py
@@ -39,7 +39,7 @@ def body(path,sig):
  static bool EnqueueCompletionCallback(std::function<void()> fn){if(!ready)return false;pending.push_back(fn);return true;}
  static void PollCompletionCallbacks(){if(!completed)return;auto tasks=std::move(pending);pending.clear();for(auto& fn:tasks)fn();}
 };
-struct Texture {std::mutex mutex;bool surface=false,resolved_on_host=false,owns_guest_memory=false;};
+struct Texture {std::mutex mutex;bool surface=false,resolved_on_host=false,owns_guest_memory=false;unsigned depth_alias_generation=0;};
 std::shared_ptr<Texture> texture=std::make_shared<Texture>();
 auto FindTexture(u32 address){return address==3?texture:nullptr;}
 bool IsNativeBuffer(u32 address){return address==1;}
@@ -56,7 +56,7 @@ def body(path,sig):
   BlockUntilNotBusyHook(resource);assert(HostDevice::skips==u32(enabled&&safe));
   assert(HostDevice::syncs==u32(!enabled||!safe));};
  check(0,false);check(100,false);check(1,true);check(2,true);
- for(u32 bits=0;bits<8;++bits){texture->surface=bits&1;texture->resolved_on_host=bits&2;texture->owns_guest_memory=bits&4;
+ for(u32 bits=0;bits<16;++bits){texture->surface=bits&1;texture->resolved_on_host=bits&2;texture->owns_guest_memory=bits&4;texture->depth_alias_generation=bits&8;
   check(3,bits==0);}
  HostDevice::syncs=0;BlockUntilIdleHook(nullptr);assert(HostDevice::syncs==1);
  HostDevice::syncs=0;BlockOnFenceHook(1);assert(HostDevice::syncs==1);
@@ -74,7 +74,7 @@ def body(path,sig):
  subprocess.run([str(exe.resolve())],check=True,env=env)
 (a.output/'verification.json').write_text(json.dumps({'actual_native_hook_bodies':True,
  'opt_in_and_default_tested':True,'checks':['known CPU buffers/shaders','borrowed CPU textures',
- 'all surface/resolved/owned texture exclusions','unknown resource fallback','explicit idle and fence waits',
+ 'all surface/resolved/owned/depth-alias texture exclusions','unknown resource fallback','explicit idle and fence waits',
  'GPU completion before guest callback','unavailable device does not call guest'],'passed':True,
  'real_GPU_gameplay_not_proven':True},indent=2))
 print('Passed actual wait hooks: conservative default, CPU-only opt-in, exclusions and deferred callback ordering.')
diff --git a/tools/gpu-aot/test_texture_header_refresh.py b/tools/gpu-aot/test_texture_header_refresh.py
index 91de100..3da8373 100644
--- a/tools/gpu-aot/test_texture_header_refresh.py
+++ b/tools/gpu-aot/test_texture_header_refresh.py
@@ -45,6 +45,7 @@
   u32 adopted_header_type=3,descriptor_index=4,resolved_descriptor_index=5,sampled_descriptor_index=6,mirror_address=0;
   std::vector<u32> previous_surface_descriptors{7};
   std::mutex mutex; std::array<u32,6> guest_fetch;
+  u32 cpu_stamp=0,depth_alias_generation=0;
 };
 std::mutex g_textures_mutex;
 std::unordered_map<u32,std::shared_ptr<TextureResource>> g_textures;
@@ -67,11 +68,13 @@
 void Check(std::array<u32,6> before,std::array<u32,6> after,bool replace,bool host=false,bool owned=false,u32 type=3) {
   g_textures.clear(); retired.clear(); descriptors.clear(); framebuffer_invalidations=0;
   auto old=std::make_shared<TextureResource>(); old->guest_fetch=before;
+  old->cpu_stamp=11; old->depth_alias_generation=12;
   old->resolved_on_host=host; old->owns_guest_memory=owned;
   header.format.dword=after; header.resource.common=type; g_textures.emplace(1,old);
   RefreshTextureHeader(1);
   if(owned || (before==after && type==3)) {
     assert(FindTexture(1)==old && old->guest_fetch==before && old->upload_source_key_valid);
+    assert(old->cpu_stamp==11 && old->depth_alias_generation==12);
     assert(retired.empty() && descriptors.empty()); return;
   }
   if(replace) {
@@ -80,6 +83,7 @@
   } else {
     assert(FindTexture(1)==old && old->guest_fetch==after);
     assert(!old->upload_source_key_valid && old->guest_uploaded==host);
+    assert(old->cpu_stamp==0 && old->depth_alias_generation==0);
     assert(retired.empty() && descriptors.empty());
   }
 }

From 719f83dcef89df8f074c9d399e65338fe3176942 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 20:46:25 +0300
Subject: [PATCH 19/44] Associate failed and captured native draws with guest
 shader objects

---
 rexlego/src/gpu_native/draw.cpp | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
index cca92d2..50d4b5a 100644
--- a/rexlego/src/gpu_native/draw.cpp
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -722,7 +722,12 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
   auto* pipeline = GetPipeline(key, declaration, topology, depth.format);
   if (!pipeline) {
     LongProbeEvent("draw_pipeline_unavailable", true, "VS=", key.vertex_shader_hash,
-        "PS=", key.pixel_shader_hash, "decl_hash=", key.declaration_hash);
+        "PS=", key.pixel_shader_hash, "decl_hash=", key.declaration_hash,
+        "guest_vs=", key.vertex_shader, "guest_ps=", key.pixel_shader,
+        "guest_decl=", bindings.vertex_declaration, "primitive=", primitive_type,
+        "indexed=", indexed, "start=", start, "count=", count,
+        "base_vertex=", base_vertex, "rt=", bindings.render_targets[0],
+        "depth=", bindings.depth_stencil);
     return false;
   }
   timing.Next();
@@ -1024,6 +1029,9 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
         LongProbeOnce(key.vertex_shader_hash ^ key.pixel_shader_hash ^ declaration.content_hash)))
       LongProbeEvent("draw_stage", false, "VS=", key.vertex_shader_hash,
           "PS=", key.pixel_shader_hash, "count=", count, "indexed=", indexed,
+          "guest_vs=", key.vertex_shader, "guest_ps=", key.pixel_shader,
+          "guest_decl=", bindings.vertex_declaration, "primitive=", primitive_type,
+          "start=", start, "base_vertex=", base_vertex,
           "decl_hash=", declaration.content_hash, "rt=", bindings.render_targets[0],
           "width=", color.width, "height=", color.height, "format=", u32(color.format),
           "depth=", bindings.depth_stencil, "write_mask=", key.color_write_mask,

From 08852374ad48148bca8f48cd3c2a60d01c7c03b2 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 21:02:55 +0300
Subject: [PATCH 20/44] Verify captured placement shaders against production
 AOT lookup

---
 tools/gpu-aot/README.md                       | 24 +++++
 .../gpu-aot/test_placement_capture_lookup.py  | 96 +++++++++++++++++++
 2 files changed, 120 insertions(+)
 create mode 100644 tools/gpu-aot/test_placement_capture_lookup.py

diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index 7a3199d..5821ca6 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -297,6 +297,30 @@ separate limitations.
 
 ## Bank118 and menu idle reproduction, September 30
 
+### October 3 hub continuation
+
+`oct03-hub-diagnostics` captured 57 additional validated placement containers
+in Vorton. The local candidate combines these with all 118 previous runtime
+containers: 175/175 compile with the October 1 compiler and its existing
+624-byte constants ABI. The merged index contains 18836 signatures and preserves
+all 18722 earlier records; runtime prelink produces 304 variants. Inputs,
+rollback archives and build proof are in
+`rexlego/out/native-gpu/session-20261003/hub-bank-candidate`.
+
+`test_placement_capture_lookup.py <placement-containers> <index.cpp>
+<runtime-cache.cpp> <output-directory>` exercises the production lookup against
+both the physical and instruction sections of each actual capture, requiring
+the exact compiled container hash. All 57 new containers pass. Raw placement
+fragments remain diagnostic-only and must never be compiler inputs.
+
+The second native capture (`oct03-hub-bank175`) shows restored HUD icons and
+additional hub geometry, and captured no further placement containers during
+this short run. The character portrait remains a blank blue circle. This is
+a partial improvement, not completed hub correctness or performance validation.
+Both native probes were closed; further probes use monitor 2 per the user's
+October 3 instruction. Do not infer a performance result from snapshot/trace
+runs. Emulated comparison and further character/portrait investigation remain.
+
 The production runtime bank now contains 118 validated containers, 18722
 microcode signatures and 192 linked runtime variants. It retains the current
 624-byte constants ABI. The six late containers had valid nonzero instruction
diff --git a/tools/gpu-aot/test_placement_capture_lookup.py b/tools/gpu-aot/test_placement_capture_lookup.py
new file mode 100644
index 0000000..51fecc8
--- /dev/null
+++ b/tools/gpu-aot/test_placement_capture_lookup.py
@@ -0,0 +1,96 @@
+"""Verify exact captured shader sections through the production AOT lookup.
+
+Inputs remain local: validated placement-containers, the installed index, and
+runtime cache. No game shader bytes are copied into the repository.
+"""
+import argparse
+from pathlib import Path
+import re
+import struct
+import subprocess
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('captures', type=Path)
+    p.add_argument('index', type=Path)
+    p.add_argument('cache', type=Path)
+    p.add_argument('output', type=Path)
+    a = p.parse_args()
+    root = Path(__file__).resolve().parents[2]
+    a.output.mkdir(parents=True, exist_ok=True)
+    cache = a.cache.read_text()
+    hashes = re.findall(r'\{ 0x([0-9A-Fa-f]+), \d+, \d+, \d+, \d+, \d+ \}', cache)
+    if not hashes:
+        raise ValueError('No compiled cache entries')
+    index = a.index.read_text()
+    sizes = re.findall(r'\{\s*0x[\dA-Fa-f]+,\s*0x[\dA-Fa-f]+,\s*0x[\dA-Fa-f]+,\s*(\d+),', index)
+    padding = max(map(int, sizes))
+    source = (root / 'rexlego/src/gpu_native/shader_archive.cpp').read_text()
+    start = source.index('const ShaderCacheEntry* FindShaderByMicrocode(')
+    end = source.index('\nuint64_t HashShaderContainer(', start)
+    cases = []
+    for path in sorted(a.captures.glob('*.bin')):
+        data = path.read_bytes()
+        flags, virtual, physical = struct.unpack_from('>III', data)
+        assert flags in (0x102A1100, 0x102A1101)
+        assert len(data) == virtual + 4 + physical
+        assert struct.unpack_from('>I', data, virtual)[0] == physical
+        shader = struct.unpack_from('>I', data, 24)[0]
+        assert 36 <= shader and shader + 8 <= virtual
+        offset, size = struct.unpack_from('>II', data, shader)
+        assert size >= 8 and offset + size <= physical
+        assert any(data[virtual + 4 + offset:virtual + 4 + offset + size])
+        path_literal = '"' + str(path.resolve()).replace('\\', '/') + '"'
+        cases.append(f'Check({path_literal}, {virtual + 4}, {offset}, {int(flags == 0x102A1100)});')
+    if not cases:
+        raise ValueError('No validated captures')
+    code = r'''
+#include <cassert>
+#include <fstream>
+#include <iterator>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+#include <iostream>
+#define XXH_INLINE_ALL
+#include <xxhash.h>
+#include "gpu_native/shader_archive.h"
+''' + index + '\nnamespace {\nShaderCacheEntry cache[] = {\n' + ''.join(
+        f'{{0x{h},0,0,0,0,0}},\n' for h in hashes) + r'''};
+const ShaderCacheEntry* FindShader(uint64_t hash) {
+ for (const auto& entry : cache) if (entry.hash == hash) return &entry;
+ return nullptr;
+}
+''' + source[start:end] + f'\nconstexpr size_t padding={padding};\n' + r'''
+void Check(const char* path, size_t physical, size_t offset, uint32_t stage) {
+ std::ifstream file(path,std::ios::binary);
+ std::vector<uint8_t> bytes((std::istreambuf_iterator<char>(file)), {});
+ const auto expected = XXH3_64bits(bytes.data(),bytes.size());
+ // The production lookup tests several candidate lengths. Supply zeroed
+ // readable trailing storage, as the guest committed allocation does.
+ bytes.resize(bytes.size()+padding);
+ for (auto start : {physical,physical+offset}) {
+  auto* result=FindShaderByMicrocode(bytes.data()+start,stage);
+  if (!result || result->hash!=expected) {
+   std::cerr << path << " section " << start << " failed exact lookup\n";
+   std::abort();
+  }
+ }
+}
+}
+int main() {
+''' + '\n'.join(cases) + '\n}\n'
+    cpp = a.output / 'captured-lookup.cpp'
+    cpp.write_text(code)
+    exe = a.output / 'captured-lookup.exe'
+    subprocess.run(['clang++', '-std=c++20', '-O2', '-I' + str(root / 'rexlego/src'),
+                    '-I' + str(root / 'rexglue-sdk/thirdparty/xxHash'), str(cpp),
+                    '-o', str(exe)], check=True)
+    subprocess.run([str(exe.resolve())], check=True)
+    print(f'PASS: {len(cases)} captured containers, physical and instruction '
+          'sections resolve to their exact compiled shader through production lookup')
+
+
+if __name__ == '__main__':
+    main()

From 460e67029f7f05d29b5d6e91869cc9161e20556d Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 21:16:08 +0300
Subject: [PATCH 21/44] Include the main shader bank in captured lookup
 verification

---
 tools/gpu-aot/test_placement_capture_lookup.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tools/gpu-aot/test_placement_capture_lookup.py b/tools/gpu-aot/test_placement_capture_lookup.py
index 51fecc8..3702dd6 100644
--- a/tools/gpu-aot/test_placement_capture_lookup.py
+++ b/tools/gpu-aot/test_placement_capture_lookup.py
@@ -16,6 +16,8 @@ def main():
     p.add_argument('index', type=Path)
     p.add_argument('cache', type=Path)
     p.add_argument('output', type=Path)
+    p.add_argument('--main-cache', type=Path,
+                   help='Also expose the compiled main archive, as gameplay does')
     a = p.parse_args()
     root = Path(__file__).resolve().parents[2]
     a.output.mkdir(parents=True, exist_ok=True)
@@ -23,6 +25,9 @@ def main():
     hashes = re.findall(r'\{ 0x([0-9A-Fa-f]+), \d+, \d+, \d+, \d+, \d+ \}', cache)
     if not hashes:
         raise ValueError('No compiled cache entries')
+    if a.main_cache:
+        hashes += re.findall(r'\{ 0x([0-9A-Fa-f]+), \d+, \d+, \d+, \d+, \d+ \}',
+                             a.main_cache.read_text())
     index = a.index.read_text()
     sizes = re.findall(r'\{\s*0x[\dA-Fa-f]+,\s*0x[\dA-Fa-f]+,\s*0x[\dA-Fa-f]+,\s*(\d+),', index)
     padding = max(map(int, sizes))

From 2b560c4f6e47e8027420d8b1eb7aa542be7a2fdd Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 21:18:03 +0300
Subject: [PATCH 22/44] Preserve native presentation aspect ratio across window
 sizes

---
 rexlego/src/gpu_native/device.cpp     |  9 ++++++-
 rexlego/src/gpu_native/present_rect.h | 23 ++++++++++++++++
 tools/gpu-aot/test_present_rect.cpp   | 39 +++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100644 rexlego/src/gpu_native/present_rect.h
 create mode 100644 tools/gpu-aot/test_present_rect.cpp

diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 909b6fa..965a948 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -1,6 +1,7 @@
 #include "gpu_native/device.h"
 #include "gpu_native/descriptor_retirement.h"
 #include "gpu_native/completion_queue.h"
+#include "gpu_native/present_rect.h"
 
 #include "gpu_native/buffers.h"
 #include "gpu_native/queries.h"
@@ -1170,8 +1171,14 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
 
   const u32 width = state.swap_chain->getWidth();
   const u32 height = state.swap_chain->getHeight();
+  const auto source_desc = static_cast<plume::D3D12Texture*>(texture)->d3d->GetDesc();
+  const auto present_rect = FitPresentRect(u32(source_desc.Width), source_desc.Height,
+                                         width, height);
+  // Clear the whole acquired image, including bars left by a previous size.
+  commands->clearColor(0, plume::RenderColor(0.0f, 0.0f, 0.0f, 1.0f));
   commands->setViewports(
-      plume::RenderViewport(0.0f, 0.0f, float(width), float(height)));
+      plume::RenderViewport(float(present_rect.x), float(present_rect.y),
+                            float(present_rect.width), float(present_rect.height)));
   commands->setScissors(plume::RenderRect(0, 0, width, height));
   commands->setGraphicsPipelineLayout(state.pipeline_layout.get());
   commands->setGraphicsDescriptorSet(state.texture_descriptors.get(), 0);
diff --git a/rexlego/src/gpu_native/present_rect.h b/rexlego/src/gpu_native/present_rect.h
new file mode 100644
index 0000000..2497827
--- /dev/null
+++ b/rexlego/src/gpu_native/present_rect.h
@@ -0,0 +1,23 @@
+#pragma once
+#include <algorithm>
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct PresentRect {
+  uint32_t x, y, width, height;
+};
+
+// Fit the complete game image, including HUD, into the host window. Integer
+// products avoid aspect comparison errors and overflow at large dimensions.
+constexpr PresentRect FitPresentRect(uint32_t source_width, uint32_t source_height,
+                                     uint32_t window_width, uint32_t window_height) {
+  if (!source_width || !source_height || !window_width || !window_height)
+    return {0, 0, window_width, window_height};
+  uint32_t width = window_width, height = window_height;
+  if (uint64_t(window_width) * source_height > uint64_t(window_height) * source_width)
+    width = std::max(1u, uint32_t(uint64_t(window_height) * source_width / source_height));
+  else
+    height = std::max(1u, uint32_t(uint64_t(window_width) * source_height / source_width));
+  return {(window_width - width) / 2, (window_height - height) / 2, width, height};
+}
+} // namespace legodimensions::gpu_native
diff --git a/tools/gpu-aot/test_present_rect.cpp b/tools/gpu-aot/test_present_rect.cpp
new file mode 100644
index 0000000..a93949c
--- /dev/null
+++ b/tools/gpu-aot/test_present_rect.cpp
@@ -0,0 +1,39 @@
+#include <cassert>
+#include <cstdint>
+#include <limits>
+#include "gpu_native/present_rect.h"
+using legodimensions::gpu_native::FitPresentRect;
+
+int main() {
+  auto r = FitPresentRect(1280, 720, 1920, 1080);
+  assert(r.x == 0 && r.y == 0 && r.width == 1920 && r.height == 1080);
+  r = FitPresentRect(1280, 720, 720, 1280);
+  assert(r.x == 0 && r.y == 437 && r.width == 720 && r.height == 405);
+  r = FitPresentRect(1280, 720, 3440, 1440);
+  assert(r.x == 440 && r.y == 0 && r.width == 2560 && r.height == 1440);
+  r = FitPresentRect(1280, 720, 1001, 1001);
+  assert(r.x == 0 && r.y == 219 && r.width == 1001 && r.height == 563);
+  r = FitPresentRect(1280, 720, 1, 1);
+  assert(r.width == 1 && r.height == 1);
+  r = FitPresentRect(1280, 720, 0, 0);
+  assert(r.width == 0 && r.height == 0);
+  r = FitPresentRect(0, 0, 1920, 1080);
+  assert(r.width == 1920 && r.height == 1080);
+  // A resize may jump between portrait and landscape. Every fitted rectangle
+  // remains centered and bounded, with less than one pixel of rounding error.
+  for (uint32_t w = 1; w <= 4096; w += 17) {
+    for (uint32_t h = 1; h <= 2160; h += 13) {
+      r = FitPresentRect(1280, 720, w, h);
+      assert(r.width && r.height && r.x + r.width <= w && r.y + r.height <= h);
+      assert(w - r.width - 2 * r.x <= 1 && h - r.height - 2 * r.y <= 1);
+      assert(r.width == w || r.height == h);
+      if (w >= 2 && h >= 2) {
+        const int64_t error = int64_t(r.width) * 720 - int64_t(r.height) * 1280;
+        assert(error > -720 && error < 1280);
+      }
+    }
+  }
+  const auto maximum = std::numeric_limits<uint32_t>::max();
+  r = FitPresentRect(maximum, maximum - 1, maximum - 1, maximum);
+  assert(r.width <= maximum - 1 && r.height <= maximum);
+}

From 4c43b28fc686c2d5440c7431b8cce4eb3a5b1d4c Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 21:39:23 +0300
Subject: [PATCH 23/44] Preserve alpha when correcting native color-resolve
 swizzles

---
 .../src/gpu_native/color_resolve_swizzle.h    | 14 +++++++
 rexlego/src/gpu_native/device.cpp             | 17 ++++++++-
 rexlego/src/gpu_native/textures.cpp           | 19 +++++++++-
 .../native-transition-findings-20261003.md    | 28 ++++++++++++++
 tools/gpu-aot/test_color_resolve_swizzle.cpp  | 38 +++++++++++++++++++
 5 files changed, 113 insertions(+), 3 deletions(-)
 create mode 100644 rexlego/src/gpu_native/color_resolve_swizzle.h
 create mode 100644 tools/gpu-aot/native-transition-findings-20261003.md
 create mode 100644 tools/gpu-aot/test_color_resolve_swizzle.cpp

diff --git a/rexlego/src/gpu_native/color_resolve_swizzle.h b/rexlego/src/gpu_native/color_resolve_swizzle.h
new file mode 100644
index 0000000..d5bae25
--- /dev/null
+++ b/rexlego/src/gpu_native/color_resolve_swizzle.h
@@ -0,0 +1,14 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+// Guest BGRA uploads use ZYX in RGB. Native color resolves already contain
+// logical RGBA. Preserve the guest alpha selection, including constant one
+// used by the title/menu transition, while removing this storage conversion.
+constexpr uint32_t NativeColorResolveSwizzle(uint32_t guest_swizzle) {
+  if ((guest_swizzle & 0x1FFu) != 0x00Au) return guest_swizzle;
+  const uint32_t alpha = (guest_swizzle >> 9) & 7u;
+  const uint32_t logical_alpha = alpha == 0 ? 2 : alpha == 2 ? 0 : alpha;
+  return (guest_swizzle & ~0xFFFu) | (logical_alpha << 9) | 0x088u;
+}
+} // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 965a948..65fbe44 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -194,6 +194,7 @@ struct State {
   std::array<double, static_cast<u32>(SyncReason::kCount)> sync_reason_ms{};
   std::chrono::steady_clock::time_point probe_next_snapshot{};
   std::chrono::steady_clock::time_point probe_next_present{};
+  u32 probe_present_burst_remaining = 0;
   std::chrono::steady_clock::time_point probe_last_anomaly_snapshot{};
   u64 probe_snapshot_bytes = 0;
   bool probe_budget_reported = false;
@@ -1365,6 +1366,17 @@ void HostDevice::SnapshotTexture(plume::RenderTexture* texture,
       // The trigger can be noticed in the middle of a pass. Capture the next
       // frame from its first snapshot, not just the remainder of this one.
       state.snapshot_selected_frame = state.present_number + 1;
+    const auto burst_path = SnapshotDirectory() / "capture-present-frames";
+    std::ifstream burst_file(burst_path);
+    u32 requested_frames = 0;
+    if (burst_file >> requested_frames) {
+      burst_file.close();
+      if (std::filesystem::remove(burst_path, trigger_error)) {
+        state.probe_present_burst_remaining = std::min(requested_frames, 240u);
+        if (LongProbeEnabled()) LongProbeEvent("present_burst_armed", false,
+            "frames=", state.probe_present_burst_remaining);
+      }
+    }
   }
   static const bool time_windows = [] {
     char* value = nullptr;
@@ -1384,7 +1396,10 @@ void HostDevice::SnapshotTexture(plume::RenderTexture* texture,
       LongProbeEvent("snapshot_scheduled", false, "target_frame=", state.snapshot_selected_frame,
           "reason=", anomaly_due ? "anomaly_candidate" : "periodic");
     }
-    const bool present_sample = label=="present" && now>=state.probe_next_present;
+    const bool present_burst = label=="present" && state.probe_present_burst_remaining;
+    if (present_burst) --state.probe_present_burst_remaining;
+    const bool present_sample = present_burst ||
+        (label=="present" && now>=state.probe_next_present);
     if (present_sample) state.probe_next_present=now+std::chrono::seconds(5);
     if (state.present_number != state.snapshot_selected_frame && !present_sample) return;
   } else if (time_windows) {
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index 8a80655..22c230f 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -1,4 +1,5 @@
 #include "gpu_native/textures.h"
+#include "gpu_native/color_resolve_swizzle.h"
 
 #include <algorithm>
 #include <array>
@@ -301,11 +302,12 @@ std::shared_ptr<TextureResource> AdoptTexture(u32 guest_address) {
     // A native color resolve instead copies logical RGBA render-target data.
     // Give that representation its own immutable SRV; applying the guest
     // storage swizzle a second time swaps red and blue in every resolved image.
+    const u32 resolved_swizzle = NativeColorResolveSwizzle(swizzle);
     if (resource->format == plume::RenderFormat::R8G8B8A8_UNORM &&
-        swizzle == 0x60Au) {
+        resolved_swizzle != swizzle) {
       view_desc.componentMapping = plume::RenderComponentMapping(
           plume::RenderSwizzle::R, plume::RenderSwizzle::G,
-          plume::RenderSwizzle::B, plume::RenderSwizzle::A);
+          plume::RenderSwizzle::B, swizzles[(resolved_swizzle >> 9) & 7]);
       resource->resolved_view = resource->texture->createTextureView(view_desc);
       if (resource->resolved_view)
         resource->resolved_descriptor_index = HostDevice::RegisterTexture(
@@ -919,6 +921,19 @@ TextureResourceView ResolveTextureResource(u32 guest_address) {
             resource->width, resource->height, resource->d3d_type, false, false};
   const bool native_color = resource->resolved_on_host &&
       resource->resolved_descriptor_index != ~u32{0};
+  if (LongProbeEnabled()) {
+    const u64 signature = (u64(guest_address) << 32) ^
+        (u64(resource->descriptor_index) << 3) ^
+        (u64(resource->resolved_descriptor_index) << 11) ^
+        (u64(resource->resolved_on_host) << 1) ^ u64(native_color);
+    if (LongProbeOnce(signature ^ 0xC010A07000000000ull))
+      LongProbeEvent("texture_color_authority", false, "guest=", guest_address,
+          "host_resolved=", resource->resolved_on_host, "logical_rgba=", native_color,
+          "raw_descriptor=", resource->descriptor_index,
+          "resolved_descriptor=", resource->resolved_descriptor_index,
+          "guest_swizzle=", u32(resource->guest_fetch.swizzle),
+          "width=", resource->width, "height=", resource->height);
+  }
   return {resource->texture.get(),
           native_color ? resource->resolved_view.get() : resource->view.get(),
           resource->format,
diff --git a/tools/gpu-aot/native-transition-findings-20261003.md b/tools/gpu-aot/native-transition-findings-20261003.md
new file mode 100644
index 0000000..aaa77b3
--- /dev/null
+++ b/tools/gpu-aot/native-transition-findings-20261003.md
@@ -0,0 +1,28 @@
+# Enter transition color investigation, October 3
+
+The user's title screenshot shows a blue LEGO tile and yellow lights, where
+the normal title has a red tile and cyan lights. Native trace
+`session-20260930/trace-oct03-enter-burst/events.tsv` caught a 1280x720
+host-resolved RGBA8 texture at frame2334 using guest swizzle0xA0A (ZYX1),
+raw descriptor515, with no logical-RGBA descriptor. The previous correction
+created a separate resolved view only for0x60A (ZYXW). Thus this transition
+variant exchanged red and blue a second time when sampling native RGBA data.
+
+`NativeColorResolveSwizzle` removes the BGRA storage conversion for all ZYX
+RGB layouts while retaining alpha selection, including constant ONE. CPU
+uploads keep their original view. `test_color_resolve_swizzle.cpp` checks the
+observed red/cyan case, all eight alpha selections by differential comparison
+against physical BGRA storage, and unchanged behavior for other RGB layouts.
+Core tests and native Release build pass. Post-fix runtime pixels are NOT
+verified: the user prohibited further game launches after this test.
+
+Opt-in tracing now records texture color authority and supports a trigger
+file `capture-present-frames` containing a frame count (maximum240). This
+captures consecutive final textures within the existing probe byte budget;
+regular rendering does not enable these readbacks. The diagnostic probe
+`oct03-enter-burst` captured240 frames and was closed. Heavy captures are not
+valid performance measurements.
+
+The earlier aspect-ratio correction was confirmed in normal title window
+captures: the portrait monitor window now has black letterbox bars. This does
+not establish the remaining HUD portrait or general gameplay correctness.
diff --git a/tools/gpu-aot/test_color_resolve_swizzle.cpp b/tools/gpu-aot/test_color_resolve_swizzle.cpp
new file mode 100644
index 0000000..ba24a75
--- /dev/null
+++ b/tools/gpu-aot/test_color_resolve_swizzle.cpp
@@ -0,0 +1,38 @@
+#include <array>
+#include <cassert>
+#include "gpu_native/color_resolve_swizzle.h"
+using legodimensions::gpu_native::NativeColorResolveSwizzle;
+constexpr std::array<unsigned, 4> Apply(unsigned swizzle,
+                                       std::array<unsigned, 4> color) {
+  std::array<unsigned, 4> output{};
+  for (unsigned c = 0; c < 4; ++c) {
+    unsigned component = (swizzle >> (3 * c)) & 7;
+    output[c] = component < 4 ? color[component] : component == 5 ? 255 : 0;
+  }
+  return output;
+}
+int main() {
+  static_assert(NativeColorResolveSwizzle(0x60A) == 0x688);
+  static_assert(NativeColorResolveSwizzle(0xA0A) == 0xA88);
+  static_assert(NativeColorResolveSwizzle(0x80A) == 0x888);
+  // The exact transition variant must keep red red and cyan cyan, with opaque
+  // alpha. The former raw ZYX1 view would exchange red and blue again.
+  assert((Apply(NativeColorResolveSwizzle(0xA0A), {255, 0, 0, 37}) ==
+          std::array<unsigned, 4>{255, 0, 0, 255}));
+  assert((Apply(NativeColorResolveSwizzle(0xA0A), {0, 255, 255, 37}) ==
+          std::array<unsigned, 4>{0, 255, 255, 255}));
+  assert(Apply(0xA0A, {255, 0, 0, 37})[2] == 255);
+  // Ordinary guest uploads retain their original view; only the distinct
+  // resolved view uses this mapping. Other RGB layouts are left untouched.
+  for (unsigned alpha = 0; alpha < 8; ++alpha) {
+    unsigned guest = (alpha << 9) | 0xA;
+    unsigned host = NativeColorResolveSwizzle(guest);
+    // Applying the original view to actual BGRA storage must equal applying
+    // the resolved view to logical RGBA, including alpha sourced from RGB.
+    assert((host & 0x1FF) == 0x88);
+    assert(Apply(guest, {53, 31, 17, 79}) == Apply(host, {17, 31, 53, 79}));
+  }
+  for (unsigned swizzle = 0; swizzle < 4096; ++swizzle)
+    if ((swizzle & 0x1FF) != 0xA)
+      assert(NativeColorResolveSwizzle(swizzle) == swizzle);
+}

From f08fee0883ec46a6c5869b5e69a5d9c4d2df6312 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 21:45:42 +0300
Subject: [PATCH 24/44] Add audited opt-in generation caching for native buffer
 uploads

---
 rexlego/src/gpu_native/buffers.cpp            | 29 ++++++++-
 rexlego/src/gpu_native/buffers.h              |  4 ++
 rexlego/src/gpu_native/device.cpp             |  5 +-
 rexlego/src/gpu_native/memory_watch.cpp       | 19 +++++-
 rexlego/src/gpu_native/memory_watch.h         |  1 +
 rexlego/src/gpu_native/textures.cpp           |  2 +-
 tools/gpu-aot/native-buffer-watch-20261003.md | 46 +++++++++++++
 tools/gpu-aot/run_native_probe.ps1            |  9 ++-
 tools/gpu-aot/test_buffer_variants.py         | 65 ++++++++++++++++++-
 tools/gpu-aot/test_memory_watch.py            |  6 +-
 tools/gpu-aot/test_memory_watch_sdk.py        |  8 ++-
 11 files changed, 179 insertions(+), 15 deletions(-)
 create mode 100644 tools/gpu-aot/native-buffer-watch-20261003.md

diff --git a/rexlego/src/gpu_native/buffers.cpp b/rexlego/src/gpu_native/buffers.cpp
index c10ba71..d3d1efc 100644
--- a/rexlego/src/gpu_native/buffers.cpp
+++ b/rexlego/src/gpu_native/buffers.cpp
@@ -3,6 +3,7 @@
 #include "gpu_native/memory_watch.h"
 
 #include <cstring>
+#include <cstdlib>
 #include <algorithm>
 #include <atomic>
 #include <chrono>
@@ -223,10 +224,32 @@ plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr<BufferResource>
   const bool timing = NativeTextureTimingEnabled();
   const auto hash_start = timing ? std::chrono::steady_clock::now()
                                 : std::chrono::steady_clock::time_point{};
-  const u64 content_hash = XXH3_64bits(source, resource->length);
+  const bool watch_enabled = CpuBufferMemoryWatchEnabled();
+  const bool watch_hit = watch_enabled && !resource->variants.empty() &&
+      CpuMemoryUnchanged(resource->cpu_stamp);
+  static const bool audit_watch = std::getenv("LEGO_NATIVE_AUDIT_BUFFER_WATCH") != nullptr;
+  CpuMemoryStamp next_stamp;
+  u64 content_hash = resource->content_hash;
+  if (!watch_hit || audit_watch) {
+    // Arm before hashing/conversion. A concurrent write invalidates this
+    // stamp, so a mixed snapshot cannot remain cached as permanently clean.
+    if (watch_enabled) {
+      const CpuMemorySpan span{resource->mirror_address, resource->length};
+      next_stamp = WatchCpuMemory({&span, 1});
+    }
+    content_hash = XXH3_64bits(source, resource->length);
+    if (timing) g_buffer_timing.hashed_bytes += resource->length;
+    if (watch_hit && audit_watch) {
+      if (timing) ++g_buffer_timing.watch_audits;
+      if (content_hash != resource->content_hash) {
+        if (timing) ++g_buffer_timing.watch_mismatches;
+        REXLOG_ERROR("Native buffer watch: stale clean stamp guest={:08X} data={:08X} bytes={}",
+            guest_address, resource->mirror_address, resource->length);
+      }
+    }
+  } else if (timing) ++g_buffer_timing.watch_hits;
   if (timing) {
     ++g_buffer_timing.calls;
-    g_buffer_timing.hashed_bytes += resource->length;
     g_buffer_timing.hash_ms += std::chrono::duration<double, std::milli>(
         std::chrono::steady_clock::now() - hash_start).count();
   }
@@ -235,6 +258,7 @@ plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr<BufferResource>
     resource->variants.clear();
     resource->buffer.reset();
   } else if (const auto it = resource->variants.find(byte_order_hash); it != resource->variants.end()) {
+    if (!watch_hit || audit_watch) resource->cpu_stamp = std::move(next_stamp);
     resource->buffer = it->second;
     return it->second.get();
   }
@@ -297,6 +321,7 @@ plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr<BufferResource>
   resource->buffer = std::move(next_buffer);
   resource->content_hash = content_hash;
   resource->byte_order_hash = byte_order_hash;
+  if (!watch_hit || audit_watch) resource->cpu_stamp = std::move(next_stamp);
   return resource->buffer.get();
 }
 
diff --git a/rexlego/src/gpu_native/buffers.h b/rexlego/src/gpu_native/buffers.h
index cf5d2bb..37c553c 100644
--- a/rexlego/src/gpu_native/buffers.h
+++ b/rexlego/src/gpu_native/buffers.h
@@ -20,6 +20,9 @@ struct BufferUploadTiming {
   u64 calls = 0;
   u64 hashed_bytes = 0;
   u64 converted_bytes = 0;
+  u64 watch_hits = 0;
+  u64 watch_audits = 0;
+  u64 watch_mismatches = 0;
   double hash_ms = 0;
 };
 BufferUploadTiming ConsumeBufferUploadTiming();
@@ -30,6 +33,7 @@ u32 LockBufferResource(u32 guest_address, u32 offset, u32 size, u32 flags,
                        BufferKind kind);
 
 // Checks current guest contents on every call, including placement buffers.
+// Opt-in buffer watches use physical page generations; unknown memory hashes.
 // Changed contents get a new upload buffer retained through the frame fence,
 // so later CPU writes cannot overwrite geometry in earlier recorded draws.
 plume::RenderBuffer* ResolveBufferResource(u32 guest_address, BufferKind kind,
diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 65fbe44..5cb8a35 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -1270,8 +1270,9 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
             "pipeline={:.2f} constants={:.2f} vertices={:.2f} issue={:.2f} tail={:.2f}ms",
             draws.calls, draws.stages_ms[0], draws.stages_ms[1], draws.stages_ms[2],
             draws.stages_ms[3], draws.stages_ms[4], draws.stages_ms[5], draws.stages_ms[6]);
-        REXLOG_INFO("Native buffer timing: calls={} hash={:.2f}ms hashed={} converted={}",
-            buffers.calls, buffers.hash_ms, buffers.hashed_bytes, buffers.converted_bytes);
+        REXLOG_INFO("Native buffer timing: calls={} hash={:.2f}ms hashed={} converted={} watch_hits={} watch_audits={} watch_mismatches={}",
+            buffers.calls, buffers.hash_ms, buffers.hashed_bytes, buffers.converted_bytes,
+            buffers.watch_hits, buffers.watch_audits, buffers.watch_mismatches);
         REXLOG_INFO("Native synchronization timing: calls={} cpu_resource_skips={} time={:.2f}ms",
                     state.sync_calls, state.cpu_resource_wait_skips, state.sync_ms);
         REXLOG_INFO("Native synchronization reasons ms: other={:.2f} idle={:.2f} fence={:.2f} resource={:.2f} callback={:.2f} query_begin={:.2f} query_release={:.2f}",
diff --git a/rexlego/src/gpu_native/memory_watch.cpp b/rexlego/src/gpu_native/memory_watch.cpp
index 8cf7034..3c9f8ac 100644
--- a/rexlego/src/gpu_native/memory_watch.cpp
+++ b/rexlego/src/gpu_native/memory_watch.cpp
@@ -12,6 +12,9 @@
 REXCVAR_DEFINE_BOOL(gpu_native_texture_watch, false, "GPU",
     "Reuse unchanged native pixel textures with physical page write invalidation")
     .lifecycle(rex::cvar::Lifecycle::kRequiresRestart);
+REXCVAR_DEFINE_BOOL(gpu_native_buffer_watch, false, "GPU",
+    "Reuse unchanged native vertex/index uploads with physical page write invalidation")
+    .lifecycle(rex::cvar::Lifecycle::kRequiresRestart);
 
 namespace legodimensions::gpu_native {
 namespace {
@@ -24,8 +27,7 @@ rex::memory::Memory* g_memory = nullptr;
 void* g_callback = nullptr;
 
 bool SelectedMemoryWatch() {
-  // Only pixel textures may use this path. Vertex animation textures and
-  // vertex/index buffers retain every-use content checks.
+  // Vertex animation textures retain every-use content checks.
   static const bool enabled = [] {
     const char* value = std::getenv("LEGO_NATIVE_STATIC_TEXTURE_WATCH");
     const bool selected = value ? std::strcmp(value, "0") != 0 :
@@ -35,6 +37,16 @@ bool SelectedMemoryWatch() {
   return enabled;
 }
 
+bool SelectedBufferWatch() {
+  static const bool enabled = [] {
+    const char* value = std::getenv("LEGO_NATIVE_BUFFER_WATCH");
+    const bool selected = value ? std::strcmp(value, "0") != 0 :
+        REXCVAR_GET(gpu_native_buffer_watch);
+    return selected && !std::getenv("LEGO_NATIVE_NO_MEMORY_WATCH");
+  }();
+  return enabled;
+}
+
 void Invalidate(uint32_t address, uint32_t length) {
   if (!length || address >= kPhysicalSize) return;
   const uint64_t end = std::min(uint64_t(address) + length, uint64_t(kPhysicalSize));
@@ -75,10 +87,11 @@ bool PhysicalSpan(rex::memory::Memory* memory, CpuMemorySpan input, CpuMemorySpa
 }  // namespace
 
 bool CpuMemoryWatchEnabled() { return SelectedMemoryWatch(); }
+bool CpuBufferMemoryWatchEnabled() { return SelectedBufferWatch(); }
 
 CpuMemoryStamp WatchCpuMemory(std::span<const CpuMemorySpan> spans) {
   CpuMemoryStamp stamp;
-  if (!CpuMemoryWatchEnabled()) return stamp;
+  if (!SelectedMemoryWatch() && !SelectedBufferWatch()) return stamp;
   auto lock = g_watch_region.Acquire();
   auto* memory = REX_KERNEL_MEMORY();
   if (!memory) return stamp;
diff --git a/rexlego/src/gpu_native/memory_watch.h b/rexlego/src/gpu_native/memory_watch.h
index 1727ab1..a2c608b 100644
--- a/rexlego/src/gpu_native/memory_watch.h
+++ b/rexlego/src/gpu_native/memory_watch.h
@@ -12,6 +12,7 @@ struct CpuMemoryStamp {
   std::vector<std::pair<uint32_t, uint64_t>> pages;
 };
 bool CpuMemoryWatchEnabled();
+bool CpuBufferMemoryWatchEnabled();
 
 // Uses the SDK's physical write notifications across A/C/E aliases. Unknown
 // virtual storage has no stamp and must still be hashed at each use.
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index 22c230f..90c9dad 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -1021,7 +1021,7 @@ bool UploadTextureResource(u32 guest_address,
     const CpuMemorySpan source_spans[] = {
       {u32(fetch.base_address << 12), layout.base.level_data_extent_bytes},
       {u32(fetch.mip_address << 12), mip_size}};
-    auto cpu_stamp = WatchCpuMemory(source_spans);
+    auto cpu_stamp = CpuMemoryWatchEnabled() ? WatchCpuMemory(source_spans) : CpuMemoryStamp{};
     const auto source_start = timer.enabled ? std::chrono::steady_clock::now()
                                             : std::chrono::steady_clock::time_point{};
     const bool source_key_valid =
diff --git a/tools/gpu-aot/native-buffer-watch-20261003.md b/tools/gpu-aot/native-buffer-watch-20261003.md
new file mode 100644
index 0000000..f90b087
--- /dev/null
+++ b/tools/gpu-aot/native-buffer-watch-20261003.md
@@ -0,0 +1,46 @@
+# Native buffer generation candidate, October 3
+
+Previous title measurements attributed9.51ms/frame to buffer hashing; the
+window candidate reduced bytes hashed but did not prove an overall FPS gain
+(see `native-performance-20261001.md`). This candidate avoids full content
+hashes when the SDK's physical page generations prove an upload unchanged.
+
+Selection: `gpu_native_buffer_watch=true`, or `LEGO_NATIVE_BUFFER_WATCH=1`.
+Default OFF. Probe switches `-BufferWatch` and `-AuditBufferWatch` explicitly
+select it; audit also sets `LEGO_NATIVE_AUDIT_BUFFER_WATCH=1`. The global
+`LEGO_NATIVE_NO_MEMORY_WATCH` disables both texture and buffer watches.
+Texture and buffer selection remain independent. Vertex animation textures
+continue to hash on every use.
+
+Physical writes through guest aliases invalidate shared generations. A dirty
+range arms its watch before hashing/conversion. Unknown virtual backing falls
+back to hashing every use. Changed contents still create immutable upload
+versions retained through their frame fence. A watch hit can reuse a different
+already-cached declaration conversion, or create that variant without changing
+older uploads. Host physical pool copies explicitly invalidate generations.
+Device shutdown invalidates every stamp.
+
+Audit hashes every otherwise-clean range, reports stale clean stamps as errors,
+and repairs the conversion using the new data. Timing logs include watch hits,
+audit checks and mismatches. Audit runs measure safety, not speed.
+
+Offline verification passed:
+
+- Actual production buffer conversion/adoption/window functions: existing10000
+  fetched-byte equivalence and1000 rebasing cases, same-frame updates, immutable
+  older uploads and bounded cache. Additional cases reuse100 watched draws with
+  zero source hashing/allocation, re-invalidate on two same-frame writes, detect
+  and repair a deliberately missed notification under audit, and hash every
+  unknown-memory draw. Tests use a fake GPU driver and page-generation model.
+- Production watcher with SDK callback model: alias mapping, races, neighbor
+  protection, physical copies, fallback and global disable. Repeated with
+  buffer-only selection while texture watches are disabled.
+- Actual installed SDK: real protected-page CPU writes,100 rearm cycles,
+  Protect, explicit host physical writes and Decommit. Passed environment and
+  config selection for both texture and buffer watches.
+- Native Release build and embedded build fingerprint verified.
+
+No new gameplay or FPS measurement. The user prohibited launching the game
+after the Enter capture. This remains opt-in until a monitor2 gameplay audit
+and clean, comparable performance run are authorized. Async CPU-resource
+wait bypass also remains OFF; this change does not alter its safety question.
diff --git a/tools/gpu-aot/run_native_probe.ps1 b/tools/gpu-aot/run_native_probe.ps1
index f21e394..73c6dae 100644
--- a/tools/gpu-aot/run_native_probe.ps1
+++ b/tools/gpu-aot/run_native_probe.ps1
@@ -7,6 +7,8 @@ param(
     [switch]$NoMemoryWatch,
     [switch]$StaticTextureWatch,
     [switch]$AuditTextureWatch,
+    [switch]$BufferWatch,
+    [switch]$AuditBufferWatch,
     [switch]$NoDrawArena,
     [switch]$AsyncCpuResources,
     [switch]$Stencil,
@@ -25,7 +27,7 @@ param(
 $ErrorActionPreference = 'Stop'
 if ($ConfiguredRenderer -and $Pm4Reference) { throw 'Select configured renderer or explicit PM4 reference, not both' }
 if ($DepthAlias -and $DisableDepthAlias) { throw 'DepthAlias and DisableDepthAlias are mutually exclusive' }
-if (($Pm4Reference -or $ConfiguredRenderer) -and ($Stencil -or $MeshTrace -or $LogoUploads -or $Viewport -or $BufferWindows -or $DepthAlias -or $DisableDepthAlias -or $LongProbe -or $LogoCaptureRenderDocDll)) {
+if (($Pm4Reference -or $ConfiguredRenderer) -and ($Stencil -or $MeshTrace -or $LogoUploads -or $Viewport -or $BufferWindows -or $BufferWatch -or $AuditBufferWatch -or $DepthAlias -or $DisableDepthAlias -or $LongProbe -or $LogoCaptureRenderDocDll)) {
     throw 'PM4 reference mode uses original XDK/SDK objects; native-only diagnostic/candidate switches must be disabled'
 }
 $projectRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
@@ -50,7 +52,7 @@ $keys = @('SDL_WINDOW_ACTIVATE_WHEN_SHOWN','LEGO_NATIVE_TIMING','LEGO_NATIVE_ALL
     'LEGO_GPU_SNAPSHOT_DIR','LEGO_GPU_SNAPSHOT_START_AT_TILING',
     'LEGO_GPU_SNAPSHOT_TIME_WINDOWS','LEGO_DUMP_MISSING_SHADERS',
     'LEGO_NATIVE_GPU_DEBUG','LEGO_XENOS_TILING_TRACE','LEGO_DUMP_TEXTURE_UPLOADS','LEGO_NATIVE_TRACE_DIR',
-    'LEGO_NATIVE_NO_MEMORY_WATCH','LEGO_NATIVE_STATIC_TEXTURE_WATCH','LEGO_NATIVE_AUDIT_TEXTURE_WATCH','LEGO_NATIVE_NO_DRAW_ARENA',
+    'LEGO_NATIVE_NO_MEMORY_WATCH','LEGO_NATIVE_STATIC_TEXTURE_WATCH','LEGO_NATIVE_AUDIT_TEXTURE_WATCH','LEGO_NATIVE_BUFFER_WATCH','LEGO_NATIVE_AUDIT_BUFFER_WATCH','LEGO_NATIVE_NO_DRAW_ARENA',
     'LEGO_NATIVE_FRAME_METRICS','LEGO_NATIVE_ASYNC_CPU_RESOURCES','LEGO_NATIVE_STENCIL','LEGO_NATIVE_MESH_TRACE',
     'LEGO_NATIVE_VIEWPORT','LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY','LEGO_NATIVE_BUFFER_WINDOWS','LEGO_NATIVE_PM4_REFERENCE',
     'LEGO_NATIVE_RENDERDOC_DLL','LEGO_NATIVE_RENDERDOC_CAPTURE','LEGO_NATIVE_DEPTH_ALIAS')
@@ -66,12 +68,14 @@ try {
     if (!$ConfiguredRenderer) {
         $env:LEGO_NATIVE_STATIC_TEXTURE_WATCH = $(if ($StaticTextureWatch -or $AuditTextureWatch) {'1'} else {'0'})
         $env:LEGO_NATIVE_BUFFER_WINDOWS = $(if ($BufferWindows) {'1'} else {'0'})
+        $env:LEGO_NATIVE_BUFFER_WATCH = $(if ($BufferWatch -or $AuditBufferWatch) {'1'} else {'0'})
     }
     if ($StaticTextureWatch) { $env:LEGO_NATIVE_STATIC_TEXTURE_WATCH = '1' }
     if ($AuditTextureWatch) {
         $env:LEGO_NATIVE_STATIC_TEXTURE_WATCH = '1'
         $env:LEGO_NATIVE_AUDIT_TEXTURE_WATCH = '1'
     }
+    if ($AuditBufferWatch) { $env:LEGO_NATIVE_AUDIT_BUFFER_WATCH = '1' }
     if ($NoDrawArena) { $env:LEGO_NATIVE_NO_DRAW_ARENA = '1' }
     if ($AsyncCpuResources) { $env:LEGO_NATIVE_ASYNC_CPU_RESOURCES = '1' }
     if ($Stencil) { $env:LEGO_NATIVE_STENCIL = '1' }
@@ -127,6 +131,7 @@ try {
         mode=$Mode;allTextures=[bool]$AllTextures;captureMissing=[bool]$CaptureMissing;monitor=$Monitor;
         longProbe=[bool]$LongProbe;noMemoryWatch=[bool]$NoMemoryWatch;staticTextureWatch=[bool]$StaticTextureWatch;
         auditTextureWatch=[bool]$AuditTextureWatch;
+        bufferWatch=[bool]$BufferWatch; auditBufferWatch=[bool]$AuditBufferWatch;
         noDrawArena=[bool]$NoDrawArena;asyncCpuResources=[bool]$AsyncCpuResources;
         stencil=[bool]$Stencil;meshTrace=[bool]$MeshTrace;
         logoUploads=[bool]$LogoUploads;viewport=[bool]$Viewport;bufferWindows=[bool]$BufferWindows;
diff --git a/tools/gpu-aot/test_buffer_variants.py b/tools/gpu-aot/test_buffer_variants.py
index 5ca4a8c..3db43bf 100644
--- a/tools/gpu-aot/test_buffer_variants.py
+++ b/tools/gpu-aot/test_buffer_variants.py
@@ -1,6 +1,7 @@
 """Check fetched-byte equivalence and actual buffer conversion cache behavior."""
 import argparse
 import json
+import os
 from pathlib import Path
 import subprocess
 p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
@@ -19,6 +20,7 @@ def body(signature):
 #include <atomic>
 #include <chrono>
 #include <cstring>
+#include <cstdlib>
 #include <memory>
 #include <mutex>
 #include <random>
@@ -66,8 +68,22 @@ def body(signature):
 Memory memory;
 #define REX_KERNEL_MEMORY() (&memory)
 bool NativeTextureTimingEnabled(){return true;}
-struct BufferUploadTiming {u64 calls=0,hashed_bytes=0,converted_bytes=0;double hash_ms=0;};
+struct BufferUploadTiming {u64 calls=0,hashed_bytes=0,converted_bytes=0,watch_hits=0,watch_audits=0,watch_mismatches=0;double hash_ms=0;};
 BufferUploadTiming g_buffer_timing;
+bool watch_enabled=false,watchable=true;
+std::array<u64,16> versions{};
+bool legodimensions::gpu_native::CpuBufferMemoryWatchEnabled(){return watch_enabled;}
+CpuMemoryStamp legodimensions::gpu_native::WatchCpuMemory(std::span<const CpuMemorySpan> spans){
+ CpuMemoryStamp stamp;if(!watch_enabled||!watchable)return stamp;
+ for(auto span:spans)for(u32 page=span.address/4096;page<=(span.address+span.length-1)/4096;++page)
+  stamp.pages.emplace_back(page,versions.at(page));
+ return stamp;
+}
+bool legodimensions::gpu_native::CpuMemoryUnchanged(const CpuMemoryStamp& stamp){
+ if(stamp.pages.empty())return false;
+ for(auto [page,version]:stamp.pages)if(versions.at(page)!=version)return false;
+ return true;
+}
 '''+resource+r'''
 std::shared_ptr<BufferResource> resource;
 auto AdoptBuffer(u32,BufferKind)->std::shared_ptr<BufferResource>{return resource;}
@@ -197,18 +213,61 @@ def body(signature):
  assert(g_buffer_timing.hashed_bytes-hashed_before==256*100);
  assert(!ResolveBufferResourceWindow(1,BufferKind::kVertex,129,256,{}).buffer);
  assert(!ResolveBufferResourceWindow(1,BufferKind::kVertex,32760,16,{}).buffer);
+ // Buffer watches retain every-use invalidation and immutable conversions.
+ // Unknown virtual storage must hash even when the candidate is selected.
+ watch_enabled=true;
+ auto warm=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+ assert(!resource->windows.at((u64(128)<<32)|256)->cpu_stamp.pages.empty());
+ const bool audit=std::getenv("LEGO_NATIVE_AUDIT_BUFFER_WATCH")!=nullptr;
+ const auto watch_hash_before=g_buffer_timing.hashed_bytes;
+ const auto allocations_before=HostDevice::device.allocations;
+ for(u32 i=0;i<100;++i)assert(ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{}).buffer==warm.buffer);
+ assert(g_buffer_timing.hashed_bytes-watch_hash_before==(audit?256*100:0));
+ assert(HostDevice::device.allocations==allocations_before);
+ assert(audit?g_buffer_timing.watch_audits>=100:g_buffer_timing.watch_hits>=100);
+ memory.bytes[16000+128+7]^=0x80;++versions[(16000+128+7)/4096];
+ auto dirty=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+ assert(dirty.buffer!=warm.buffer&&dirty.buffer->data!=warm.buffer->data);
+ const auto dirty_bytes=dirty.buffer->data;
+ // Another same-frame physical/alias write re-invalidates a rearmed stamp.
+ memory.bytes[16000+128+8]^=0x40;++versions[(16000+128+8)/4096];
+ auto dirty_again=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+ assert(dirty_again.buffer!=dirty.buffer&&dirty.buffer->data==dirty_bytes);
+ if(audit){
+  // Simulate a writer missing notification: audit must catch and repair it.
+  memory.bytes[16000+128+9]^=0x20;
+  auto repaired=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+  assert(repaired.buffer!=dirty_again.buffer&&g_buffer_timing.watch_mismatches==1);
+ }
+ watchable=false;resource->windows.clear();resource->window_cache_bytes=0;
+ auto unknown=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+ auto unknown_hash_before=g_buffer_timing.hashed_bytes;
+ for(u32 i=0;i<100;++i)assert(ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{}).buffer==unknown.buffer);
+ assert(g_buffer_timing.hashed_bytes-unknown_hash_before==256*100);
+ // Global shutdown invalidates all otherwise-clean stamps.
+ watchable=true;resource->windows.clear();resource->window_cache_bytes=0;
+ auto before_reset=ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{});
+ for(auto& version:versions)++version;
+ auto reset_hash_before=g_buffer_timing.hashed_bytes;
+ assert(ResolveBufferResourceWindow(1,BufferKind::kVertex,128,256,{}).buffer==before_reset.buffer);
+ assert(g_buffer_timing.hashed_bytes-reset_hash_before==256);
 }
 ''')
 exe=a.output/'buffer-variants-test.exe'
 subprocess.run(['clang++','-std=c++20','-DNOMINMAX','-I'+str(root/'rexlego/src'),
  '-I'+str(root/'rexglue-sdk/thirdparty/xxHash'),str(h),'-o',str(exe)],check=True)
-subprocess.run([str(exe.resolve())],check=True)
+env=dict(os.environ);env.pop('LEGO_NATIVE_AUDIT_BUFFER_WATCH',None)
+subprocess.run([str(exe.resolve())],env=env,check=True)
+env['LEGO_NATIVE_AUDIT_BUFFER_WATCH']='1'
+subprocess.run([str(exe.resolve())],env=env,check=True)
 (a.output/'verification.json').write_text(json.dumps({'actual_buffer_resolve_body':True,'fake_driver':True,
  'canonical_equivalence_cases':10000,'checks':['same-phase reuse','alternating declarations','same-frame CPU update',
  'immutable old versions','8-version bound','invalid offset','index endian','every-use content hashing',
  'actual placement adoption','32 alternating owner-header frames','same-header relocation',
  'same-frame rewritten placement data','all recorded placement versions preserved',
  '1000 window rebasing/conversion cases','byte/entry cache bound','256-range working set reused without allocations','immutable rewritten windows',
- 'every-use hashing restricted to requested window','invalid window/overflow rejection'],
+ 'every-use hashing restricted to requested window','invalid window/overflow rejection',
+ 'opt-in generation reuse eliminates 100 unchanged hashes','same-frame watched rewrites',
+ 'audit detects and repairs false clean stamp','unknown memory hashes every use','shutdown generation invalidation'],
  'actual_buffer_adoption_body':True,'passed':True},indent=2))
 print('Passed 10000 fetched-byte equivalence cases and actual conversion-cache body, including same-frame writes and old-version lifetime.')
diff --git a/tools/gpu-aot/test_memory_watch.py b/tools/gpu-aot/test_memory_watch.py
index a7d1ee4..82289b1 100644
--- a/tools/gpu-aot/test_memory_watch.py
+++ b/tools/gpu-aot/test_memory_watch.py
@@ -14,7 +14,7 @@
 enum class Lifecycle { kRequiresRestart };
 struct FixtureCvar { FixtureCvar lifecycle(Lifecycle) { return *this; } };
 }
-#define REXCVAR_DEFINE_BOOL(...) static auto fixture_cvar = rex::cvar::FixtureCvar{}
+#define REXCVAR_DEFINE_BOOL(name,...) static auto fixture_##name = rex::cvar::FixtureCvar{}
 #define REXCVAR_GET(...) false
 ''')
 (stub/'xmemory.h').write_text(r'''
@@ -122,6 +122,10 @@
 subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True)
 env.pop('LEGO_NATIVE_STATIC_TEXTURE_WATCH')
 subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True)
+env['LEGO_NATIVE_BUFFER_WATCH']='1';env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='0'
+subprocess.run([str(exe.resolve())],env=env,check=True)
+env['LEGO_NATIVE_NO_MEMORY_WATCH']='1'
+subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True)
 (a.output/'verification.json').write_text(json.dumps({'production_source':True,'sdk_model':True,'checks':[
  'A/C/E aliases and +4KB','same-frame writes','neighbor watch retained','base+mips','physical host write',
  'write between arm and reuse','unknown/uncommitted/overflow fallback','device recreation','disabled fallback'],
diff --git a/tools/gpu-aot/test_memory_watch_sdk.py b/tools/gpu-aot/test_memory_watch_sdk.py
index 7205f35..d997ac7 100644
--- a/tools/gpu-aot/test_memory_watch_sdk.py
+++ b/tools/gpu-aot/test_memory_watch_sdk.py
@@ -23,9 +23,11 @@
 rex::memory::Memory* tested_memory=nullptr;
 using namespace legodimensions::gpu_native;
 REXCVAR_DECLARE(bool, gpu_native_texture_watch);
+REXCVAR_DECLARE(bool, gpu_native_buffer_watch);
 int main(int argc,char**){
  rex::InitLogging();
- if(argc>1) REXCVAR_SET(gpu_native_texture_watch,true);
+ if(argc==2) REXCVAR_SET(gpu_native_texture_watch,true);
+ if(argc==3) REXCVAR_SET(gpu_native_buffer_watch,true);
  rex::memory::Memory memory;tested_memory=&memory;
  assert(memory.Initialize());
  auto* heap=memory.LookupHeapByType(true,4096);
@@ -68,3 +70,7 @@
 subprocess.run([str(exe)],cwd=a.output,env=env,check=True,timeout=30)
 env.pop('LEGO_NATIVE_STATIC_TEXTURE_WATCH')
 subprocess.run([str(exe),'configured'],cwd=a.output,env=env,check=True,timeout=30)
+env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='0';env['LEGO_NATIVE_BUFFER_WATCH']='1'
+subprocess.run([str(exe)],cwd=a.output,env=env,check=True,timeout=30)
+env.pop('LEGO_NATIVE_BUFFER_WATCH')
+subprocess.run([str(exe),'buffer','configured'],cwd=a.output,env=env,check=True,timeout=30)

From 72ca532d7dbf718ec814c9c2bf10d79ed63f94c4 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 21:56:24 +0300
Subject: [PATCH 25/44] Record consecutive-frame evidence of the Enter color
 swap

---
 tools/gpu-aot/native-transition-findings-20261003.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tools/gpu-aot/native-transition-findings-20261003.md b/tools/gpu-aot/native-transition-findings-20261003.md
index aaa77b3..cfe9f2a 100644
--- a/tools/gpu-aot/native-transition-findings-20261003.md
+++ b/tools/gpu-aot/native-transition-findings-20261003.md
@@ -23,6 +23,14 @@ regular rendering does not enable these readbacks. The diagnostic probe
 `oct03-enter-burst` captured240 frames and was closed. Heavy captures are not
 valid performance measurements.
 
+Offline examination of the240 final DDS frames confirms the corruption:
+snapshot-relative frames1557..1560 have a blue tile (1049 sampled blue pixels,
+zero red); the surrounding title frames have a red tile and zero sampled blue.
+Saved image `session-20261003/enter-gpu-f1557.png` reproduces the user's yellow
+lights and blue tile; `enter-gpu-f1563.png` shows normal cyan/red again. The
+snapshot anchor is absolute frame777, so frame1557 corresponds to the trace's
+frame2334 swizzle0xA0A event. These are pre-fix captures, not proof of the fix.
+
 The earlier aspect-ratio correction was confirmed in normal title window
 captures: the portrait monitor window now has black letterbox bars. This does
 not establish the remaining HUD portrait or general gameplay correctness.

From 84be59f4db17fa39b4011aa0ffc98e8e9ba0cbbb Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 22:07:18 +0300
Subject: [PATCH 26/44] Fuse aligned packed-vertex conversion into write-only
 uploads

---
 rexlego/src/gpu_native/buffers.cpp            | 15 +++--
 rexlego/src/gpu_native/vertex_upload.h        | 64 +++++++++++++++++++
 tools/gpu-aot/benchmark_vertex_upload.cpp     | 51 +++++++++++++++
 .../gpu-aot/native-vertex-upload-20261003.md  | 43 +++++++++++++
 tools/gpu-aot/test_buffer_variants.py         |  3 +-
 tools/gpu-aot/test_vertex_upload.cpp          | 42 ++++++++++++
 6 files changed, 212 insertions(+), 6 deletions(-)
 create mode 100644 rexlego/src/gpu_native/vertex_upload.h
 create mode 100644 tools/gpu-aot/benchmark_vertex_upload.cpp
 create mode 100644 tools/gpu-aot/native-vertex-upload-20261003.md
 create mode 100644 tools/gpu-aot/test_vertex_upload.cpp

diff --git a/rexlego/src/gpu_native/buffers.cpp b/rexlego/src/gpu_native/buffers.cpp
index d3d1efc..2b503fb 100644
--- a/rexlego/src/gpu_native/buffers.cpp
+++ b/rexlego/src/gpu_native/buffers.cpp
@@ -1,6 +1,7 @@
 #include "gpu_native/buffers.h"
 #include "gpu_native/buffer_header.h"
 #include "gpu_native/memory_watch.h"
+#include "gpu_native/vertex_upload.h"
 
 #include <cstring>
 #include <cstdlib>
@@ -297,13 +298,17 @@ plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr<BufferResource>
   // then perform a sequential write-only copy into the new GPU version.
   std::vector<u8> converted;
   void* conversion_target = mapped;
-  if (kind == BufferKind::kVertex && !byte_order.reversed_elements.empty()) {
-    converted.resize(resource->length);
-    conversion_target = converted.data();
+  const bool fused_upload = kind == BufferKind::kVertex &&
+      WriteAlignedVertexUpload(static_cast<u8*>(mapped), source, resource->length, conversion_order);
+  if (!fused_upload) {
+    if (kind == BufferKind::kVertex && !byte_order.reversed_elements.empty()) {
+      converted.resize(resource->length);
+      conversion_target = converted.data();
+    }
+    ByteSwapElements(conversion_target, source, resource->length, element_size);
   }
-  ByteSwapElements(conversion_target, source, resource->length, element_size);
   if (timing) g_buffer_timing.converted_bytes += resource->length;
-  if (kind == BufferKind::kVertex &&
+  if (!fused_upload && kind == BufferKind::kVertex &&
       !ApplyVertexByteOrder(static_cast<u8*>(conversion_target), resource->length, conversion_order)) {
     next_buffer->unmap();
     REXLOG_WARN("Native GPU: rejected invalid packed vertex layout stride={}",
diff --git a/rexlego/src/gpu_native/vertex_upload.h b/rexlego/src/gpu_native/vertex_upload.h
new file mode 100644
index 0000000..44f325d
--- /dev/null
+++ b/rexlego/src/gpu_native/vertex_upload.h
@@ -0,0 +1,64 @@
+#pragma once
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <numeric>
+#include "gpu_native/vertex_byte_order.h"
+#if defined(__SSSE3__)
+#include <tmmintrin.h>
+#endif
+
+namespace legodimensions::gpu_native {
+// Fuse DWORD endian conversion and the packed-field reversal into one
+// sequential, write-only upload. Unsupported layouts retain the old path.
+inline bool WriteAlignedVertexUpload(uint8_t* destination, const uint8_t* source,
+                                    size_t size, const VertexByteOrder& order) {
+  if (order.reversed_elements.empty() || order.stride < 4 || order.stride > 256 ||
+      order.stride % 4 || order.stream_offset % 4 || order.stream_offset > size)
+    return false;
+  uint32_t previous = 0;
+  bool first = true;
+  for (uint32_t field : order.reversed_elements) {
+    if (field % 4 || field > order.stride - 4 || (!first && field <= previous)) return false;
+    previous = field; first = false;
+  }
+  const auto packed = [&](size_t offset) {
+    if (offset < order.stream_offset) return false;
+    const auto field = uint32_t((offset - order.stream_offset) % order.stride);
+    return std::binary_search(order.reversed_elements.begin(), order.reversed_elements.end(), field);
+  };
+  size_t offset = 0;
+#if defined(__SSSE3__)
+  // The stride and sixteen-byte SIMD chunks repeat after their LCM. Bound
+  // this small table to64 chunks; every shuffle stays inside its own DWORD.
+  const size_t blocks = order.stride / std::gcd(order.stride, 16u);
+  alignas(16) std::array<std::array<uint8_t, 16>, 64> masks{};
+  const size_t first_vector = (size_t(order.stream_offset) + 15) / 16 * 16;
+  for (size_t b = 0; b < blocks; ++b)
+    for (size_t lane = 0; lane < 16; lane += 4) {
+      const bool identity = packed(first_vector + b * 16 + lane);
+      for (size_t byte = 0; byte < 4; ++byte)
+        masks[b][lane + byte] = uint8_t(lane + (identity ? byte : 3 - byte));
+    }
+  // A non-canonical caller may leave a long unconverted prefix. Handle it
+  // before the repeating phase, without applying packed fields too early.
+  for (; offset < std::min(first_vector, size / 4 * 4); offset += 4) {
+    if (packed(offset)) std::memcpy(destination + offset, source + offset, 4);
+    else for (size_t byte = 0; byte < 4; ++byte) destination[offset + byte] = source[offset + 3 - byte];
+  }
+  size_t block = 0;
+  for (; size - offset >= 16; offset += 16) {
+    const auto value = _mm_loadu_si128(reinterpret_cast<const __m128i*>(source + offset));
+    const auto mask = _mm_load_si128(reinterpret_cast<const __m128i*>(masks[block].data()));
+    _mm_storeu_si128(reinterpret_cast<__m128i*>(destination + offset), _mm_shuffle_epi8(value, mask));
+    if (++block == blocks) block = 0;
+  }
+#endif
+  for (; size - offset >= 4; offset += 4) {
+    if (packed(offset)) std::memcpy(destination + offset, source + offset, 4);
+    else for (size_t byte = 0; byte < 4; ++byte) destination[offset + byte] = source[offset + 3 - byte];
+  }
+  if (offset != size) std::memcpy(destination + offset, source + offset, size - offset);
+  return true;
+}
+} // namespace legodimensions::gpu_native
diff --git a/tools/gpu-aot/benchmark_vertex_upload.cpp b/tools/gpu-aot/benchmark_vertex_upload.cpp
new file mode 100644
index 0000000..93db536
--- /dev/null
+++ b/tools/gpu-aot/benchmark_vertex_upload.cpp
@@ -0,0 +1,51 @@
+#include <d3d12.h>
+#include <dxgi1_6.h>
+#include <wrl/client.h>
+#include <chrono>
+#include <iostream>
+#include <vector>
+#include <cassert>
+#include "gpu_native/vertex_upload.h"
+using Microsoft::WRL::ComPtr;
+using namespace legodimensions::gpu_native;
+int main() {
+  ComPtr<IDXGIFactory4> factory; assert(SUCCEEDED(CreateDXGIFactory1(IID_PPV_ARGS(&factory))));
+  ComPtr<IDXGIAdapter1> adapter; assert(SUCCEEDED(factory->EnumAdapters1(0, &adapter)));
+  DXGI_ADAPTER_DESC1 desc; adapter->GetDesc1(&desc); std::wcout << desc.Description << L"\n";
+  ComPtr<ID3D12Device> device; assert(SUCCEEDED(D3D12CreateDevice(adapter.Get(),D3D_FEATURE_LEVEL_11_0,IID_PPV_ARGS(&device))));
+  constexpr size_t bytes = 6 * 1024 * 1024;
+  D3D12_HEAP_PROPERTIES heap{};heap.Type=D3D12_HEAP_TYPE_UPLOAD;
+  D3D12_RESOURCE_DESC resource{};resource.Dimension=D3D12_RESOURCE_DIMENSION_BUFFER;
+  resource.Width=bytes;resource.Height=1;resource.DepthOrArraySize=1;resource.MipLevels=1;
+  resource.SampleDesc.Count=1;resource.Layout=D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
+  ComPtr<ID3D12Resource> buffer;
+  assert(SUCCEEDED(device->CreateCommittedResource(&heap,D3D12_HEAP_FLAG_NONE,&resource,D3D12_RESOURCE_STATE_GENERIC_READ,nullptr,IID_PPV_ARGS(&buffer))));
+  void* raw=nullptr;D3D12_RANGE no_read{0,0};assert(SUCCEEDED(buffer->Map(0,&no_read,&raw)));
+  auto* mapped=static_cast<uint8_t*>(raw);
+  std::vector<uint8_t> src(bytes), scratch(bytes);
+  for(size_t i=0;i<bytes;++i)src[i]=uint8_t(i*73+(i>>8));
+  const std::array<uint32_t,4> fields{8,12,16,20};const VertexByteOrder order{24,0,fields};
+  auto old=[&]{
+    std::vector<uint8_t> converted(bytes);
+    for(size_t i=0;i<bytes;i+=4)for(size_t j=0;j<4;++j)converted[i+j]=src[i+3-j];
+    assert(ApplyVertexByteOrder(converted.data(),bytes,order));
+    std::memcpy(mapped,converted.data(),bytes);
+  };
+  old();std::memcpy(scratch.data(),mapped,bytes);
+  assert(WriteAlignedVertexUpload(mapped,src.data(),bytes,order));
+  assert(std::memcmp(mapped,scratch.data(),bytes)==0);
+  auto time=[&](auto fn){
+    auto start=std::chrono::steady_clock::now();
+    for(unsigned i=0;i<40;++i)fn();
+    return std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count()/40;
+  };
+  // Alternate the order over three trials; no window, present, GPU readback,
+  // game process or user input. This measures only CPU writes/conversion.
+  for(unsigned trial=0;trial<3;++trial){
+    double legacy,fused;
+    if(trial%2){fused=time([&]{assert(WriteAlignedVertexUpload(mapped,src.data(),bytes,order));});legacy=time(old);}
+    else{legacy=time(old);fused=time([&]{assert(WriteAlignedVertexUpload(mapped,src.data(),bytes,order));});}
+    std::cout << "trial="<<trial<<" bytes="<<bytes<<" legacy_ms="<<legacy<<" fused_ms="<<fused<<"\n";
+  }
+  buffer->Unmap(0,nullptr);
+}
diff --git a/tools/gpu-aot/native-vertex-upload-20261003.md b/tools/gpu-aot/native-vertex-upload-20261003.md
new file mode 100644
index 0000000..25a10eb
--- /dev/null
+++ b/tools/gpu-aot/native-vertex-upload-20261003.md
@@ -0,0 +1,43 @@
+# Fused packed-vertex upload, October 3
+
+The old packed-vertex path allocates and zeroes a temporary CPU buffer, swaps
+all DWORDs, reverses UBYTE4 fields, then copies the entire result into the D3D12
+UPLOAD mapping. These reads/writes are expensive even though the final mapped
+copy is write-only. The measured title vertex cost was10.84ms/frame on October1.
+
+`WriteAlignedVertexUpload` combines the two permutations into a single
+sequential write from guest bytes to the upload mapping. On the game's x86 ISA,
+SSSE3 shuffles use a bounded repeating mask table. The helper reads no mapped
+destination bytes and allocates no temporary full buffer. It accepts aligned,
+sorted, distinct packed fields in strides4..256; unusual declarations retain
+the original conversion. Upload versioning, hashes, fences and guest RAM are
+unchanged. This exact-byte optimization is active without selecting watches.
+
+Verification:
+
+- `test_vertex_upload.cpp`:30000 differential cases against the old two-step
+  conversion, varied strides/phases/fields, misaligned source/destination
+  pointers, partial records and tails, guarded output bounds, and untouched
+  output on rejected layouts. SSSE3 build passed.
+- Actual buffer resolve/adoption/window body tests pass with the new helper:
+  immutable old uploads, same-frame edits,1000 rebasings, buffer watches and
+  audits. Native Release build passed.
+- `benchmark_vertex_upload.cpp` creates a D3D12 device and committed UPLOAD
+  buffer on RTX4060, with no game, window, swap chain or input. One6MiB synthetic
+  stride24 layout with packed offsets8/12/16/20 is converted identically by both
+  paths and compared byte-for-byte on the real mapping. Each trial performs40
+  writes; trial order alternates. Clang20.1.8,-O2,-march=x86-64-v3:
+  old7.67321/7.59433/8.03351ms, fused0.624520/0.576703/0.585677ms.
+  Raw log: `session-20261003/vertex-upload-benchmark-v3.log`.
+
+Reproduce after loading `E:\devtools\env.ps1`:
+
+```
+clang++ -std=c++20 -O2 -mssse3 -Irexlego/src tools/gpu-aot/test_vertex_upload.cpp -o <test.exe>
+clang++ -std=c++20 -O2 -march=x86-64-v3 -DNOMINMAX -Irexlego/src tools/gpu-aot/benchmark_vertex_upload.cpp -ld3d12 -ldxgi -o <benchmark.exe>
+```
+
+These are CPU conversion measurements on a synthetic layout, not gameplay
+FPS, frame-time or visual correctness measurements. No post-change game launch
+was performed because the user prohibited it. A clean comparable hub/level run
+must establish the actual benefit and check the full scene once authorized.
diff --git a/tools/gpu-aot/test_buffer_variants.py b/tools/gpu-aot/test_buffer_variants.py
index 3db43bf..c735a1f 100644
--- a/tools/gpu-aot/test_buffer_variants.py
+++ b/tools/gpu-aot/test_buffer_variants.py
@@ -30,6 +30,7 @@ def body(signature):
 #define XXH_INLINE_ALL
 #include <xxhash.h>
 #include "gpu_native/vertex_byte_order.h"
+#include "gpu_native/vertex_upload.h"
 #include "gpu_native/memory_watch.h"
 #include "gpu_native/buffer_header.h"
 #include "gpu_native/buffer_window.h"
@@ -254,7 +255,7 @@ def body(signature):
 }
 ''')
 exe=a.output/'buffer-variants-test.exe'
-subprocess.run(['clang++','-std=c++20','-DNOMINMAX','-I'+str(root/'rexlego/src'),
+subprocess.run(['clang++','-std=c++20','-O2','-mssse3','-DNOMINMAX','-I'+str(root/'rexlego/src'),
  '-I'+str(root/'rexglue-sdk/thirdparty/xxHash'),str(h),'-o',str(exe)],check=True)
 env=dict(os.environ);env.pop('LEGO_NATIVE_AUDIT_BUFFER_WATCH',None)
 subprocess.run([str(exe.resolve())],env=env,check=True)
diff --git a/tools/gpu-aot/test_vertex_upload.cpp b/tools/gpu-aot/test_vertex_upload.cpp
new file mode 100644
index 0000000..7eabca4
--- /dev/null
+++ b/tools/gpu-aot/test_vertex_upload.cpp
@@ -0,0 +1,42 @@
+#include <cassert>
+#include <iostream>
+#include <random>
+#include <vector>
+#include "gpu_native/vertex_upload.h"
+using namespace legodimensions::gpu_native;
+void Reference(std::vector<uint8_t>& dst, const std::vector<uint8_t>& src,
+               const VertexByteOrder& order) {
+  size_t offset = 0;
+  for (; offset + 4 <= src.size(); offset += 4)
+    for (size_t byte = 0; byte < 4; ++byte) dst[offset + byte] = src[offset + 3 - byte];
+  for (; offset < src.size(); ++offset) dst[offset] = src[offset];
+  assert(ApplyVertexByteOrder(dst.data(), dst.size(), order));
+}
+int main() {
+  std::mt19937 rng(337);
+  for (unsigned n = 0; n < 30000; ++n) {
+    const unsigned stride = 4 * (1 + rng() % 64), offset = 4 * (rng() % 128);
+    const unsigned length = offset + rng() % 4096;
+    std::vector<uint32_t> fields;
+    for (unsigned word = 0; word < stride / 4; ++word) if (rng() % 4 == 0) fields.push_back(word * 4);
+    if (fields.empty()) fields.push_back(0);
+    const VertexByteOrder order{stride, offset, fields};
+    std::vector<uint8_t> src(length + 32), expected(length + 32, 0xCC), actual(length + 32, 0xCC);
+    for (auto& byte : src) byte = uint8_t(rng());
+    std::vector<uint8_t> bounded(src.begin() + 7, src.begin() + 7 + length), reference(length);
+    Reference(reference, bounded, order);
+    std::copy(reference.begin(), reference.end(), expected.begin() + 3);
+    assert(WriteAlignedVertexUpload(actual.data() + 3, src.data() + 7, length, order));
+    assert(actual == expected); // Exact bytes and no writes beyond the window.
+  }
+  std::vector<uint8_t> source(1024, 73), output(1024, 0xCC);
+  const std::vector<uint32_t> duplicate{0, 0}, unordered{4, 0}, unaligned{1}, outside{16}, good{0, 4};
+  for (auto fields : {duplicate, unordered, unaligned, outside})
+    assert(!WriteAlignedVertexUpload(output.data(), source.data(), source.size(), {16, 0, fields}));
+  assert(!WriteAlignedVertexUpload(output.data(), source.data(), source.size(), {18, 0, good}));
+  assert(!WriteAlignedVertexUpload(output.data(), source.data(), source.size(), {16, 2, good}));
+  assert(!WriteAlignedVertexUpload(output.data(), source.data(), source.size(), {16, 1028, good}));
+  assert(!WriteAlignedVertexUpload(output.data(), source.data(), source.size(), {260, 0, good}));
+  assert(std::all_of(output.begin(), output.end(), [](auto b) { return b == 0xCC; }));
+  std::cout << "PASS 30000 differential vertex uploads, misaligned pointers, partial records/tails, bounded writes and untouched fallback\n";
+}

From a0bb17645805865f8fcfe60ae9f1812cbcc5aacc Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 22:09:04 +0300
Subject: [PATCH 27/44] Document portrait reference anchors and native upload
 measurements

---
 .../avengers-reference-findings-20261003.md   | 42 +++++++++++++++++++
 .../gpu-aot/native-vertex-upload-20261003.md  |  3 ++
 2 files changed, 45 insertions(+)
 create mode 100644 tools/gpu-aot/avengers-reference-findings-20261003.md

diff --git a/tools/gpu-aot/avengers-reference-findings-20261003.md b/tools/gpu-aot/avengers-reference-findings-20261003.md
new file mode 100644
index 0000000..3b535ff
--- /dev/null
+++ b/tools/gpu-aot/avengers-reference-findings-20261003.md
@@ -0,0 +1,42 @@
+# Avengers reference follow-up, October 3
+
+Read `research/avengers/REPORT.md`, `avgref.py`, `ids.py` and `ids2.py`.
+Used the installed PC executable and extracted shader index read-only. No PC
+game was launched. Reference bytecode/disassembly remains in ignored local
+`session-20261003/avengers-reference`; none is embedded or committed.
+
+The remaining native portrait is empty for both Sonic and Batman, while the
+emulated slot2 Vorton reference has Sonic's portrait. Binding all texture slots
+did not repair it. These captures establish a shared HUD defect, not its cause.
+
+Concrete engine anchors from `avgref.py xref portrait` and local x86 decoding:
+
+- PC `portrait_mask_texture` string0x0192101C is used in0x00BD36A0 and
+  0x00C68A10. `m_portraitMaskMtlOverride` string0x0193D57C is used in0x00C68A10.
+  That setup loads a portrait mask property, tests availability, constructs an
+  override material and updates its texture/material flags. This is setup code,
+  not an identified missing native draw.
+- Dimensions' same property is string0x8224CBF8, referenced in0x83293E68
+  (property registration) and0x833738B8 (setup). Its override string0x8227F2F0
+  is used in0x833738B8. Setup stores the loaded object at owner+84; after the
+  property lookup succeeds, it creates override storage at owner+200 and passes
+  the mask's texture to material+956. Those exact pointers can connect future
+  HUD draw/texture captures to the CPU object rather than guessing from color.
+- `83373B00` handles event119 and rounds requested dimensions using scene+320,
+  with several character/owner prerequisites. It is not proven to issue the
+  portrait draw. Nearby `83373850` is a destructor; address proximity does not
+  establish a render entry point.
+
+Queried `levels\icons\hud_icons\hud_icons`: several shaders have no PC family
+match. VS20 has a same-asset PC match atVS0; its reflected constants are camera,
+instance and material with position/UV inputs. It applies world/view projection
+and UV transforms. However2500 PC shaders share that reduced name signature.
+This is a material-family reference, not proof of bytecode equivalence or that
+this shader is the missing portrait pass. Other HUD pixel-family matches chose
+different assets, so they were not treated as exact semantic ground truth.
+
+No portrait fix is claimed. Next required evidence is the actual override
+material's bound shader/texture and its sampled alpha/UV at a confirmed HUD
+draw, compared with the reference. Generic bank175 coverage no longer reports
+unavailable pipelines in the captured candidate hub, so more guessed shaders
+or texture-slot overrides are not justified by that evidence.
diff --git a/tools/gpu-aot/native-vertex-upload-20261003.md b/tools/gpu-aot/native-vertex-upload-20261003.md
index 25a10eb..9e8b5e1 100644
--- a/tools/gpu-aot/native-vertex-upload-20261003.md
+++ b/tools/gpu-aot/native-vertex-upload-20261003.md
@@ -29,6 +29,9 @@ Verification:
   writes; trial order alternates. Clang20.1.8,-O2,-march=x86-64-v3:
   old7.67321/7.59433/8.03351ms, fused0.624520/0.576703/0.585677ms.
   Raw log: `session-20261003/vertex-upload-benchmark-v3.log`.
+  Repeated at the actual native build's-O3 ISA flags: old6.56941/6.61659/
+  6.52649ms, fused0.548512/0.405378/0.413095ms; raw log
+  `session-20261003/vertex-upload-benchmark-native.log`.
 
 Reproduce after loading `E:\devtools\env.ps1`:
 

From 395d369cc635506a38092ca24a4b1f0c25ef4c07 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sat, 3 Oct 2026 22:10:51 +0300
Subject: [PATCH 28/44] Record remaining renderer validation and the no-launch
 constraint

---
 tools/gpu-aot/README.md                       | 12 +++--
 .../native-validation-pending-20261003.md     | 44 +++++++++++++++++++
 2 files changed, 53 insertions(+), 3 deletions(-)
 create mode 100644 tools/gpu-aot/native-validation-pending-20261003.md

diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index 5821ca6..60eefdf 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -319,9 +319,15 @@ this short run. The character portrait remains a blank blue circle. This is
 a partial improvement, not completed hub correctness or performance validation.
 Both native probes were closed; further probes use monitor 2 per the user's
 October 3 instruction. Do not infer a performance result from snapshot/trace
-runs. Emulated comparison and further character/portrait investigation remain.
-
-The production runtime bank now contains 118 validated containers, 18722
+runs. The subsequent emulated Vorton comparison shows Sonic's portrait; native
+Sonic and Batman both have the shared empty portrait. After the Enter burst
+test the user prohibited further game launches until explicitly authorized.
+Offline follow-ups: `native-transition-findings-20261003.md`,
+`native-buffer-watch-20261003.md`, `native-vertex-upload-20261003.md`, and
+`avengers-reference-findings-20261003.md`. The fused vertex optimization is
+built; buffer watches remain opt-in. No post-change gameplay FPS claim.
+
+The September30 production runtime bank contained 118 validated containers, 18722
 microcode signatures and 192 linked runtime variants. It retains the current
 624-byte constants ABI. The six late containers had valid nonzero instruction
 streams despite leading zero words. `merge_microcode_index.py` preserves the
diff --git a/tools/gpu-aot/native-validation-pending-20261003.md b/tools/gpu-aot/native-validation-pending-20261003.md
new file mode 100644
index 0000000..204d54d
--- /dev/null
+++ b/tools/gpu-aot/native-validation-pending-20261003.md
@@ -0,0 +1,44 @@
+# Required runtime validation after October 3 offline fixes
+
+Do not launch the game until the user authorizes it. All subsequent tests use
+monitor2, no window activation or injected input. Keep the exact executable,
+SDK/runtime/plugin hashes and configuration with every probe; close the exact
+owned process afterward. See `session-20261003/build-proof.json` for the build.
+
+Current output1920x1080, actual game render targets1280x720. This is not a
+1440p native-rendering benchmark. The60FPS1440p target is still unverified.
+
+Once launches are authorized:
+
+1. Verify Enter transition against the saved blue/yellow corruption and normal
+   cyan/red frame. The swizzle core passes tests; no post-fix pixels yet.
+2. Slot2 controllable Vorton, matching native/emulated character and camera.
+   Check head, body, hands, HUD icons, portrait, effects, shadows and portals.
+   Preserve the override material/texture identity if the portrait remains blank.
+3. Audit buffer watches in hub and three story levels, loading/cutscenes and
+   mesh-pool relocations. Require zero stale-clean errors. Until then buffer
+   watches stay OFF by default, independent of the existing texture watch.
+4. Measure clean comparable runs without snapshots, missing-shader dumps,
+   RenderDoc or long traces: warmed hub plus the same level/cutscene sections.
+   Record FPS, median/p95 intervals, draws, texture/constants/vertices/hash and
+   synchronization breakdowns. Confirm which output and actual RT dimensions
+   were measured. Compare fused upload and watch candidates individually.
+5. Level completion and return to hub: record spikes, frame-slot/resource waits
+   and callback latency; check recovery without Alt+Tab. Resize/minimize/restore
+   should preserve aspect and fence/descriptor lifetime.
+6. AMD hardware pass on at least one available RX6600/RX9060XT/Vega8-class
+   machine using the same D3D12 route and matched SDK pair. Compare identical
+   hub, three story levels and cutscenes with NVIDIA captures: HDR brightness,
+   resolve exponent/gamma, RGBA8 transitions, depth/shadow atlas, alpha/HUD,
+   skinned vertex data, level-end spikes and resize/device diagnostics. Record
+   GPU/driver/CPU, dimensions and timing data. No AMD hardware validation today.
+7. Promote native defaults only after these correctness/performance gates.
+   Retain the tested emulated switch. Async CPU-resource wait bypass remains
+   separately opt-in until its CPU-worker/fence contracts are resolved.
+
+The confirmed A58 memexport pool-copy use is already recognized as CPU
+memmove with cache/generation invalidation and immutable host uploads. Today's
+hub log records verified copies with VS A58 and no byte-verification failure
+in those recorded samples. This does not implement arbitrary memexport GPU
+writeback or establish every possible consumer; GPU-owned copy ranges still
+need their own ordered host-resource copy path if observed.

From 345f80526e6a8c2a407ff23ab0b7d65580af0cc8 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 11:36:32 +0300
Subject: [PATCH 29/44] Trace portrait mask identity through native texture
 draws

---
 rexlego/CMakeLists.txt                    |  3 +
 rexlego/src/gpu_native/draw.cpp           |  4 +
 rexlego/src/gpu_native/hooks_portrait.cpp | 96 +++++++++++++++++++++++
 rexlego/src/gpu_native/portrait_probe.h   | 42 ++++++++++
 tools/gpu-aot/test_portrait_identity.cpp  | 37 +++++++++
 5 files changed, 182 insertions(+)
 create mode 100644 rexlego/src/gpu_native/hooks_portrait.cpp
 create mode 100644 rexlego/src/gpu_native/portrait_probe.h
 create mode 100644 tools/gpu-aot/test_portrait_identity.cpp

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index 991c37c..6bc96a1 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -64,6 +64,7 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         src/gpu_native/format.cpp
         src/gpu_native/hooks_buffer.cpp
         src/gpu_native/hooks_device.cpp
+        src/gpu_native/hooks_portrait.cpp
         src/gpu_native/hooks_shader.cpp
         src/gpu_native/hooks_state.cpp
         src/gpu_native/hooks_texture.cpp
@@ -316,6 +317,8 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/buffer_header.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_buffer.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_device.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_portrait.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/portrait_probe.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/hooks_shader.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/shaders.h"
diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
index 50d4b5a..21b2270 100644
--- a/rexlego/src/gpu_native/draw.cpp
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -37,6 +37,7 @@
 #include "gpu_native/textures.h"
 #include "gpu_native/tile_extent.h"
 #include "gpu_native/tonemap_dof.h"
+#include "gpu_native/portrait_probe.h"
 #include "gpu_native/long_probe.h"
 #include "gpu_native/renderdoc_probe.h"
 #include "gpu_native/vertex_declarations.h"
@@ -292,6 +293,9 @@ bool BindConstants(
     // in invalidation, verify their contents at every use, even with watches.
     UploadTextureResource(bindings.textures[i], commands,
         (BoundShaderTextureMask(ShaderStage::kVertex) & (1u << i)) != 0);
+    if (PortraitProbeEnabled())
+      TracePortraitDraw(bindings.textures[i], i, BoundShaderHash(ShaderStage::kVertex),
+                        BoundShaderHash(ShaderStage::kPixel));
     TextureFetchWords fetch;
     for (u32 word = 0; word < fetch.size(); ++word) fetch[word] = device->fetch_constants[i].dword[word];
     if (LongProbeEnabled()) {
diff --git a/rexlego/src/gpu_native/hooks_portrait.cpp b/rexlego/src/gpu_native/hooks_portrait.cpp
new file mode 100644
index 0000000..6f4a386
--- /dev/null
+++ b/rexlego/src/gpu_native/hooks_portrait.cpp
@@ -0,0 +1,96 @@
+#include "gpu_native/renderer_route.h"
+#include "gpu_native/portrait_probe.h"
+#include "gpu_native/long_probe.h"
+#include <rex/system/kernel_state.h>
+#include <rex/system/xmemory.h>
+#include <rex/types.h>
+#include <array>
+#include <algorithm>
+#include <cstring>
+#include <mutex>
+
+extern "C" void __imp__sub_833738B8(PPCContext& __restrict ctx, u8* base);
+extern "C" void __imp__sub_83373850(PPCContext& __restrict ctx, u8* base);
+
+namespace legodimensions::gpu_native {
+namespace {
+std::mutex g_portrait_mutex;
+// Diagnostics only: bounded owner set, removed by the actual destructor.
+std::array<u32, 32> g_portrait_owners{};
+
+bool ReadPortraitWord(u32 address, u32& value) {
+  auto* memory = REX_KERNEL_MEMORY();
+  if (!memory || !address || u64(address) + 4 > 0x100000000ull) return false;
+  auto* heap = memory->LookupHeap(address);
+  if (!heap || !heap->IsRangeCommittedReadable(address, 4)) return false;
+  u32 word;
+  std::memcpy(&word, memory->TranslateVirtual<const u8*>(address), sizeof(word));
+  value = __builtin_bswap32(word);
+  return true;
+}
+
+void TracePortraitSetup(u32 owner) {
+  if (!PortraitProbeEnabled()) return;
+  const auto identity = ReadPortraitIdentity(owner, ReadPortraitWord);
+  {
+    std::lock_guard lock(g_portrait_mutex);
+    if (std::find(g_portrait_owners.begin(), g_portrait_owners.end(), owner) ==
+        g_portrait_owners.end())
+      for (auto& entry : g_portrait_owners)
+        if (!entry) { entry = owner; break; }
+  }
+  LongProbeEvent("portrait_setup", false, "owner=", owner, "scene=", identity.scene,
+      "material=", identity.material, "texture_object=", identity.texture_object,
+      "texture=", identity.texture, "material_flags=", identity.material_flags,
+      "texture_flags=", identity.texture_flags, "readable_fields=", identity.readable_fields);
+}
+
+void ForgetPortrait(u32 owner) {
+  if (!PortraitProbeEnabled()) return;
+  std::lock_guard lock(g_portrait_mutex);
+  for (auto& entry : g_portrait_owners) if (entry == owner) entry = 0;
+  LongProbeEvent("portrait_destroy", false, "owner=", owner);
+}
+}  // namespace
+
+bool PortraitProbeEnabled() {
+  static const bool requested = [] {
+    const auto* value = std::getenv("LEGO_NATIVE_PORTRAIT_TRACE");
+    return value && std::strcmp(value, "1") == 0;
+  }();
+  return requested && LongProbeEnabled();
+}
+
+void TracePortraitDraw(u32 texture, u32 slot, u64 vertex_shader, u64 pixel_shader) {
+  if (!PortraitProbeEnabled() || !texture) return;
+  std::lock_guard lock(g_portrait_mutex);
+  static u32 previous_frame = ~u32{0}, reports = 0;
+  const auto frame = g_probe_frame.load();
+  if (frame != previous_frame) { previous_frame = frame; reports = 0; }
+  if (reports >= 64) return;
+  for (auto owner : g_portrait_owners) {
+    if (!owner) continue;
+    // Read current identity: setup pointers or the NuTexture's D3D resource
+    // may have changed since loading. Never retain a host resource here.
+    const auto identity = ReadPortraitIdentity(owner, ReadPortraitWord);
+    if (identity.texture != texture || !(identity.readable_fields & 32)) continue;
+    ++reports;
+    LongProbeEvent("portrait_texture_draw", false, "owner=", owner,
+        "material=", identity.material, "texture_object=", identity.texture_object,
+        "texture=", texture, "slot=", slot, "VS=", vertex_shader, "PS=", pixel_shader,
+        "material_flags=", identity.material_flags, "texture_flags=", identity.texture_flags);
+    break;
+  }
+}
+}  // namespace legodimensions::gpu_native
+
+REX_HOOK_RAW(sub_833738B8) {
+  const u32 owner = ctx.r3.u32;
+  __imp__sub_833738B8(ctx, base);
+  legodimensions::gpu_native::TracePortraitSetup(owner);
+}
+
+REX_HOOK_RAW(sub_83373850) {
+  legodimensions::gpu_native::ForgetPortrait(ctx.r3.u32);
+  __imp__sub_83373850(ctx, base);
+}
diff --git a/rexlego/src/gpu_native/portrait_probe.h b/rexlego/src/gpu_native/portrait_probe.h
new file mode 100644
index 0000000..e15681f
--- /dev/null
+++ b/rexlego/src/gpu_native/portrait_probe.h
@@ -0,0 +1,42 @@
+// Read-only TU23 portrait identity. Offsets come from 833738B8, 82CC4E50
+// and the texture leaf 82B6FC70; they are not guessed from a HUD draw.
+#pragma once
+#include <cstdint>
+#include <limits>
+
+namespace legodimensions::gpu_native {
+struct PortraitIdentity {
+  uint32_t owner = 0;
+  uint32_t scene = 0;
+  uint32_t material = 0;
+  uint32_t texture_object = 0;
+  uint32_t texture = 0;
+  uint32_t material_flags = 0;
+  uint32_t texture_flags = 0;
+  uint32_t readable_fields = 0;
+};
+
+template<class ReadWord>
+PortraitIdentity ReadPortraitIdentity(uint32_t owner, ReadWord&& read) {
+  PortraitIdentity result;
+  result.owner = owner;
+  const auto field = [&](uint32_t object, uint32_t offset, uint32_t& value,
+                         uint32_t bit) {
+    if (!object || uint64_t(object) + offset + 4 > (uint64_t{1} << 32)) return false;
+    if (!read(object + offset, value)) return false;
+    result.readable_fields |= bit;
+    return true;
+  };
+  field(owner, 84, result.scene, 1);
+  if (!field(owner, 200, result.material, 2) || !result.material) return result;
+  field(result.material, 948, result.material_flags, 4);
+  field(result.material, 952, result.texture_flags, 8);
+  if (!field(result.material, 956, result.texture_object, 16)) return result;
+  field(result.texture_object, 4, result.texture, 32);
+  return result;
+}
+
+bool PortraitProbeEnabled();
+void TracePortraitDraw(uint32_t texture, uint32_t slot, uint64_t vertex_shader,
+                       uint64_t pixel_shader);
+}  // namespace legodimensions::gpu_native
diff --git a/tools/gpu-aot/test_portrait_identity.cpp b/tools/gpu-aot/test_portrait_identity.cpp
new file mode 100644
index 0000000..2d2e071
--- /dev/null
+++ b/tools/gpu-aot/test_portrait_identity.cpp
@@ -0,0 +1,37 @@
+#include "gpu_native/portrait_probe.h"
+#include <cassert>
+#include <iostream>
+#include <unordered_map>
+#include <vector>
+using namespace legodimensions::gpu_native;
+int main() {
+  std::unordered_map<uint32_t,uint32_t> words{
+      {0x1000+84,0x2000},{0x1000+200,0x3000},
+      {0x3000+948,0x12345678},{0x3000+952,0x87654321},
+      {0x3000+956,0x4000},{0x4000+4,0x5000}};
+  const auto original=words;
+  std::vector<uint32_t> reads;
+  auto read=[&](uint32_t address,uint32_t& out) {
+    reads.push_back(address);
+    auto it=words.find(address);if(it==words.end())return false;
+    out=it->second;return true;
+  };
+  auto identity=ReadPortraitIdentity(0x1000,read);
+  assert(identity.readable_fields==63 && identity.texture==0x5000);
+  assert(identity.scene==0x2000 && identity.material==0x3000 &&
+         identity.texture_object==0x4000 && identity.material_flags==0x12345678);
+  assert(words==original);
+  words[0x4000+4]=0x6000;
+  assert(ReadPortraitIdentity(0x1000,read).texture==0x6000);
+  words.erase(0x4000+4);
+  assert(!(ReadPortraitIdentity(0x1000,read).readable_fields&32));
+  words[0x3000+956]=0;
+  assert(ReadPortraitIdentity(0x1000,read).texture==0);
+  words[0x1000+200]=0;
+  reads.clear();ReadPortraitIdentity(0x1000,read);assert(reads.size()==2);
+  reads.clear();ReadPortraitIdentity(0,read);assert(reads.empty());
+  reads.clear();ReadPortraitIdentity(0xFFFFFFF0,read);assert(reads.empty());
+  words[0x1000+200]=0xFFFFFFF0;
+  reads.clear();ReadPortraitIdentity(0x1000,read);assert(reads.size()==2);
+  std::cout<<"PASS: actual portrait identity, relocation, unreadable/null objects, 32-bit bounds, read-only fields\n";
+}

From 2e28b8c0001b147cf31c14f4e699e4355d0ba83d Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 11:37:32 +0300
Subject: [PATCH 30/44] Attribute native frame waits and report frame-time
 tails

---
 rexlego/src/gpu_native/device.cpp        | 42 ++++++++++++++++++---
 tools/gpu-aot/summarize_frame_metrics.py | 19 ++++++++--
 tools/gpu-aot/test_draw_upload.py        |  8 ++++
 tools/gpu-aot/test_frame_metrics.py      | 48 ++++++++++++++++++++++++
 4 files changed, 109 insertions(+), 8 deletions(-)
 create mode 100644 tools/gpu-aot/test_frame_metrics.py

diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp
index 5cb8a35..5e06a41 100644
--- a/rexlego/src/gpu_native/device.cpp
+++ b/rexlego/src/gpu_native/device.cpp
@@ -190,6 +190,11 @@ struct State {
   u32 timing_log_count = 0;
   u64 sync_calls = 0, cpu_resource_wait_skips = 0;
   double sync_ms = 0;
+  u64 frame_slot_wait_calls = 0;
+  double frame_slot_wait_ms = 0;
+  double acquire_cpu_ms = 0;
+  double present_submit_cpu_ms = 0;
+  double swap_present_cpu_ms = 0;
   std::array<u64, static_cast<u32>(SyncReason::kCount)> sync_reason_calls{};
   std::array<double, static_cast<u32>(SyncReason::kCount)> sync_reason_ms{};
   std::chrono::steady_clock::time_point probe_next_snapshot{};
@@ -548,6 +553,10 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) {
     const auto wait_ms = std::chrono::duration<double, std::milli>(
                              std::chrono::steady_clock::now() - wait_start)
                              .count();
+    if (NativeTextureTimingEnabled()) {
+      ++state.frame_slot_wait_calls;
+      state.frame_slot_wait_ms += wait_ms;
+    }
     if (wait_ms > 20.0 && state.long_wait_log_count++ < 20) {
       REXLOG_WARN("Native GPU: frame-slot {} GPU wait took {:.2f} ms", slot,
                   wait_ms);
@@ -1147,9 +1156,14 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
   const u32 slot = state.frame_slot;
 
   u32 image_index = 0;
-  if (!state.swap_chain->acquireTexture(
-          state.acquire_semaphores[slot].get(), &image_index) ||
-      image_index >= state.framebuffers.size()) {
+  const bool timing_enabled = NativeTextureTimingEnabled();
+  const auto acquire_start = timing_enabled ? std::chrono::steady_clock::now()
+      : std::chrono::steady_clock::time_point{};
+  const bool acquired = state.swap_chain->acquireTexture(
+      state.acquire_semaphores[slot].get(), &image_index);
+  if (timing_enabled) state.acquire_cpu_ms += std::chrono::duration<double, std::milli>(
+      std::chrono::steady_clock::now() - acquire_start).count();
+  if (!acquired || image_index >= state.framebuffers.size()) {
     // Close and submit any recorded uploads/draws so the allocator remains in
     // a valid state while the window is minimized or the swap chain changes.
     commands->end();
@@ -1207,10 +1221,18 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
       state.acquire_semaphores[slot].get()};
   plume::RenderCommandSemaphore* signals[] = {
       state.render_semaphores[image_index].get()};
+  const auto submit_start = timing_enabled ? std::chrono::steady_clock::now()
+      : std::chrono::steady_clock::time_point{};
   state.queue->executeCommandLists(lists, 1, waits, 1, signals, 1,
                                    state.frame_fences[slot].get());
+  if (timing_enabled) state.present_submit_cpu_ms += std::chrono::duration<double, std::milli>(
+      std::chrono::steady_clock::now() - submit_start).count();
   MarkSubmissionLocked(state, slot);
+  const auto swap_start = timing_enabled ? std::chrono::steady_clock::now()
+      : std::chrono::steady_clock::time_point{};
   const bool presented = state.swap_chain->present(image_index, signals, 1);
+  if (timing_enabled) state.swap_present_cpu_ms += std::chrono::duration<double, std::milli>(
+      std::chrono::steady_clock::now() - swap_start).count();
   ReportLogoCapture();
   ++state.present_number;
   g_probe_frame.store(state.present_number);
@@ -1231,7 +1253,9 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
           for (const char* name : {"other", "idle", "fence", "resource", "callback", "query_begin", "query_release"})
             stream << ",sync_" << name << "_calls,sync_" << name << "_ms";
           stream << ",callbacks_enqueued,callbacks_executed,callbacks_pending"
-                 << ",bindings_ms,begin_ms,pipeline_ms,issue_ms,tail_ms,buffer_converted_bytes\n";
+                 << ",bindings_ms,begin_ms,pipeline_ms,issue_ms,tail_ms,buffer_converted_bytes"
+                 << ",frame_slot_wait_calls,frame_slot_wait_ms,acquire_cpu_ms,present_submit_cpu_ms,swap_present_cpu_ms"
+                 << ",buffer_watch_hits,buffer_watch_audits,buffer_watch_mismatches\n";
         }
         return stream;
       }();
@@ -1247,7 +1271,12 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
                       << ',' << state.completion_callbacks.size()
                       << ',' << draws.stages_ms[0] << ',' << draws.stages_ms[1]
                       << ',' << draws.stages_ms[2] << ',' << draws.stages_ms[5]
-                      << ',' << draws.stages_ms[6] << ',' << buffers.converted_bytes << '\n';
+                      << ',' << draws.stages_ms[6] << ',' << buffers.converted_bytes
+                      << ',' << state.frame_slot_wait_calls << ',' << state.frame_slot_wait_ms
+                      << ',' << state.acquire_cpu_ms << ',' << state.present_submit_cpu_ms
+                      << ',' << state.swap_present_cpu_ms
+                      << ',' << buffers.watch_hits << ',' << buffers.watch_audits
+                      << ',' << buffers.watch_mismatches << '\n';
         if (state.present_number % 120 == 0) frame_metrics.flush();
       }
       if (LongProbeEnabled()) LongProbeEvent("frame", interval_ms > 2000.0,
@@ -1286,6 +1315,9 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture,
     state.sync_calls = 0;
     state.cpu_resource_wait_skips = 0;
     state.sync_ms = 0;
+    state.frame_slot_wait_calls = 0;
+    state.frame_slot_wait_ms = state.acquire_cpu_ms = 0;
+    state.present_submit_cpu_ms = state.swap_present_cpu_ms = 0;
     state.sync_reason_calls.fill(0);
     state.sync_reason_ms.fill(0);
     state.callbacks_enqueued = state.callbacks_executed = 0;
diff --git a/tools/gpu-aot/summarize_frame_metrics.py b/tools/gpu-aot/summarize_frame_metrics.py
index 752bb8b..e3b6c43 100644
--- a/tools/gpu-aot/summarize_frame_metrics.py
+++ b/tools/gpu-aot/summarize_frame_metrics.py
@@ -3,6 +3,7 @@
 import csv
 import io
 import json
+import math
 from pathlib import Path
 
 p=argparse.ArgumentParser(description=__doc__);p.add_argument('file',type=Path)
@@ -10,10 +11,15 @@
 p.add_argument('--after-frame',type=int,default=0)
 p.add_argument('--through-frame',type=int)
 a=p.parse_args()
+if not math.isfinite(a.tail_seconds) or a.tail_seconds <= 0:
+ p.error('--tail-seconds must be finite and positive')
 text=a.file.read_text();text=text[:text.rfind('\n')+1] # Exclude a partial live row.
 rows=[]
 for row in csv.DictReader(io.StringIO(text)):
- try: rows.append({k:float(v) for k,v in row.items()})
+ try:
+  parsed={k:float(v) for k,v in row.items()}
+  if (parsed.get('interval_ms',0)>0 and 'frame' in parsed and
+      all(math.isfinite(v) for v in parsed.values())): rows.append(parsed)
  except (TypeError,ValueError):continue
 rows=[r for r in rows if r['frame']>a.after_frame and
       (a.through_frame is None or r['frame']<=a.through_frame)]
@@ -27,22 +33,29 @@ def stats(data):
  mean=lambda key:sum(r[key] for r in data)/len(data)
  result={'frames':len(data),'seconds':sum(times)/1000,'mean_fps':1000/mean('interval_ms'),
  'frame_ms_p50':times[(len(times)-1)//2],'frame_ms_p95':times[int((len(times)-1)*.95)],
+ 'frame_ms_p99':times[int((len(times)-1)*.99)],
  'frame_ms_max':max(times),'over_33_5ms_percent':100*sum(t>33.5 for t in times)/len(times),
+ 'over_16_67ms_percent':100*sum(t>1000/60 for t in times)/len(times),
+ 'over_33_33ms_percent':100*sum(t>1000/30 for t in times)/len(times),
  'texture_ms_mean':mean('texture_ms'),'constants_ms_mean':mean('constants_ms'),
  'vertices_ms_mean':mean('vertices_ms'),'buffer_hash_ms_mean':mean('buffer_hash_ms'),
  'present_cpu_ms_mean':mean('present_cpu_ms'),'draws_mean':mean('draw_calls')}
  for key in ['cpu_resource_wait_skips','bindings_ms','begin_ms','pipeline_ms','issue_ms',
-             'tail_ms','buffer_converted_bytes'] + [k for k in data[0] if k.startswith(('sync_', 'callbacks_'))]:
+             'tail_ms','buffer_converted_bytes','frame_slot_wait_calls','frame_slot_wait_ms',
+             'acquire_cpu_ms','present_submit_cpu_ms','swap_present_cpu_ms',
+             'buffer_watch_hits','buffer_watch_audits','buffer_watch_mismatches'] + [k for k in data[0] if k.startswith(('sync_', 'callbacks_'))]:
   if key in data[0]:result[key+'_mean']=mean(key)
  return result
 record_path=a.file.with_name(a.file.name.replace('-frames.csv','-process.json'))
 record=json.loads(record_path.read_text(encoding='utf-8-sig')) if record_path.exists() else None
 heavy_capture=bool(record and (record.get('mode')=='snapshot' or any(record.get(k) for k in
- ['longProbe','logoUploads','logoCaptureRenderDocDll','captureMissing','meshTrace','auditTextureWatch'])))
+ ['longProbe','logoUploads','logoCaptureRenderDocDll','captureMissing','meshTrace','auditTextureWatch','auditBufferWatch','portraitTrace'])))
 report={'source':str(a.file),'whole_run':stats(rows),'tail':stats(tail),
         'latest_frame':rows[-1]['frame'] if rows else None,
         'after_frame':a.after_frame,'through_frame':a.through_frame,
         'scene_identity_requires_visual_confirmation':True,
+        'percentile_method':'lower order statistic: floor((n-1)*q)',
+        'frame_budgets_ms':{'60fps':1000/60,'30fps':1000/30},
         'clean_performance_probe':not heavy_capture if record else None}
 a.file.with_suffix('.summary.json').write_text(json.dumps(report,indent=2))
 print(json.dumps(report,indent=2))
diff --git a/tools/gpu-aot/test_draw_upload.py b/tools/gpu-aot/test_draw_upload.py
index 6aaedcd..9952358 100644
--- a/tools/gpu-aot/test_draw_upload.py
+++ b/tools/gpu-aot/test_draw_upload.py
@@ -66,6 +66,7 @@ def function(signature):
  std::array<std::vector<std::unique_ptr<DrawUploadPage>>,3> draw_uploads;
  u32 frame_slot=0,draw_upload_page=0,long_wait_log_count=0;
  u64 draw_upload_offset=0;bool command_list_open=false;
+ u64 frame_slot_wait_calls=0;double frame_slot_wait_ms=0;
  DescriptorRetirement retired_descriptors;std::array<bool,2> texture_slots{};
  State(){for(u32 s=0;s<3;++s){command_lists[s]=std::make_unique<plume::RenderCommandList>();frame_fences[s]=std::make_unique<plume::Fence>();}}
 };
@@ -73,6 +74,8 @@ def function(signature):
 void RefreshCompletedSubmissionsLocked(State&){}
 void InitializeNullTextures(State&,plume::RenderCommandList*){}
 bool RebuildSwapChain(State&){return true;}
+bool timing_enabled=true;
+bool NativeTextureTimingEnabled(){return timing_enabled;}
 std::unique_ptr<State> g_state=std::make_unique<State>();std::mutex g_mutex;
 struct HostDevice {
  static auto LockRecording(){static std::recursive_mutex m;return std::unique_lock<std::recursive_mutex>(m);}
@@ -95,6 +98,11 @@ def function(signature):
  g_state->command_list_open=false;g_state->frame_submitted[1]=true;g_state->frame_slot=0;
  auto reuse=HostDevice::AllocateDrawUpload(8816);assert(reuse.buffer==first.buffer&&reuse.offset==0);
  assert(g_state->queue->waits==1); // Reset only after old fence wait.
+ assert(g_state->frame_slot_wait_calls==1 && g_state->frame_slot_wait_ms>=0);
+ timing_enabled=false;
+ g_state->command_list_open=false;g_state->frame_submitted[0]=true;
+ auto untimed=HostDevice::AllocateDrawUpload(8816);assert(untimed);
+ assert(g_state->queue->waits==2 && g_state->frame_slot_wait_calls==1);
  std::memset(reuse.mapped,0xDD,8816);assert(static_cast<u8*>(other.mapped)[0]==0xCC);
  auto large=HostDevice::AllocateDrawUpload(5*1024*1024);assert(large&&large.offset==0);
  assert(g_state->device->allocations==4);
diff --git a/tools/gpu-aot/test_frame_metrics.py b/tools/gpu-aot/test_frame_metrics.py
new file mode 100644
index 0000000..a3992e6
--- /dev/null
+++ b/tools/gpu-aot/test_frame_metrics.py
@@ -0,0 +1,48 @@
+"""Exercise the report CLI with controlled frame budgets and diagnostic records."""
+import argparse
+import csv
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+p=argparse.ArgumentParser(description=__doc__)
+p.add_argument('output',type=Path)
+a=p.parse_args();a.output.mkdir(parents=True,exist_ok=True)
+script=Path(__file__).with_name('summarize_frame_metrics.py')
+path=a.output/'fixture-frames.csv'
+fields=['frame','interval_ms','texture_ms','constants_ms','vertices_ms',
+        'buffer_hash_ms','present_cpu_ms','draw_calls','frame_slot_wait_ms',
+        'acquire_cpu_ms','present_submit_cpu_ms','swap_present_cpu_ms',
+        'buffer_watch_hits','buffer_watch_audits','buffer_watch_mismatches']
+with path.open('w',newline='') as f:
+ w=csv.DictWriter(f,fieldnames=fields);w.writeheader()
+ for n in range(1,101):
+  w.writerow(dict.fromkeys(fields,0)|{'frame':n,'interval_ms':n,'draw_calls':505,
+      'frame_slot_wait_ms':2,'acquire_cpu_ms':3,'present_submit_cpu_ms':4,
+      'swap_present_cpu_ms':5,'buffer_watch_audits':6})
+ for interval in ('nan','inf',0,-1):
+  w.writerow(dict.fromkeys(fields,0)|{'frame':101,'interval_ms':interval})
+ f.write('102,1000,')
+record=path.with_name('fixture-process.json')
+def report(*args):
+ return json.loads(subprocess.check_output([sys.executable,str(script),str(path),*args],text=True))
+record.write_text(json.dumps({'mode':'timing','auditBufferWatch':True}))
+r=report('--tail-seconds','0.1')
+s=r['whole_run']
+assert s['frames']==100 and s['seconds']==5.05 and s['mean_fps']==1000/50.5
+assert (s['frame_ms_p50'],s['frame_ms_p95'],s['frame_ms_p99'],s['frame_ms_max'])==(50,95,99,100)
+assert s['over_16_67ms_percent']==84 and s['over_33_33ms_percent']==67
+assert s['frame_slot_wait_ms_mean']==2 and s['swap_present_cpu_ms_mean']==5
+assert s['buffer_watch_audits_mean']==6 and s['buffer_watch_mismatches_mean']==0
+assert r['tail']['frames']==1 and not r['clean_performance_probe']
+record.write_text(json.dumps({'mode':'timing','portraitTrace':True}))
+assert not report()['clean_performance_probe']
+record.write_text(json.dumps({'mode':'timing'}))
+assert report()['clean_performance_probe']
+r=report('--after-frame','90','--through-frame','95')
+assert r['whole_run']['frames']==5 and r['latest_frame']==95
+assert report('--after-frame','100')['whole_run']=={}
+record.unlink()
+assert report()['clean_performance_probe'] is None
+print('PASS: CLI frame budgets, percentile tails, malformed/partial rows, window selection, audit exclusion')

From dd8f4ea2ee5683ed4e26a9ea8d70da6d0b0540fa Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 11:37:33 +0300
Subject: [PATCH 31/44] Record native 1080p target and prepare monitor-two
 probes

---
 .../native-validation-pending-20261003.md     | 18 +++--
 tools/gpu-aot/native-work-20261004.md         | 75 +++++++++++++++++++
 tools/gpu-aot/run_native_probe.ps1            |  8 +-
 3 files changed, 94 insertions(+), 7 deletions(-)
 create mode 100644 tools/gpu-aot/native-work-20261004.md

diff --git a/tools/gpu-aot/native-validation-pending-20261003.md b/tools/gpu-aot/native-validation-pending-20261003.md
index 204d54d..6bfb73c 100644
--- a/tools/gpu-aot/native-validation-pending-20261003.md
+++ b/tools/gpu-aot/native-validation-pending-20261003.md
@@ -6,22 +6,30 @@ SDK/runtime/plugin hashes and configuration with every probe; close the exact
 owned process afterward. See `session-20261003/build-proof.json` for the build.
 
 Current output1920x1080, actual game render targets1280x720. This is not a
-1440p native-rendering benchmark. The60FPS1440p target is still unverified.
+1080p native-rendering benchmark. The owner changed the target to stable60FPS
+at actual1920x1080 internal rendering on October4; this is still unverified.
 
 Once launches are authorized:
 
-1. Verify Enter transition against the saved blue/yellow corruption and normal
-   cyan/red frame. The swizzle core passes tests; no post-fix pixels yet.
+1. The owner confirmed on October4 that Enter no longer causes the yellow
+   glitch. Preserve that user-confirmed result; an assistant post-fix capture
+   comparison has not yet been made.
 2. Slot2 controllable Vorton, matching native/emulated character and camera.
    Check head, body, hands, HUD icons, portrait, effects, shadows and portals.
    Preserve the override material/texture identity if the portrait remains blank.
+   October4: use -LongProbe -PortraitTrace to correlate the actual mask's
+   NuTexture+4 D3D resource with draw shader/texture/constant captures. See
+   native-work-20261004.md. Keep this separate from clean FPS measurements.
 3. Audit buffer watches in hub and three story levels, loading/cutscenes and
    mesh-pool relocations. Require zero stale-clean errors. Until then buffer
    watches stay OFF by default, independent of the existing texture watch.
 4. Measure clean comparable runs without snapshots, missing-shader dumps,
    RenderDoc or long traces: warmed hub plus the same level/cutscene sections.
-   Record FPS, median/p95 intervals, draws, texture/constants/vertices/hash and
-   synchronization breakdowns. Confirm which output and actual RT dimensions
+   Record FPS, median/p95/p99/max intervals, draws, texture/constants/vertices/
+   hash and synchronization breakdowns, 60/30FPS-budget misses,
+   frame-slot/acquire/submit/present CPU times and buffer watch hits/audits/
+   mismatches. The new columns are CPU wall times, not GPU timestamps.
+   Confirm which output and actual RT dimensions
    were measured. Compare fused upload and watch candidates individually.
 5. Level completion and return to hub: record spikes, frame-slot/resource waits
    and callback latency; check recovery without Alt+Tab. Resize/minimize/restore
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
new file mode 100644
index 0000000..f6966ea
--- /dev/null
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -0,0 +1,75 @@
+# Native D3D12 work, October 4
+
+The owner changed the target to **stable 60 FPS at actual 1920x1080 internal
+rendering**, retaining C++. Another agent is working on ReXGlue Vulkan fixes.
+This work modifies the game-side D3D12 renderer only; it does not rebuild,
+deploy or modify the SDK or Plume. Game launches remain pending explicit
+authorization, and future probes default to monitor 2.
+
+The owner confirms the Enter yellow transition glitch no longer appears.
+This is user-confirmed visual evidence, not a new assistant capture.
+
+## Portrait identity evidence and diagnostic
+
+The read-only Avengers research identified Dimensions setup 833738B8 and its
+material override. Further inspection of the actual TU23 generated bodies:
+
+- 833738B8 stores the loaded object at owner+84 and the override at owner+200.
+- 83022C20 returns a smart-pointer wrapper for the property texture.
+- 82CC4E50 copies the wrapper's pointee into material+956 and retains/releases
+  through the object's vtable. Material+956 is therefore a NuTexture pointer,
+  not directly a D3D texture header.
+- 82B6FC70 is exactly a load from texture object+4. This gives the D3D resource
+  identity to compare with native texture bindings.
+- 83373850 destroys this owner. The diagnostic removes it before destruction.
+
+New C++ hooks retain the original setup/destructor behavior. With
+LEGO_NATIVE_PORTRAIT_TRACE=1 and LEGO_NATIVE_TRACE_DIR set, they log
+portrait_setup / portrait_destroy. BindConstants reports portrait_texture_draw
+when the current owner/material/NuTexture chain matches a referenced texture
+slot, including VS/PS hashes. Re-reading the chain avoids keeping stale setup
+identities; reads check commitment and 32-bit bounds. At most 32 owners are
+tracked and 64 matched draws reported per frame. Disabled mode performs no
+guest reads, and the emulated route bypasses the diagnostic hook bodies.
+
+run_native_probe.ps1 -PortraitTrace requires -LongProbe, which provides the
+existing correlated draw, constants and intermediate-image capture. This is a
+heavy correctness probe, never an FPS benchmark. No missing portrait cause or
+fix is claimed until the actual material/texture/alpha/UV is captured.
+
+## Performance attribution
+
+Timing CSV now includes frame_slot_wait_calls/ms, acquire_cpu_ms,
+present_submit_cpu_ms and swap_present_cpu_ms, plus buffer watch hits/audits/
+mismatches. These are CPU wall times around existing calls; they are not GPU
+timestamp durations. Frame-slot timing includes the existing completion refresh
+after the fence wait. Present submission excludes separate KickOff submissions.
+The counters do not change fences, descriptor lifetime or submission order.
+
+The summarizer includes p99 and the proportions over exact 60/30 FPS frame
+budgets. It documents its lower-order-statistic percentile convention, ignores
+nonfinite/nonpositive intervals and unfinished live CSV rows, and excludes
+buffer audits and portrait tracing from clean performance claims. Earlier CSV
+columns and the previous 33.5ms statistic remain supported.
+
+Offline checks passed: actual portrait identity chain (null/unreadable/relocated
+objects and overflow), actual renderer routing macros for all 58 hooks across
+12 environment/config cases, actual draw upload/slot-reset functions with timing
+ON/OFF, actual fence/resize functions with deterministic driver/events, and
+report CLI percentile/budget/window/diagnostic exclusion fixtures. Native Release
+build verification is recorded under session-20261004. No new gameplay FPS.
+
+## Actual 1080p is separate work
+
+Output remains 1920x1080; game targets remain 1280x720. The generated initializer
+82B67190 creates multiple full-screen targets via 82B64790 / 82B64828, a 640x360
+target, and specially arranged 960x800 and 1280x608 surfaces with stack-supplied
+offset parameters. Owner+924 is also initialized to 1280. Scaling only the
+1280x720 allocations would leave this layout inconsistent. The 1080p change
+must account for engine dimensions, viewport/scissor/resolve rectangles,
+postprocessing and the depth-bank mapping together; no blind allocation-size
+override was enabled.
+
+Next authorized runs: a matching Vorton portrait capture, clean warmed baseline,
+buffer-watch audit, then controlled OFF/ON comparisons. Actual 1080p, three story
+levels, level-completion spikes and AMD remain open validation gates.
diff --git a/tools/gpu-aot/run_native_probe.ps1 b/tools/gpu-aot/run_native_probe.ps1
index 73c6dae..ef16498 100644
--- a/tools/gpu-aot/run_native_probe.ps1
+++ b/tools/gpu-aot/run_native_probe.ps1
@@ -4,6 +4,7 @@ param(
     [switch]$AllTextures,
     [switch]$CaptureMissing,
     [switch]$LongProbe,
+    [switch]$PortraitTrace,
     [switch]$NoMemoryWatch,
     [switch]$StaticTextureWatch,
     [switch]$AuditTextureWatch,
@@ -21,10 +22,11 @@ param(
     [switch]$Pm4Reference,
     [switch]$ConfiguredRenderer,
     [string]$LogoCaptureRenderDocDll,
-    [ValidateRange(1,8)][int]$Monitor = 1,
+    [ValidateRange(1,8)][int]$Monitor = 2,
     [switch]$Close
 )
 $ErrorActionPreference = 'Stop'
+if ($PortraitTrace -and !$LongProbe) { throw 'PortraitTrace requires LongProbe for correlated draw/constants capture' }
 if ($ConfiguredRenderer -and $Pm4Reference) { throw 'Select configured renderer or explicit PM4 reference, not both' }
 if ($DepthAlias -and $DisableDepthAlias) { throw 'DepthAlias and DisableDepthAlias are mutually exclusive' }
 if (($Pm4Reference -or $ConfiguredRenderer) -and ($Stencil -or $MeshTrace -or $LogoUploads -or $Viewport -or $BufferWindows -or $BufferWatch -or $AuditBufferWatch -or $DepthAlias -or $DisableDepthAlias -or $LongProbe -or $LogoCaptureRenderDocDll)) {
@@ -55,7 +57,7 @@ $keys = @('SDL_WINDOW_ACTIVATE_WHEN_SHOWN','LEGO_NATIVE_TIMING','LEGO_NATIVE_ALL
     'LEGO_NATIVE_NO_MEMORY_WATCH','LEGO_NATIVE_STATIC_TEXTURE_WATCH','LEGO_NATIVE_AUDIT_TEXTURE_WATCH','LEGO_NATIVE_BUFFER_WATCH','LEGO_NATIVE_AUDIT_BUFFER_WATCH','LEGO_NATIVE_NO_DRAW_ARENA',
     'LEGO_NATIVE_FRAME_METRICS','LEGO_NATIVE_ASYNC_CPU_RESOURCES','LEGO_NATIVE_STENCIL','LEGO_NATIVE_MESH_TRACE',
     'LEGO_NATIVE_VIEWPORT','LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY','LEGO_NATIVE_BUFFER_WINDOWS','LEGO_NATIVE_PM4_REFERENCE',
-    'LEGO_NATIVE_RENDERDOC_DLL','LEGO_NATIVE_RENDERDOC_CAPTURE','LEGO_NATIVE_DEPTH_ALIAS')
+    'LEGO_NATIVE_RENDERDOC_DLL','LEGO_NATIVE_RENDERDOC_CAPTURE','LEGO_NATIVE_DEPTH_ALIAS','LEGO_NATIVE_PORTRAIT_TRACE')
 $saved = @{}
 foreach ($key in $keys) {
     $saved[$key] = [Environment]::GetEnvironmentVariable($key,'Process')
@@ -104,6 +106,7 @@ try {
         $env:LEGO_GPU_SNAPSHOT_START_AT_TILING = '1'
     }
     if ($AllTextures) { $env:LEGO_NATIVE_ALL_TEXTURES = '1' }
+    if ($PortraitTrace) { $env:LEGO_NATIVE_PORTRAIT_TRACE = '1' }
     if ($CaptureMissing) {
         $env:LEGO_DUMP_MISSING_SHADERS = Join-Path $runRoot "missing-$Label"
     }
@@ -130,6 +133,7 @@ try {
     $record = [ordered]@{id=$game.Id;path=$exePath;startedAt=$game.StartTime.ToString('o');
         mode=$Mode;allTextures=[bool]$AllTextures;captureMissing=[bool]$CaptureMissing;monitor=$Monitor;
         longProbe=[bool]$LongProbe;noMemoryWatch=[bool]$NoMemoryWatch;staticTextureWatch=[bool]$StaticTextureWatch;
+        portraitTrace=[bool]$PortraitTrace;
         auditTextureWatch=[bool]$AuditTextureWatch;
         bufferWatch=[bool]$BufferWatch; auditBufferWatch=[bool]$AuditBufferWatch;
         noDrawArena=[bool]$NoDrawArena;asyncCpuResources=[bool]$AsyncCpuResources;

From 5c6ca107ae3d4988c7476d77fe30e1d2865b740b Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 11:42:21 +0300
Subject: [PATCH 32/44] Follow active inline D3D header in portrait texture
 trace

---
 rexlego/src/gpu_native/hooks_portrait.cpp     |  4 +-
 rexlego/src/gpu_native/portrait_probe.h       | 14 ++-
 .../native-validation-pending-20261003.md     |  2 +-
 tools/gpu-aot/native-work-20261004.md         |  7 +-
 tools/gpu-aot/test_portrait_identity.cpp      | 11 ++-
 tools/gpu-aot/test_portrait_probe.py          | 96 +++++++++++++++++++
 6 files changed, 125 insertions(+), 9 deletions(-)
 create mode 100644 tools/gpu-aot/test_portrait_probe.py

diff --git a/rexlego/src/gpu_native/hooks_portrait.cpp b/rexlego/src/gpu_native/hooks_portrait.cpp
index 6f4a386..58d86c6 100644
--- a/rexlego/src/gpu_native/hooks_portrait.cpp
+++ b/rexlego/src/gpu_native/hooks_portrait.cpp
@@ -41,6 +41,7 @@ void TracePortraitSetup(u32 owner) {
   }
   LongProbeEvent("portrait_setup", false, "owner=", owner, "scene=", identity.scene,
       "material=", identity.material, "texture_object=", identity.texture_object,
+      "texture_backend=", identity.texture_backend, "active_index=", identity.active_texture_index,
       "texture=", identity.texture, "material_flags=", identity.material_flags,
       "texture_flags=", identity.texture_flags, "readable_fields=", identity.readable_fields);
 }
@@ -73,10 +74,11 @@ void TracePortraitDraw(u32 texture, u32 slot, u64 vertex_shader, u64 pixel_shade
     // Read current identity: setup pointers or the NuTexture's D3D resource
     // may have changed since loading. Never retain a host resource here.
     const auto identity = ReadPortraitIdentity(owner, ReadPortraitWord);
-    if (identity.texture != texture || !(identity.readable_fields & 32)) continue;
+    if (identity.texture != texture || !(identity.readable_fields & 128)) continue;
     ++reports;
     LongProbeEvent("portrait_texture_draw", false, "owner=", owner,
         "material=", identity.material, "texture_object=", identity.texture_object,
+        "texture_backend=", identity.texture_backend, "active_index=", identity.active_texture_index,
         "texture=", texture, "slot=", slot, "VS=", vertex_shader, "PS=", pixel_shader,
         "material_flags=", identity.material_flags, "texture_flags=", identity.texture_flags);
     break;
diff --git a/rexlego/src/gpu_native/portrait_probe.h b/rexlego/src/gpu_native/portrait_probe.h
index e15681f..4d46d1e 100644
--- a/rexlego/src/gpu_native/portrait_probe.h
+++ b/rexlego/src/gpu_native/portrait_probe.h
@@ -1,5 +1,5 @@
 // Read-only TU23 portrait identity. Offsets come from 833738B8, 82CC4E50
-// and the texture leaf 82B6FC70; they are not guessed from a HUD draw.
+// and the texture leaf 82B6FC70 / binding function 82BCE7F8.
 #pragma once
 #include <cstdint>
 #include <limits>
@@ -10,6 +10,8 @@ struct PortraitIdentity {
   uint32_t scene = 0;
   uint32_t material = 0;
   uint32_t texture_object = 0;
+  uint32_t texture_backend = 0;
+  uint32_t active_texture_index = 0;
   uint32_t texture = 0;
   uint32_t material_flags = 0;
   uint32_t texture_flags = 0;
@@ -32,7 +34,15 @@ PortraitIdentity ReadPortraitIdentity(uint32_t owner, ReadWord&& read) {
   field(result.material, 948, result.material_flags, 4);
   field(result.material, 952, result.texture_flags, 8);
   if (!field(result.material, 956, result.texture_object, 16)) return result;
-  field(result.texture_object, 4, result.texture, 32);
+  if (!field(result.texture_object, 4, result.texture_backend, 32) ||
+      !field(result.texture_backend, 116, result.active_texture_index, 64)) return result;
+  // 82BCE7F8 passes the selected inline D3D header, NOT the backend pointer,
+  // to SetTexture: backend + 120 + index * 52. Reject wrap/unreadable headers.
+  const uint64_t header = uint64_t(result.texture_backend) + 120 +
+      uint64_t(result.active_texture_index) * 52;
+  uint32_t ignored = 0;
+  if (header + 4 <= (uint64_t{1} << 32) &&
+      field(uint32_t(header), 0, ignored, 128)) result.texture = uint32_t(header);
   return result;
 }
 
diff --git a/tools/gpu-aot/native-validation-pending-20261003.md b/tools/gpu-aot/native-validation-pending-20261003.md
index 6bfb73c..988d16d 100644
--- a/tools/gpu-aot/native-validation-pending-20261003.md
+++ b/tools/gpu-aot/native-validation-pending-20261003.md
@@ -18,7 +18,7 @@ Once launches are authorized:
    Check head, body, hands, HUD icons, portrait, effects, shadows and portals.
    Preserve the override material/texture identity if the portrait remains blank.
    October4: use -LongProbe -PortraitTrace to correlate the actual mask's
-   NuTexture+4 D3D resource with draw shader/texture/constant captures. See
+   NuTexture/backend/active D3D header with draw shader/texture/constant captures. See
    native-work-20261004.md. Keep this separate from clean FPS measurements.
 3. Audit buffer watches in hub and three story levels, loading/cutscenes and
    mesh-pool relocations. Require zero stale-clean errors. Until then buffer
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index f6966ea..42788a2 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -19,8 +19,11 @@ material override. Further inspection of the actual TU23 generated bodies:
 - 82CC4E50 copies the wrapper's pointee into material+956 and retains/releases
   through the object's vtable. Material+956 is therefore a NuTexture pointer,
   not directly a D3D texture header.
-- 82B6FC70 is exactly a load from texture object+4. This gives the D3D resource
-  identity to compare with native texture bindings.
+- 82B6FC70 is exactly a load from texture object+4, giving the backend object.
+  Further inspection of binding function 82BCE7F8 shows that this is **not** the
+  D3D header: SetTexture receives backend+120+52*index, with index at backend+116.
+  The diagnostic checks the active inline header's readability and arithmetic
+  bounds before comparing that address with native draw bindings.
 - 83373850 destroys this owner. The diagnostic removes it before destruction.
 
 New C++ hooks retain the original setup/destructor behavior. With
diff --git a/tools/gpu-aot/test_portrait_identity.cpp b/tools/gpu-aot/test_portrait_identity.cpp
index 2d2e071..8306ca4 100644
--- a/tools/gpu-aot/test_portrait_identity.cpp
+++ b/tools/gpu-aot/test_portrait_identity.cpp
@@ -8,7 +8,9 @@ int main() {
   std::unordered_map<uint32_t,uint32_t> words{
       {0x1000+84,0x2000},{0x1000+200,0x3000},
       {0x3000+948,0x12345678},{0x3000+952,0x87654321},
-      {0x3000+956,0x4000},{0x4000+4,0x5000}};
+      {0x3000+956,0x4000},{0x4000+4,0x5000},
+      {0x5000+116,1},{0x5000+120+52,0x10001},
+      {0x6000+116,0},{0x6000+120,0x10001}};
   const auto original=words;
   std::vector<uint32_t> reads;
   auto read=[&](uint32_t address,uint32_t& out) {
@@ -17,12 +19,15 @@ int main() {
     out=it->second;return true;
   };
   auto identity=ReadPortraitIdentity(0x1000,read);
-  assert(identity.readable_fields==63 && identity.texture==0x5000);
+  assert(identity.readable_fields==255 && identity.texture==0x5000+120+52);
   assert(identity.scene==0x2000 && identity.material==0x3000 &&
          identity.texture_object==0x4000 && identity.material_flags==0x12345678);
   assert(words==original);
   words[0x4000+4]=0x6000;
-  assert(ReadPortraitIdentity(0x1000,read).texture==0x6000);
+  assert(ReadPortraitIdentity(0x1000,read).texture==0x6000+120);
+  words[0x6000+116]=0xFFFFFFFF;
+  assert(ReadPortraitIdentity(0x1000,read).texture==0);
+  words[0x6000+116]=0;
   words.erase(0x4000+4);
   assert(!(ReadPortraitIdentity(0x1000,read).readable_fields&32));
   words[0x3000+956]=0;
diff --git a/tools/gpu-aot/test_portrait_probe.py b/tools/gpu-aot/test_portrait_probe.py
new file mode 100644
index 0000000..f096aae
--- /dev/null
+++ b/tools/gpu-aot/test_portrait_probe.py
@@ -0,0 +1,96 @@
+"""Compile actual portrait hook bodies with a bounded fake guest-memory SDK."""
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+
+p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args()
+a.output.mkdir(parents=True,exist_ok=True)
+root=Path(__file__).resolve().parents[2];inc=a.output/'include/rex'
+(inc/'system').mkdir(parents=True,exist_ok=True)
+(inc/'types.h').write_text('''#pragma once
+#include <cstdint>
+using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;
+struct Register {uint32_t u32=0;};
+struct PPCContext {Register r3;uint64_t result=0,canary=0x1122334455667788ull;};
+''')
+(inc/'hook.h').write_text('''#pragma once
+#include <rex/types.h>
+#define REX_FUNC(name) void name(PPCContext& ctx,u8* base)
+''')
+(inc/'logging.h').write_text('''#pragma once
+#define REXLOG_INFO(...) ((void)0)
+#define REXLOG_ERROR(...) ((void)0)
+''')
+(inc/'cvar.h').write_text('''#pragma once
+#define REXCVAR_DECLARE(type,name) type& FLAGS_##name##_storage_()
+#define REXCVAR_GET(name) FLAGS_##name##_storage_()
+''')
+(inc/'system/kernel_state.h').write_text('''#pragma once
+#include <rex/system/xmemory.h>
+#define REX_KERNEL_MEMORY() (&fixture_memory)
+''')
+(inc/'system/xmemory.h').write_text('''#pragma once
+#include <rex/types.h>
+#include <array>
+struct Heap {int reads=0;bool IsRangeCommittedReadable(u32 at,u32 size){++reads;return uint64_t(at)+size<=65536;}};
+struct Memory {Heap heap;std::array<u8,65536> bytes{};
+ Heap* LookupHeap(u32){return &heap;}
+ template<class T>T TranslateVirtual(u32 at){return reinterpret_cast<T>(bytes.data()+at);}
+};
+inline Memory fixture_memory;
+''')
+cpp=a.output/'probe-test.cpp'
+cpp.write_text(r'''
+#include <rex/system/xmemory.h>
+#include "gpu_native/portrait_probe.h"
+#include <cassert>
+#include <cstring>
+bool& FLAGS_gpu_native_pm4_storage_(){static bool selected=false;return selected;}
+extern "C" void sub_833738B8(PPCContext&,u8*);
+extern "C" void sub_83373850(PPCContext&,u8*);
+int setups=0,destroys=0;
+extern "C" void __imp__sub_833738B8(PPCContext& ctx,u8*){++setups;ctx.r3.u32=0xCAFE;ctx.result=42;}
+extern "C" void __imp__sub_83373850(PPCContext& ctx,u8*){++destroys;ctx.r3.u32=0xBEEF;ctx.result=43;}
+void store(u32 at,u32 value){value=__builtin_bswap32(value);std::memcpy(fixture_memory.bytes.data()+at,&value,4);}
+int main(int argc,char**){
+ using namespace legodimensions::gpu_native;
+ const bool enabled=argc>1;
+ store(0x1000+84,0x2000);store(0x1000+200,0x3000);
+ store(0x3000+948,0x1234);store(0x3000+952,0x5678);store(0x3000+956,0x4000);
+ store(0x4000+4,0x5000);store(0x5000+116,1);store(0x5000+120+52,0x10001);
+ auto before=fixture_memory.bytes;PPCContext ctx;ctx.r3.u32=0x1000;
+ sub_833738B8(ctx,fixture_memory.bytes.data());
+ assert(setups==1 && ctx.r3.u32==0xCAFE && ctx.result==42 && ctx.canary==0x1122334455667788ull);
+ assert(fixture_memory.bytes==before);
+ assert(PortraitProbeEnabled()==enabled);
+ TracePortraitDraw(0x5000,0,123,456); // Backend pointer must not match the D3D header.
+ TracePortraitDraw(0x5000+120+52,0,123,456);
+ ctx.r3.u32=0x1000;sub_83373850(ctx,fixture_memory.bytes.data());
+ assert(destroys==1 && ctx.r3.u32==0xBEEF && ctx.result==43 && fixture_memory.bytes==before);
+ const auto reads=fixture_memory.heap.reads;
+ TracePortraitDraw(0x5000+120+52,0,123,456);assert(fixture_memory.heap.reads==reads);
+ if(!enabled)assert(reads==0);
+}
+''')
+exe=a.output/'portrait-probe-test.exe'
+subprocess.run(['clang++','-std=c++20','-D_CRT_SECURE_NO_WARNINGS','-I'+str(a.output/'include'),
+ '-I'+str(root/'rexlego/src'),str(root/'rexlego/src/gpu_native/hooks_portrait.cpp'),str(cpp),'-o',str(exe)],check=True)
+for name,trace,reference,enabled in [('disabled',None,'0',False),('zero','0','0',False),
+                                  ('enabled','1','0',True),('reference','1','1',False)]:
+ env=dict(os.environ);directory=a.output/name;directory.mkdir(exist_ok=True)
+ events=directory/'events.tsv'
+ if events.exists():events.unlink()
+ env['LEGO_NATIVE_TRACE_DIR']=str(directory.resolve());env['LEGO_NATIVE_PM4_REFERENCE']=reference
+ env.pop('LEGO_NATIVE_PORTRAIT_TRACE',None)
+ if trace is not None:env['LEGO_NATIVE_PORTRAIT_TRACE']=trace
+ # The reference bypasses hooks; the standalone enable predicate still sees
+ # trace settings. Leave its flag off here; routing itself is covered separately.
+ if reference=='1':env.pop('LEGO_NATIVE_PORTRAIT_TRACE',None)
+ subprocess.run([str(exe.resolve())]+(['enabled'] if enabled else []),env=env,check=True)
+ if enabled:
+  text=events.read_text();assert text.count('\tportrait_setup\t')==1
+  assert text.count('\tportrait_texture_draw\t')==1 and text.count('\tportrait_destroy\t')==1
+ else:assert not events.exists()
+print('PASS: actual portrait hooks, original register/results preserved, no guest writes, exact header match, destructor removal, disabled reads=0')

From 5492353a603187402de704c56780462ad01c2b06 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 11:58:44 +0300
Subject: [PATCH 33/44] Use monitor three for authorized native probes

---
 tools/gpu-aot/native-validation-pending-20261003.md | 6 ++++--
 tools/gpu-aot/native-work-20261004.md               | 5 +++--
 tools/gpu-aot/run_native_probe.ps1                  | 2 +-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/tools/gpu-aot/native-validation-pending-20261003.md b/tools/gpu-aot/native-validation-pending-20261003.md
index 988d16d..2ba598d 100644
--- a/tools/gpu-aot/native-validation-pending-20261003.md
+++ b/tools/gpu-aot/native-validation-pending-20261003.md
@@ -1,7 +1,9 @@
 # Required runtime validation after October 3 offline fixes
 
-Do not launch the game until the user authorizes it. All subsequent tests use
-monitor2, no window activation or injected input. Keep the exact executable,
+October4: the owner authorized resuming game tests and requested monitor3 for
+all current probes, superseding the earlier no-launch/monitor2 instruction.
+Use questions for user-assisted hub readiness; no activation or injected input.
+Keep the exact executable,
 SDK/runtime/plugin hashes and configuration with every probe; close the exact
 owned process afterward. See `session-20261003/build-proof.json` for the build.
 
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index 42788a2..c77250d 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -3,8 +3,9 @@
 The owner changed the target to **stable 60 FPS at actual 1920x1080 internal
 rendering**, retaining C++. Another agent is working on ReXGlue Vulkan fixes.
 This work modifies the game-side D3D12 renderer only; it does not rebuild,
-deploy or modify the SDK or Plume. Game launches remain pending explicit
-authorization, and future probes default to monitor 2.
+deploy or modify the SDK or Plume. The owner subsequently authorized game tests
+on monitor 3. Probes now default to monitor 3, superseding the initial monitor 2
+and no-launch instructions. Ask through the question UI for hub readiness.
 
 The owner confirms the Enter yellow transition glitch no longer appears.
 This is user-confirmed visual evidence, not a new assistant capture.
diff --git a/tools/gpu-aot/run_native_probe.ps1 b/tools/gpu-aot/run_native_probe.ps1
index ef16498..f356aad 100644
--- a/tools/gpu-aot/run_native_probe.ps1
+++ b/tools/gpu-aot/run_native_probe.ps1
@@ -22,7 +22,7 @@ param(
     [switch]$Pm4Reference,
     [switch]$ConfiguredRenderer,
     [string]$LogoCaptureRenderDocDll,
-    [ValidateRange(1,8)][int]$Monitor = 2,
+    [ValidateRange(1,8)][int]$Monitor = 3,
     [switch]$Close
 )
 $ErrorActionPreference = 'Stop'

From 98c2f3632f708296c8b35b48b830b565f4dd9bf9 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 12:07:58 +0300
Subject: [PATCH 34/44] Correct portrait backend from actual asset binding path

---
 rexlego/src/gpu_native/portrait_probe.h  |  6 ++-
 tools/gpu-aot/native-work-20261004.md    | 23 +++++++---
 tools/gpu-aot/read_guest_memory.cpp      | 57 ++++++++++++++++++++++++
 tools/gpu-aot/test_portrait_identity.cpp |  8 ++--
 tools/gpu-aot/test_portrait_probe.py     |  2 +-
 5 files changed, 85 insertions(+), 11 deletions(-)
 create mode 100644 tools/gpu-aot/read_guest_memory.cpp

diff --git a/rexlego/src/gpu_native/portrait_probe.h b/rexlego/src/gpu_native/portrait_probe.h
index 4d46d1e..80bc18d 100644
--- a/rexlego/src/gpu_native/portrait_probe.h
+++ b/rexlego/src/gpu_native/portrait_probe.h
@@ -1,5 +1,5 @@
 // Read-only TU23 portrait identity. Offsets come from 833738B8, 82CC4E50
-// and the texture leaf 82B6FC70 / binding function 82BCE7F8.
+// and binding functions 82BCBB78 / 82BCE7F8. Object+4 is a reference count.
 #pragma once
 #include <cstdint>
 #include <limits>
@@ -34,7 +34,9 @@ PortraitIdentity ReadPortraitIdentity(uint32_t owner, ReadWord&& read) {
   field(result.material, 948, result.material_flags, 4);
   field(result.material, 952, result.texture_flags, 8);
   if (!field(result.material, 956, result.texture_object, 16)) return result;
-  if (!field(result.texture_object, 4, result.texture_backend, 32) ||
+  // 82BCBB78 loads asset+44 into r30, then passes r30 as r6 to
+  // 82BCE7F8. The asset+40 member instead supplies sampler parameters.
+  if (!field(result.texture_object, 44, result.texture_backend, 32) ||
       !field(result.texture_backend, 116, result.active_texture_index, 64)) return result;
   // 82BCE7F8 passes the selected inline D3D header, NOT the backend pointer,
   // to SetTexture: backend + 120 + index * 52. Reject wrap/unreadable headers.
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index c77250d..0d7754e 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -20,11 +20,14 @@ material override. Further inspection of the actual TU23 generated bodies:
 - 82CC4E50 copies the wrapper's pointee into material+956 and retains/releases
   through the object's vtable. Material+956 is therefore a NuTexture pointer,
   not directly a D3D texture header.
-- 82B6FC70 is exactly a load from texture object+4, giving the backend object.
-  Further inspection of binding function 82BCE7F8 shows that this is **not** the
-  D3D header: SetTexture receives backend+120+52*index, with index at backend+116.
-  The diagnostic checks the active inline header's readability and arithmetic
-  bounds before comparing that address with native draw bindings.
+- The first live Sonic trace disproved the earlier backend inference: asset+4
+  is the reference count (3), and 82B6FC70 is a generic reference-count getter.
+  Read-only memory identifies the asset as sonic_portraitalpha_nxg.tex:1.
+- Actual binder 82BCBB78 loads asset+44 into r30 and passes r30 as r6 to
+  82BCE7F8. Asset+40 supplies sampler parameters. 82BCE7F8 selects the inline
+  D3D header at backend+120+52*index, with index at backend+116. The corrected
+  probe follows this statically verified binder path; a live matching draw is
+  still required.
 - 83373850 destroys this owner. The diagnostic removes it before destruction.
 
 New C++ hooks retain the original setup/destructor behavior. With
@@ -77,3 +80,13 @@ override was enabled.
 Next authorized runs: a matching Vorton portrait capture, clean warmed baseline,
 buffer-watch audit, then controlled OFF/ON comparisons. Actual 1080p, three story
 levels, level-completion spikes and AMD remain open validation gates.
+
+## First Sonic capture and added world coverage
+
+The October 4 Sonic capture shows the head and HUD icons present, with an empty
+portrait circle. It remains 1280x720 internally. This heavily instrumented run
+is not an FPS baseline. The probe was closed after the capture and read-only
+object inspection. The owner also reports mostly black surfaces in the upper
+Vorton portal layer and Lord of the Rings world, though some textures appear
+correct and the world loads. Include both in correctness coverage; diagnose
+shader/resource failures before applying texture color adjustments.
diff --git a/tools/gpu-aot/read_guest_memory.cpp b/tools/gpu-aot/read_guest_memory.cpp
new file mode 100644
index 0000000..19de9cb
--- /dev/null
+++ b/tools/gpu-aot/read_guest_memory.cpp
@@ -0,0 +1,57 @@
+// Read-only, bounded live guest-object snapshots for renderer diagnostics.
+// No writes, suspension, input, window activation or remote execution.
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <windows.h>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <array>
+#include <string>
+
+int wmain(int argc, wchar_t** argv) {
+  if (argc < 6) {
+    std::cerr << "usage: reader PID EXPECTED_EXE HOST_GUEST_BASE OUTPUT_DIR GUEST_ADDRESS...\n";
+    return 2;
+  }
+  try {
+    const auto pid = std::stoull(argv[1], nullptr, 0);
+    const auto base = std::stoull(argv[3], nullptr, 0);
+    if (!pid || pid > MAXDWORD || argc > 37) return 2;
+    HANDLE process = OpenProcess(PROCESS_VM_READ | PROCESS_QUERY_LIMITED_INFORMATION,
+                                 FALSE, DWORD(pid));
+    if (!process) return 3;
+    struct Close { HANDLE handle; ~Close() { CloseHandle(handle); } } close{process};
+    std::array<wchar_t, 32768> path{};
+    DWORD count = DWORD(path.size());
+    if (!QueryFullProcessImageNameW(process, 0, path.data(), &count) ||
+        !std::filesystem::equivalent(path.data(), argv[2])) {
+      std::cerr << "Process path mismatch\n"; return 4;
+    }
+    std::filesystem::create_directories(argv[4]);
+    bool failed = false;
+    for (int i = 5; i < argc; ++i) {
+      const auto address = std::stoull(argv[i], nullptr, 0);
+      constexpr size_t length = 1024;
+      if (!address || address > 0x100000000ull - length ||
+          base > UINT64_MAX - address - length) return 2;
+      std::array<char, length> bytes{};
+      SIZE_T read = 0;
+      const bool complete = ReadProcessMemory(process,
+          reinterpret_cast<const void*>(base + address), bytes.data(), bytes.size(), &read);
+      if (!complete || read != length) {
+        std::cerr << "Read failed guest=" << std::hex << address << " bytes=" << read << '\n';
+        failed = true; continue;
+      }
+      const auto file = std::filesystem::path(argv[4]) /
+          (L"guest-" + std::to_wstring(address) + L".bin");
+      std::ofstream output(file, std::ios::binary);
+      output.write(bytes.data(), bytes.size());
+      if (!output) { failed = true; continue; }
+      std::cout << "Saved guest=" << std::hex << address << " bytes=" << std::dec << read << '\n';
+    }
+    return failed ? 5 : 0;
+  } catch (const std::exception& error) {
+    std::cerr << error.what() << '\n'; return 2;
+  }
+}
diff --git a/tools/gpu-aot/test_portrait_identity.cpp b/tools/gpu-aot/test_portrait_identity.cpp
index 8306ca4..d87cd05 100644
--- a/tools/gpu-aot/test_portrait_identity.cpp
+++ b/tools/gpu-aot/test_portrait_identity.cpp
@@ -8,7 +8,8 @@ int main() {
   std::unordered_map<uint32_t,uint32_t> words{
       {0x1000+84,0x2000},{0x1000+200,0x3000},
       {0x3000+948,0x12345678},{0x3000+952,0x87654321},
-      {0x3000+956,0x4000},{0x4000+4,0x5000},
+      {0x4000+4,3}, // Captured Sonic object reference count, never a pointer.
+      {0x3000+956,0x4000},{0x4000+44,0x5000},
       {0x5000+116,1},{0x5000+120+52,0x10001},
       {0x6000+116,0},{0x6000+120,0x10001}};
   const auto original=words;
@@ -23,12 +24,13 @@ int main() {
   assert(identity.scene==0x2000 && identity.material==0x3000 &&
          identity.texture_object==0x4000 && identity.material_flags==0x12345678);
   assert(words==original);
-  words[0x4000+4]=0x6000;
+  for(auto address:reads) assert(address!=0x4000+4 && address!=3+116);
+  words[0x4000+44]=0x6000;
   assert(ReadPortraitIdentity(0x1000,read).texture==0x6000+120);
   words[0x6000+116]=0xFFFFFFFF;
   assert(ReadPortraitIdentity(0x1000,read).texture==0);
   words[0x6000+116]=0;
-  words.erase(0x4000+4);
+  words.erase(0x4000+44);
   assert(!(ReadPortraitIdentity(0x1000,read).readable_fields&32));
   words[0x3000+956]=0;
   assert(ReadPortraitIdentity(0x1000,read).texture==0);
diff --git a/tools/gpu-aot/test_portrait_probe.py b/tools/gpu-aot/test_portrait_probe.py
index f096aae..d6f131a 100644
--- a/tools/gpu-aot/test_portrait_probe.py
+++ b/tools/gpu-aot/test_portrait_probe.py
@@ -59,7 +59,7 @@
  const bool enabled=argc>1;
  store(0x1000+84,0x2000);store(0x1000+200,0x3000);
  store(0x3000+948,0x1234);store(0x3000+952,0x5678);store(0x3000+956,0x4000);
- store(0x4000+4,0x5000);store(0x5000+116,1);store(0x5000+120+52,0x10001);
+ store(0x4000+4,3);store(0x4000+44,0x5000);store(0x5000+116,1);store(0x5000+120+52,0x10001);
  auto before=fixture_memory.bytes;PPCContext ctx;ctx.r3.u32=0x1000;
  sub_833738B8(ctx,fixture_memory.bytes.data());
  assert(setups==1 && ctx.r3.u32==0xCAFE && ctx.result==42 && ctx.canary==0x1122334455667788ull);

From d16069dc6cc36fa38009344da56d858150eac21b Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 12:19:33 +0300
Subject: [PATCH 35/44] Trace rejected portrait draws and document expanded
 world shader coverage

---
 rexlego/src/gpu_native/draw.cpp           | 11 +++++--
 rexlego/src/gpu_native/hooks_portrait.cpp |  2 +-
 tools/gpu-aot/native-work-20261004.md     | 38 +++++++++++++++++++++--
 tools/gpu-aot/test_portrait_probe.py      |  2 +-
 4 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
index 21b2270..140a351 100644
--- a/rexlego/src/gpu_native/draw.cpp
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -293,9 +293,6 @@ bool BindConstants(
     // in invalidation, verify their contents at every use, even with watches.
     UploadTextureResource(bindings.textures[i], commands,
         (BoundShaderTextureMask(ShaderStage::kVertex) & (1u << i)) != 0);
-    if (PortraitProbeEnabled())
-      TracePortraitDraw(bindings.textures[i], i, BoundShaderHash(ShaderStage::kVertex),
-                        BoundShaderHash(ShaderStage::kPixel));
     TextureFetchWords fetch;
     for (u32 word = 0; word < fetch.size(); ++word) fetch[word] = device->fetch_constants[i].dword[word];
     if (LongProbeEnabled()) {
@@ -586,6 +583,14 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
   if (!device || !count) {
     return false;
   }
+  // Trace attempted bindings before shader/declaration/pipeline early exits.
+  // A mask on a rejected draw is diagnostic evidence, not a submitted draw.
+  if (PortraitProbeEnabled()) {
+    const auto portrait_bindings = SnapshotDrawBindings();
+    for (u32 i = 0; i < kNativeTextureSlots; ++i)
+      TracePortraitDraw(portrait_bindings.textures[i], i,
+          BoundShaderHash(ShaderStage::kVertex), BoundShaderHash(ShaderStage::kPixel));
+  }
   if (g_logo_capture_api && indexed && count == 6 && start == 24 && base_vertex == 16 &&
       BoundShaderHash(ShaderStage::kVertex) == 0x91007ACB3E640E3Dull &&
       BoundShaderHash(ShaderStage::kPixel) == 0xEEE573BE160E037Dull &&
diff --git a/rexlego/src/gpu_native/hooks_portrait.cpp b/rexlego/src/gpu_native/hooks_portrait.cpp
index 58d86c6..0374ea0 100644
--- a/rexlego/src/gpu_native/hooks_portrait.cpp
+++ b/rexlego/src/gpu_native/hooks_portrait.cpp
@@ -76,7 +76,7 @@ void TracePortraitDraw(u32 texture, u32 slot, u64 vertex_shader, u64 pixel_shade
     const auto identity = ReadPortraitIdentity(owner, ReadPortraitWord);
     if (identity.texture != texture || !(identity.readable_fields & 128)) continue;
     ++reports;
-    LongProbeEvent("portrait_texture_draw", false, "owner=", owner,
+    LongProbeEvent("portrait_bound_draw", false, "owner=", owner,
         "material=", identity.material, "texture_object=", identity.texture_object,
         "texture_backend=", identity.texture_backend, "active_index=", identity.active_texture_index,
         "texture=", texture, "slot=", slot, "VS=", vertex_shader, "PS=", pixel_shader,
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index 0d7754e..054631f 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -32,8 +32,8 @@ material override. Further inspection of the actual TU23 generated bodies:
 
 New C++ hooks retain the original setup/destructor behavior. With
 LEGO_NATIVE_PORTRAIT_TRACE=1 and LEGO_NATIVE_TRACE_DIR set, they log
-portrait_setup / portrait_destroy. BindConstants reports portrait_texture_draw
-when the current owner/material/NuTexture chain matches a referenced texture
+portrait_setup / portrait_destroy. DispatchDraw reports portrait_bound_draw
+when the current owner/material/NuTexture chain matches an attempted texture
 slot, including VS/PS hashes. Re-reading the chain avoids keeping stale setup
 identities; reads check commitment and 32-bit bounds. At most 32 owners are
 tracked and 64 matched draws reported per frame. Disabled mode performs no
@@ -90,3 +90,37 @@ object inspection. The owner also reports mostly black surfaces in the upper
 Vorton portal layer and Lord of the Rings world, though some textures appear
 correct and the world loads. Include both in correctness coverage; diagnose
 shader/resource failures before applying texture color adjustments.
+
+## Upper hub / LOTR coverage candidate, and latest launch constraint
+
+The owner reports every character and vehicle portrait missing, plus world
+textures; diagnose a shared path. The second Sonic trace follows asset+44 to
+a readable backend and active inline header. No successful BindConstants draw
+matched its mask. This does not prove no attempted draw: the original trace ran
+after pipeline creation. The revised trace now observes attempted bindings
+before shader, declaration and pipeline rejection, including unused slots;
+portrait_bound_draw is explicitly not evidence of GPU submission.
+
+The owner moved to the upper Vorton portal layer and LOTR world in the same
+probe. Captures saved locally as upper-hub-before.png and the LOTR present DDS.
+Large black surfaces are visible in the upper-hub image. The run captured 75
+new valid containers, merged with all 175 existing runtime inputs. The current
+624-byte alpha/viewport compiler successfully compiled 250/250. The merged
+index preserves every previous record, 18836 -> 18983 (147 new signatures).
+All 250 inputs pass exact physical/instruction lookup with the main bank present.
+This addresses observed shader coverage failures; visual correction remains
+unverified. Original mask missingness can have another cause.
+
+The diagnostic process 12764 was closed. The owner subsequently prohibited new
+launches until explicitly requested; do not launch validation automatically.
+A separate user-owned hotfix/test process 8696 was observed and left untouched.
+The dev build can be prepared offline; SDK/Vulkan/Plume and hotfix stay untouched.
+
+Installed offline dev build: runtime 250 containers / 448 prelinked variants,
+index 18983, fingerprint 5a3f2a542b8519f7. All previous shader hashes and
+specialization masks preserved, all specialization subsets present. Executable
+contains the matching fingerprint and portrait_bound_draw marker; runtime DLL,
+GPU plugin and saved configuration hashes match the prior verified build.
+Proof: session-20261004/world-bank-candidate/installed-build-proof.json.
+The build has not been launched. Current internal targets are still 1280x720;
+native 1920x1080 and stable 60 FPS remain unfulfilled gates.
diff --git a/tools/gpu-aot/test_portrait_probe.py b/tools/gpu-aot/test_portrait_probe.py
index d6f131a..c5ac064 100644
--- a/tools/gpu-aot/test_portrait_probe.py
+++ b/tools/gpu-aot/test_portrait_probe.py
@@ -91,6 +91,6 @@
  subprocess.run([str(exe.resolve())]+(['enabled'] if enabled else []),env=env,check=True)
  if enabled:
   text=events.read_text();assert text.count('\tportrait_setup\t')==1
-  assert text.count('\tportrait_texture_draw\t')==1 and text.count('\tportrait_destroy\t')==1
+  assert text.count('\tportrait_bound_draw\t')==1 and text.count('\tportrait_destroy\t')==1
  else:assert not events.exists()
 print('PASS: actual portrait hooks, original register/results preserved, no guest writes, exact header match, destructor removal, disabled reads=0')

From 1e56230dd0c054c8d0978731c6d2d3dc661a7eb2 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 12:50:17 +0300
Subject: [PATCH 36/44] Handle Alt Enter borderless toggle in game app

---
 rexlego/CMakeLists.txt                |  1 +
 rexlego/src/legodimensions_app.h      | 17 ++++++++
 rexlego/src/window_mode.h             | 24 ++++++++++
 tools/gpu-aot/native-work-20261004.md | 27 ++++++++++++
 tools/gpu-aot/test_window_mode.cpp    | 30 +++++++++++++
 tools/gpu-aot/test_window_mode.py     | 63 +++++++++++++++++++++++++++
 6 files changed, 162 insertions(+)
 create mode 100644 rexlego/src/window_mode.h
 create mode 100644 tools/gpu-aot/test_window_mode.cpp
 create mode 100644 tools/gpu-aot/test_window_mode.py

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index 6bc96a1..a649a52 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -271,6 +271,7 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderer_mode.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderdoc_probe.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/legodimensions_app.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/window_mode.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/long_probe.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/memory_watch.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/memory_watch.cpp"
diff --git a/rexlego/src/legodimensions_app.h b/rexlego/src/legodimensions_app.h
index 41f3d1a..3cdb0f0 100644
--- a/rexlego/src/legodimensions_app.h
+++ b/rexlego/src/legodimensions_app.h
@@ -31,6 +31,7 @@
 #include "toypad_app.h"
 #include "ui_theme.h"
 #include "updates.h"
+#include "window_mode.h"
 #ifdef LEGODIMENSIONS_NATIVE_GPU
 #include "gpu_native/device.h"
 #include "gpu_native/renderer_mode.h"
@@ -178,7 +179,23 @@ class LegodimensionsApp : public rex::ReXApp {
     main_menu_ = legodimensions::main_menu::CreateOverlay(drawer, std::move(host));
   }
 
+  // Handle the convention in the title even with an older deployed SDK bind.
+  // Consume repeats so holding the chord cannot resize every keyboard repeat.
+  void OnKeyDown(rex::ui::KeyEvent& e) override {
+    if (e.virtual_key() == rex::ui::VirtualKey::kReturn && e.is_alt_pressed() &&
+        !e.is_ctrl_pressed() && !e.is_shift_pressed() && !e.is_super_pressed()) {
+      if (!e.prev_state() && window()) {
+        const bool fullscreen = legodimensions::ToggleBorderless(*window(), windowed_size_);
+        rex::cvar::SetFlagByName("fullscreen", fullscreen ? "true" : "false");
+      }
+      e.set_handled(true);
+      return;
+    }
+    rex::ui::ProcessKeyEvent(e);
+  }
+
  private:
+  legodimensions::WindowedSize windowed_size_;
   // Last-resort exit if the close path wedges. Detached on purpose: if the
   // process gets where it is going first, _Exit takes this thread with it.
   static void StartShutdownWatchdog() {
diff --git a/rexlego/src/window_mode.h b/rexlego/src/window_mode.h
new file mode 100644
index 0000000..c658e8f
--- /dev/null
+++ b/rexlego/src/window_mode.h
@@ -0,0 +1,24 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions {
+struct WindowedSize { uint32_t width = 0, height = 0; };
+
+// SDL's SetFullscreen uses the window's current display and borderless mode.
+// Remember the actual windowed size, rather than the fullscreen display size.
+template<class Window>
+bool ToggleBorderless(Window& window, WindowedSize& saved) {
+  const bool fullscreen = !window.IsFullscreen();
+  if (fullscreen) {
+    const auto width = window.GetActualLogicalWidth();
+    const auto height = window.GetActualLogicalHeight();
+    if (width && height) saved = {width, height};
+    window.SetFullscreen(true);
+  } else {
+    window.SetFullscreen(false);
+    if (saved.width && saved.height)
+      window.SetDesiredLogicalSize(saved.width, saved.height);
+  }
+  return fullscreen;
+}
+}  // namespace legodimensions
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index 054631f..927efb9 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -124,3 +124,30 @@ GPU plugin and saved configuration hashes match the prior verified build.
 Proof: session-20261004/world-bank-candidate/installed-build-proof.json.
 The build has not been launched. Current internal targets are still 1280x720;
 native 1920x1080 and stable 60 FPS remain unfulfilled gates.
+
+## Owner test after bank250 and offline follow-up
+
+Owner confirms the world-transition tube and upper Vorton are fixed; LOTR is
+partly improved. Character/vehicle portraits remain blank, gamepad button
+prompts are missing, black objects remain, and textures show repeated lines.
+A computer-use window capture saved as lotr-owner-test-window.png shows the
+lines and black gaps. The title process29716 was closed and subsequent process
+inspection found no game. No new launch is authorized.
+
+The Oct4 12:3x slice of the manual log still has one unknown pixel resource,
+A863A580 with backing ED5CA160 and zero prefix C77B3ABB6F87ACD9. This run was
+manual without missing-shader capture, so the shader's full bytes are unavailable.
+A zero prefix does not establish shader identity. Do not substitute a similar
+shader or attribute every missing UI element to this one without evidence.
+The existing native 2D untile path uses the SDK GetTiledOffset2D; the repeated
+lines are not yet localized to data conversion, UV decoding, or mip sampling.
+
+Game-side Alt+Enter now explicitly handles the chord before SDK binds, consumes
+key repeats, toggles SDL borderless on the current monitor, restores the actual
+windowed logical dimensions, and synchronizes the fullscreen setting. Other
+keys use the SDK's ProcessKeyEvent. It does not modify/rebuild the SDK. Offline
+helper checks cover repeated toggles, window size changes, monitor retention,
+startup fullscreen and zero-size handling; full native build passes. Actual
+window behavior awaits an authorized launch.
+
+Final window-mode build fingerprint b2aae4e91c598df6; proof session-20261004/build-proof-window-mode.json. Actual OnKeyDown handler fixture also passes repeat/modifier/null-window/SDK-bind checks. No game launch; visual renderer failures remain unresolved.
diff --git a/tools/gpu-aot/test_window_mode.cpp b/tools/gpu-aot/test_window_mode.cpp
new file mode 100644
index 0000000..bb7c72d
--- /dev/null
+++ b/tools/gpu-aot/test_window_mode.cpp
@@ -0,0 +1,30 @@
+#include "window_mode.h"
+#include <cassert>
+#include <iostream>
+struct Window {
+  bool fullscreen = false;
+  uint32_t width = 1280, height = 720, monitor = 3, changes = 0;
+  bool IsFullscreen() const { return fullscreen; }
+  uint32_t GetActualLogicalWidth() const { return width; }
+  uint32_t GetActualLogicalHeight() const { return height; }
+  void SetFullscreen(bool value) {
+    fullscreen = value; ++changes;
+    if (value) { width = 1920; height = 1080; }
+  }
+  void SetDesiredLogicalSize(uint32_t w, uint32_t h) { width=w; height=h; }
+};
+int main() {
+  Window w; legodimensions::WindowedSize saved;
+  assert(legodimensions::ToggleBorderless(w,saved));
+  assert(w.fullscreen && w.monitor==3 && saved.width==1280 && saved.height==720);
+  assert(!legodimensions::ToggleBorderless(w,saved));
+  assert(!w.fullscreen && w.width==1280 && w.height==720 && w.monitor==3);
+  w.width=900; w.height=600;
+  legodimensions::ToggleBorderless(w,saved); legodimensions::ToggleBorderless(w,saved);
+  assert(w.width==900 && w.height==600);
+  w.fullscreen=true; saved={};
+  legodimensions::ToggleBorderless(w,saved); assert(!w.fullscreen && w.width==900);
+  w.width=0; w.height=0;
+  legodimensions::ToggleBorderless(w,saved); assert(!saved.width && !saved.height);
+  std::cout << "PASS: borderless toggle, windowed restoration, monitor retention, startup fullscreen, zero-size\n";
+}
diff --git a/tools/gpu-aot/test_window_mode.py b/tools/gpu-aot/test_window_mode.py
new file mode 100644
index 0000000..67d10cf
--- /dev/null
+++ b/tools/gpu-aot/test_window_mode.py
@@ -0,0 +1,63 @@
+"""Exercise the actual app Alt+Enter handler with a fake window/event SDK."""
+import argparse
+from pathlib import Path
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+a.output.mkdir(parents=True, exist_ok=True)
+root = Path(__file__).resolve().parents[2]
+s = (root/'rexlego/src/legodimensions_app.h').read_text()
+start = s.index('  void OnKeyDown(rex::ui::KeyEvent& e) override {')
+i = s.index('{', start)
+depth, end = 1, i+1
+while depth:
+    depth += (s[end] == '{') - (s[end] == '}')
+    end += 1
+handler = s[start:end].replace(' override', '')
+code = r'''
+#include "window_mode.h"
+#include <cassert>
+#include <string>
+#include <iostream>
+int delegated=0, flags=0;
+namespace rex::ui {
+enum class VirtualKey { kReturn,kF4,kA };
+struct KeyEvent {
+ VirtualKey key=VirtualKey::kReturn;bool alt=true,ctrl=false,shift=false,super=false,repeat=false,handled=false;
+ VirtualKey virtual_key()const{return key;}bool is_alt_pressed()const{return alt;}
+ bool is_ctrl_pressed()const{return ctrl;}bool is_shift_pressed()const{return shift;}
+ bool is_super_pressed()const{return super;}bool prev_state()const{return repeat;}
+ void set_handled(bool v){handled=v;}
+};
+void ProcessKeyEvent(KeyEvent&){++delegated;}
+}
+namespace rex::cvar { void SetFlagByName(const char* name,const char* value){assert(std::string(name)=="fullscreen");assert(std::string(value)=="true"||std::string(value)=="false");++flags;} }
+struct Window {
+ bool fs=false;unsigned w=900,h=600,changes=0,monitor=3;
+ bool IsFullscreen(){return fs;}unsigned GetActualLogicalWidth(){return w;}unsigned GetActualLogicalHeight(){return h;}
+ void SetFullscreen(bool v){fs=v;++changes;if(v){w=1920;h=1080;}}
+ void SetDesiredLogicalSize(unsigned width,unsigned height){w=width;h=height;}
+};
+struct App {
+ Window win;bool has_window=true;legodimensions::WindowedSize windowed_size_;
+ Window* window(){return has_window?&win:nullptr;}
+''' + handler + r'''
+};
+int main(){
+ App a;rex::ui::KeyEvent e;
+ a.OnKeyDown(e);assert(e.handled&&a.win.fs&&flags==1&&delegated==0&&a.win.monitor==3);
+ e.repeat=true;a.OnKeyDown(e);assert(a.win.fs&&a.win.changes==1&&flags==1&&delegated==0);
+ e.repeat=false;a.OnKeyDown(e);assert(!a.win.fs&&a.win.w==900&&a.win.h==600&&flags==2);
+ for(int modifier=0;modifier<4;++modifier){e={};if(modifier==0)e.alt=false;if(modifier==1)e.ctrl=true;if(modifier==2)e.shift=true;if(modifier==3)e.super=true;a.OnKeyDown(e);assert(!e.handled&&!a.win.fs);}
+ e={};e.key=rex::ui::VirtualKey::kF4;a.OnKeyDown(e);assert(!e.handled&&delegated==5);
+ a.has_window=false;e={};a.OnKeyDown(e);assert(e.handled&&flags==2&&delegated==5);
+ std::cout<<"PASS: actual Alt+Enter handler, repeats consumed, modifiers respected, SDK binds preserved, null window\n";
+}
+'''
+source = a.output/'window-event-test.cpp'
+source.write_text(code)
+exe = a.output/'window-event-test.exe'
+subprocess.run(['clang++', '-std=c++20', '-I'+str(root/'rexlego/src'), str(source), '-o', str(exe)], check=True)
+subprocess.run([str(exe.resolve())], check=True)

From d95e918d236a0d12bafa6472742169991699d026 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 13:06:32 +0300
Subject: [PATCH 37/44] Integrate explicit mip level translation with matching
 native constants

---
 rexlego/src/gpu_native/draw.cpp            |  8 +++-
 tools/gpu-aot/README.md                    | 21 +++++++++
 tools/gpu-aot/native-work-20261004.md      | 27 +++++++++++
 tools/gpu-aot/prepare_lod_candidate.py     | 32 ++++++++-----
 tools/gpu-aot/test_alpha_test.py           |  7 +--
 tools/gpu-aot/test_lod_shared_constants.py | 53 ++++++++++++++++++++++
 tools/gpu-aot/test_viewport_state.py       |  2 +-
 7 files changed, 132 insertions(+), 18 deletions(-)
 create mode 100644 tools/gpu-aot/test_lod_shared_constants.py

diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
index 140a351..a3997ed 100644
--- a/rexlego/src/gpu_native/draw.cpp
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -72,8 +72,10 @@ struct SharedConstants {
   u32 alpha_function = 7;
   u32 viewport_mode = 0;
   float color_output_scale[4]{1, 1, 1, 1};
+  float fetch_lod_bias[32]{};
 };
-static_assert(sizeof(SharedConstants) == 624);
+static_assert(sizeof(SharedConstants) == 752);
+static_assert(offsetof(SharedConstants, fetch_lod_bias) == 624);
 static_assert(offsetof(SharedConstants, booleans) == 512);
 static_assert(offsetof(SharedConstants, color_output_scale) == 608);
 static_assert(offsetof(SharedConstants, alpha_function) == 600);
@@ -308,6 +310,8 @@ bool BindConstants(
       }
     }
     shared.samplers[i] = HostDevice::RegisterSampler(fetch);
+    const int32_t lod_bias = int32_t((fetch[4] >> 12) & 1023);
+    shared.fetch_lod_bias[i] = float(lod_bias >= 512 ? lod_bias - 1024 : lod_bias) / 32.0f;
     if (trace_constants && i >= 16) {
       struct VertexTextureTrace {
         u32 guest = 0;
@@ -345,7 +349,7 @@ bool BindConstants(
     if (!name.empty()) LongProbeEvent("draw_constants", false, "file=", name,
         "VS=", BoundShaderHash(ShaderStage::kVertex),
         "PS=", BoundShaderHash(ShaderStage::kPixel), "texture_mask=", texture_mask,
-        "VS_offset=0 PS_offset=4096 shared_offset=8192 shared_bytes=624");
+        "VS_offset=0 PS_offset=4096 shared_offset=8192 shared_bytes=752");
   }
   std::memcpy(upload.mapped, data.data(), data.size());
   commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset), 0);
diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index 60eefdf..fb4f479 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -4,6 +4,27 @@ This directory keeps the reproducible, project-owned part of the native D3D12
 shader pipeline. Generated archives, HLSL, DXIL, and build trees stay under
 `rexlego/out/native-gpu` and are intentionally not committed.
 
+## Current compiler and constants ABI (October 4)
+
+The current production compiler is the combined alpha/viewport/explicit-LOD
+source in `session-20261004/lod-combined/source`, prepared reproducibly with:
+
+```powershell
+python tools/gpu-aot/prepare_lod_candidate.py rexlego/out/native-gpu/session-20261004/lod-combined --base-source rexlego/out/native-gpu/session-20261001/viewport-candidate/source
+```
+
+It requires `XENOS_RECOMP_LEGO_NATIVE_SCALE=1` and **752 shared bytes**: the
+existing 624-byte layout plus 32 fetch LOD biases at offset624. Regenerate both
+main and runtime archives and their linked variants together with the matching
+host upload. Older 624-byte compiler commands below are historical examples.
+The original isolated September30 LOD candidate lacks the later alpha/viewport
+fixes and must not replace this combined compiler.
+
+Main coverage remains9290 (known A58 memexport exclusion), runtime250; all prior
+hashes and specialization masks are preserved. This corrects ignored explicit
+mip-level instructions; it does not establish repaired game pixels or FPS.
+No game launch was authorized for this integration. See `native-work-20261004.md`.
+
 ## Inputs
 
 - Read-only game archives from the complete Xbox 360 disc set.
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index 927efb9..6617062 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -151,3 +151,30 @@ startup fullscreen and zero-size handling; full native build passes. Actual
 window behavior awaits an authorized launch.
 
 Final window-mode build fingerprint b2aae4e91c598df6; proof session-20261004/build-proof-window-mode.json. Actual OnKeyDown handler fixture also passes repeat/modifier/null-window/SDK-bind checks. No game launch; visual renderer failures remain unresolved.
+
+## Combined explicit mip-level correction (offline)
+
+Static audit of250 captured runtime containers found70 explicit fetches in46
+shaders and41 SetTextureLod instructions. The installed624-byte compiler ignored
+SetTextureLod and sampled pixel textures with computed derivatives instead.
+Prepared a combined compiler from the exact October1 alpha/viewport source,
+then added the previously tested explicit-LOD translation. This preserves the
+later alpha gate, eight comparisons, fixed16 output scales and viewport code.
+The host upload now appends32 signed fetch biases at offset624, total752 bytes.
+
+Tests pass: seven actual synthetic microcode->HLSL->DXIL cases, predication,
+positive/negative bias, vertex/computed behavior and relative-source refusal;
+32768 actual host bias/slot cases; existing968 alpha compares/five shader cases;
+900 viewport pixel mappings; all actual runtime SetLOD sources and explicit
+fetch bias slots found in generated HLSL. Runtime250/250 and main9290/9291
+compile, with only the already excluded A58 memexport shader failing. All prior
+shader identities/masks retained, no shader coverage removed. Both archives
+and prelinked variants are rebuilt together. Rollbackexe/banks/drawsource in
+session-20261004/lod-combined/before.
+
+This is a concrete missing instruction implementation, not a proven fix for
+the owner's texture lines or missing portraits/button prompts. The remaining
+manual-run shader miss needs a complete captured container; pixel validation
+and clean FPS remain pending. Game stays closed, SDK/Vulkan/Plume/config untouched.
+
+Verified installed combined build: fingerprint 7ccb1c0c3f20bbde, executable SHA256 d31c258f8f798ed4fb8293f789870d854e9879389e11a3683187e4e8f7af0a05. Main9290/18444 and runtime250/448 all prelinked specialization subsets present. DLL pair/config unchanged. No launch. Proof lod-combined/installed-build-proof.json.
diff --git a/tools/gpu-aot/prepare_lod_candidate.py b/tools/gpu-aot/prepare_lod_candidate.py
index f030531..8ad0f12 100644
--- a/tools/gpu-aot/prepare_lod_candidate.py
+++ b/tools/gpu-aot/prepare_lod_candidate.py
@@ -20,10 +20,11 @@ def replace_once(source, old, new):
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('scratch', type=Path)
+    parser.add_argument('--base-source', type=Path, help='Preserve an already validated alpha/viewport compiler source')
     args = parser.parse_args()
     root = Path(__file__).resolve().parents[2]
     checkout = root / 'research/reblue/thirdparty/XenosRecomp'
-    module = checkout / 'XenosRecomp'
+    module = args.base_source.resolve() if args.base_source else checkout / 'XenosRecomp'
     source_dir = args.scratch.resolve() / 'source'
     source_dir.mkdir(parents=True, exist_ok=True)
     manifest = {}
@@ -83,9 +84,11 @@ def main():
         '#define g_ColorOutputScale        vk::RawBufferLoad<float4>(g_PushConstants.SharedConstants + 608)',
         '#define g_ColorOutputScale        vk::RawBufferLoad<float4>(g_PushConstants.SharedConstants + 608)\n'
         '#define g_FetchLodBias(i)          vk::RawBufferLoad<float>(g_PushConstants.SharedConstants + 624 + (i)*4)')
+    color_anchor = ('#define LEGO_SHARED_CONSTANTS() float4 g_ColorOutputScale : packoffset(c38);'
+                    if not args.base_source else '    float4 g_ColorOutputScale : packoffset(c38);')
     common = replace_once(common,
-        '#define LEGO_SHARED_CONSTANTS() float4 g_ColorOutputScale : packoffset(c38);',
-        '#define LEGO_SHARED_CONSTANTS() float4 g_ColorOutputScale : packoffset(c38); \\\n'
+        color_anchor,
+        color_anchor + ' \\\n'
         '    float4 g_FetchLodBiasArr[8] : packoffset(c39);\n'
         '#define g_FetchLodBias(i) (g_FetchLodBiasArr[(i) / 4][(i) % 4])')
     common = replace_once(common, 'float4 tfetchVertex2D(', '''float4 tfetchLevel2D(uint resourceDescriptorIndex, uint samplerDescriptorIndex, float2 texCoord, float2 offset, float lod)
@@ -112,15 +115,20 @@ def main():
     # This ABI change must accompany the candidate shaders at eventual runtime.
     # Keep it as an isolated source copy until the sampler-only A/B is observed.
     draw = (root / 'rexlego/src/gpu_native/draw.cpp').read_text()
-    draw = replace_once(draw, '  float color_output_scale[4]{1, 1, 1, 1};\n',
-                        '  float color_output_scale[4]{1, 1, 1, 1};\n  float fetch_lod_bias[32]{};\n')
-    draw = replace_once(draw, 'static_assert(sizeof(SharedConstants) == 624);',
-                        'static_assert(sizeof(SharedConstants) == 752);\n'
-                        'static_assert(offsetof(SharedConstants, fetch_lod_bias) == 624);')
-    draw = replace_once(draw, '    shared.samplers[i] = HostDevice::RegisterSampler(fetch);',
-                        '    shared.samplers[i] = HostDevice::RegisterSampler(fetch);\n'
-                        '    const int32_t lod_bias = int32_t((fetch[4] >> 12) & 1023);\n'
-                        '    shared.fetch_lod_bias[i] = float(lod_bias >= 512 ? lod_bias - 1024 : lod_bias) / 32.0f;')
+    if 'float fetch_lod_bias[32]' not in draw:
+        draw = replace_once(draw, '  float color_output_scale[4]{1, 1, 1, 1};\n',
+                            '  float color_output_scale[4]{1, 1, 1, 1};\n  float fetch_lod_bias[32]{};\n')
+        draw = replace_once(draw, 'static_assert(sizeof(SharedConstants) == 624);',
+                            'static_assert(sizeof(SharedConstants) == 752);\n'
+                            'static_assert(offsetof(SharedConstants, fetch_lod_bias) == 624);')
+        draw = replace_once(draw, '    shared.samplers[i] = HostDevice::RegisterSampler(fetch);',
+                            '    shared.samplers[i] = HostDevice::RegisterSampler(fetch);\n'
+                            '    const int32_t lod_bias = int32_t((fetch[4] >> 12) & 1023);\n'
+                            '    shared.fetch_lod_bias[i] = float(lod_bias >= 512 ? lod_bias - 1024 : lod_bias) / 32.0f;')
+    else:
+        assert 'sizeof(SharedConstants) == 752' in draw
+        assert 'offsetof(SharedConstants, fetch_lod_bias) == 624' in draw
+    draw = draw.replace('shared_bytes=624', 'shared_bytes=752')
     (source_dir / 'draw-candidate.cpp').write_text(draw)
 
     dep = checkout / 'thirdparty'
diff --git a/tools/gpu-aot/test_alpha_test.py b/tools/gpu-aot/test_alpha_test.py
index e3732e2..8c43d0d 100644
--- a/tools/gpu-aot/test_alpha_test.py
+++ b/tools/gpu-aot/test_alpha_test.py
@@ -59,6 +59,7 @@ def body(source, sig):
                    'bool InitializeRenderDefaults(')
 shared = body((root / 'rexlego/src/gpu_native/draw.cpp').read_text(), 'struct SharedConstants') + ';'
 h = a.output / 'alpha-test.cpp'
+shared_bytes = 752 if 'fetch_lod_bias[32]' in shared else 624
 h.write_text(r'''
 #include <array>
 #include <bit>
@@ -140,12 +141,12 @@ def body(source, sig):
   auto d=DecodeNativeAlphaState(Load(device+10556),std::bit_cast<float>(Load(device+10620)));
   assert(d.enabled&&d.function==f&&d.reference==0.37f);
  }
- static_assert(sizeof(SharedConstants)==624);
+ static_assert(sizeof(SharedConstants)==EXPECTED_SHARED_BYTES);
  static_assert(offsetof(SharedConstants,alpha_function)==600);
  static_assert(offsetof(SharedConstants,alpha_threshold)==556);
  static_assert(offsetof(SharedConstants,color_output_scale)==608);
 }
-''')
+'''.replace('EXPECTED_SHARED_BYTES', str(shared_bytes)))
 exe = a.output / 'alpha-test.exe'
 subprocess.run(['clang++', '-std=c++20', '-DNOMINMAX', '-I'+str(root/'rexlego/src'),
                 '-I'+str(root/'tools/gpu-aot'), str(h), '-o', str(exe)], check=True)
@@ -172,7 +173,7 @@ def body(source, sig):
     assert ('oDepth : SV_Depth' in s) == opts.get('depth',False)
 (a.output/'verification.json').write_text(json.dumps({
     'passed': True, 'actual_tu23_setters': [f'{t[1]:08X}' for t in table],
-    'table_defaults': [f'{t[2]:08X}' for t in table], 'shared_bytes': 624,
+    'table_defaults': [f'{t[2]:08X}' for t in table], 'shared_bytes': shared_bytes,
     'checks': ['original leaf setters and native initializer', 'table validation before mutation',
                '968 comparisons incl NaN/inf', 'raw float reference', 'no cull/fill writes',
                '5 actual microcode/HLSL/DXIL fixtures', 'depth-only gate', 'compare before EDRAM scale'],
diff --git a/tools/gpu-aot/test_lod_shared_constants.py b/tools/gpu-aot/test_lod_shared_constants.py
new file mode 100644
index 0000000..e350277
--- /dev/null
+++ b/tools/gpu-aot/test_lod_shared_constants.py
@@ -0,0 +1,53 @@
+"""Check the actual candidate host layout and bias upload against its HLSL ABI."""
+import argparse
+from pathlib import Path
+import re
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('draw', type=Path)
+p.add_argument('common', type=Path)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+a.output.mkdir(parents=True, exist_ok=True)
+s = a.draw.read_text()
+start = s.index('struct SharedConstants {')
+shared = s[start:s.index('\n};', start)+3]
+bias = re.search(r'    const int32_t lod_bias = [^\n]+;\n    shared.fetch_lod_bias\[i\] = [^\n]+;', s)[0]
+common = a.common.read_text()
+assert 'g_FetchLodBiasArr[8] : packoffset(c39)' in common
+assert 'SharedConstants + 624 + (i)*4' in common
+assert 'g_AlphaFunction : packoffset(c37.z)' in common
+assert 'g_ViewportMode : packoffset(c37.w)' in common
+code = '''#include <cstdint>
+#include <cstddef>
+#include <array>
+#include <cassert>
+#include <iostream>
+using u32=uint32_t;
+''' + shared + '''
+int main(){
+ static_assert(sizeof(SharedConstants)==752);
+ static_assert(offsetof(SharedConstants,fetch_lod_bias)==624);
+ static_assert(offsetof(SharedConstants,alpha_function)==600);
+ static_assert(offsetof(SharedConstants,viewport_mode)==604);
+ static_assert(offsetof(SharedConstants,color_output_scale)==608);
+ SharedConstants initial;for(float value:initial.fetch_lod_bias)assert(value==0);
+ for(uint32_t encoded=0;encoded<1024;++encoded){
+  for(uint32_t i=0;i<32;++i){
+   SharedConstants shared;std::array<uint32_t,6> fetch{};
+   fetch[4]=0xFFC00FFFu|(encoded<<12);
+''' + bias + '''
+   int signed_value=encoded>=512?int(encoded)-1024:int(encoded);
+   assert(shared.fetch_lod_bias[i]==float(signed_value)/32.0f);
+   for(uint32_t slot=0;slot<32;++slot)if(slot!=i)assert(shared.fetch_lod_bias[slot]==0);
+  }
+ }
+ std::cout<<"PASS: actual host upload, HLSL ABI752, all 1024 signed biases in 32 slots, old offsets preserved\\n";
+}
+'''
+source = a.output/'lod-host-test.cpp'
+source.write_text(code)
+exe = a.output/'lod-host-test.exe'
+subprocess.run(['clang++', '-std=c++20', str(source), '-o', str(exe)], check=True)
+subprocess.run([str(exe.resolve())], check=True)
diff --git a/tools/gpu-aot/test_viewport_state.py b/tools/gpu-aot/test_viewport_state.py
index 94761e0..ae5ef4a 100644
--- a/tools/gpu-aot/test_viewport_state.py
+++ b/tools/gpu-aot/test_viewport_state.py
@@ -89,7 +89,7 @@ def body(s, signature):
 subprocess.run(['clang++','-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True)
 subprocess.run([str(exe)],check=True)
 proof = {'passed': True, 'original_setter_cases': 16, 'pixel_mapping_cases': 900,
-         'modes': ['TU23 0x43F', 'TU23 0x400'], 'shared_bytes': 624,
+         'modes': ['TU23 0x43F', 'TU23 0x400'], 'shared_bytes': 752 if 'fetch_lod_bias[32]' in draw else 624,
          'depth_clip_pipeline_cache': True, 'game_launched': False,
          'scope': 'CPU setter, exact emitted HLSL arithmetic, host viewport mapping and PSO identity; no pixel/character repair claim'}
 (a.output/'verification.json').write_text(json.dumps(proof,indent=2)+'\n')

From 50606edb644913e561d14b096ceca95d944460d7 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 13:58:48 +0300
Subject: [PATCH 38/44] Roll back explicit LOD integration after failed LOTR
 visual test

---
 rexlego/src/gpu_native/draw.cpp       |  8 ++------
 tools/gpu-aot/README.md               | 24 +++++++++++++++++-------
 tools/gpu-aot/native-work-20261004.md | 22 ++++++++++++++++++++++
 3 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
index a3997ed..140a351 100644
--- a/rexlego/src/gpu_native/draw.cpp
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -72,10 +72,8 @@ struct SharedConstants {
   u32 alpha_function = 7;
   u32 viewport_mode = 0;
   float color_output_scale[4]{1, 1, 1, 1};
-  float fetch_lod_bias[32]{};
 };
-static_assert(sizeof(SharedConstants) == 752);
-static_assert(offsetof(SharedConstants, fetch_lod_bias) == 624);
+static_assert(sizeof(SharedConstants) == 624);
 static_assert(offsetof(SharedConstants, booleans) == 512);
 static_assert(offsetof(SharedConstants, color_output_scale) == 608);
 static_assert(offsetof(SharedConstants, alpha_function) == 600);
@@ -310,8 +308,6 @@ bool BindConstants(
       }
     }
     shared.samplers[i] = HostDevice::RegisterSampler(fetch);
-    const int32_t lod_bias = int32_t((fetch[4] >> 12) & 1023);
-    shared.fetch_lod_bias[i] = float(lod_bias >= 512 ? lod_bias - 1024 : lod_bias) / 32.0f;
     if (trace_constants && i >= 16) {
       struct VertexTextureTrace {
         u32 guest = 0;
@@ -349,7 +345,7 @@ bool BindConstants(
     if (!name.empty()) LongProbeEvent("draw_constants", false, "file=", name,
         "VS=", BoundShaderHash(ShaderStage::kVertex),
         "PS=", BoundShaderHash(ShaderStage::kPixel), "texture_mask=", texture_mask,
-        "VS_offset=0 PS_offset=4096 shared_offset=8192 shared_bytes=752");
+        "VS_offset=0 PS_offset=4096 shared_offset=8192 shared_bytes=624");
   }
   std::memcpy(upload.mapped, data.data(), data.size());
   commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset), 0);
diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index fb4f479..c3b306d 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -4,25 +4,35 @@ This directory keeps the reproducible, project-owned part of the native D3D12
 shader pipeline. Generated archives, HLSL, DXIL, and build trees stay under
 `rexlego/out/native-gpu` and are intentionally not committed.
 
-## Current compiler and constants ABI (October 4)
-
-The current production compiler is the combined alpha/viewport/explicit-LOD
+## Current compiler and constants ABI (October 4, after owner visual test)
+
+The owner rejected the combined explicit-LOD build: LOTR looked worse, with
+vertical artifacts, while texture stripes and blank portraits remained. It has
+been rolled back as a coupled host/constants/shader-bank change. Production
+again uses the October1 alpha/viewport compiler in
+`session-20261001/viewport-candidate/build/alpha_compiler.exe`, its matching
+`source/shader_common.h`, and **624 shared bytes**. Main9290/runtime250 shader
+coverage, the Enter transition correction and Alt+Enter handling are retained.
+The rollback is an offline build verification, not a new visual success claim.
+
+The rejected candidate compiler is the combined alpha/viewport/explicit-LOD
 source in `session-20261004/lod-combined/source`, prepared reproducibly with:
 
 ```powershell
 python tools/gpu-aot/prepare_lod_candidate.py rexlego/out/native-gpu/session-20261004/lod-combined --base-source rexlego/out/native-gpu/session-20261001/viewport-candidate/source
 ```
 
-It requires `XENOS_RECOMP_LEGO_NATIVE_SCALE=1` and **752 shared bytes**: the
+That candidate requires `XENOS_RECOMP_LEGO_NATIVE_SCALE=1` and **752 shared bytes**: the
 existing 624-byte layout plus 32 fetch LOD biases at offset624. Regenerate both
 main and runtime archives and their linked variants together with the matching
-host upload. Older 624-byte compiler commands below are historical examples.
+host upload. Do not reinstall this candidate without isolating its regression.
 The original isolated September30 LOD candidate lacks the later alpha/viewport
 fixes and must not replace this combined compiler.
 
 Main coverage remains9290 (known A58 memexport exclusion), runtime250; all prior
-hashes and specialization masks are preserved. This corrects ignored explicit
-mip-level instructions; it does not establish repaired game pixels or FPS.
+hashes and specialization masks are preserved. The candidate implements ignored
+explicit mip-level instructions but failed the owner's image check. Synthetic
+fixture success did not establish the game's complete sampling correctness.
 No game launch was authorized for this integration. See `native-work-20261004.md`.
 
 ## Inputs
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index 6617062..02747bf 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -178,3 +178,25 @@ manual-run shader miss needs a complete captured container; pixel validation
 and clean FPS remain pending. Game stays closed, SDK/Vulkan/Plume/config untouched.
 
 Verified installed combined build: fingerprint 7ccb1c0c3f20bbde, executable SHA256 d31c258f8f798ed4fb8293f789870d854e9879389e11a3683187e4e8f7af0a05. Main9290/18444 and runtime250/448 all prelinked specialization subsets present. DLL pair/config unchanged. No launch. Proof lod-combined/installed-build-proof.json.
+
+## Owner rejects explicit-LOD candidate; coupled rollback
+
+The owner subsequently manually tested the candidate and reports no repairs,
+with worse LOTR rendering. The supplied screenshot shows stripes on Sonic and
+terrain, a blank blue portrait and tall vertical artifacts. Preserve it and
+the manual-run log in lod-combined/owner-regression.png and
+owner-regression-game.log. No game process was running at rollback time.
+
+Restore draw.cpp and both raw/prelinked banks from lod-combined/before as one
+change: production returns to624 shared bytes and the Oct1 alpha/viewport
+compiler, retaining250 runtime shaders/18983 index records and Alt+Enter.
+Copied backup timestamps must be advanced before building: otherwise Ninja
+can relink old752-byte objects alongside copied624-byte sources. Force the
+affected inputs newer and verify actual recompilation in rollback-build.log.
+
+Offline alpha, viewport and actual Alt+Enter handler checks pass. The screenshot
+does not localize the pre-existing texture stripes to LOD, tiling, endian
+conversion or UVs. Existing texture-upload dumps contain only the first eight
+base textures, so they cannot establish the affected LOTR mip contents. Do not
+claim portraits, prompts or LOTR are repaired. No launch, FPS measurement,
+SDK/Plume change or hotfix change is authorized/implied by this rollback.

From abb1c75181c3c7adeb8d60a65ea8b196a5ef1a44 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 15:03:20 +0300
Subject: [PATCH 39/44] Correct native texture channels and DXT3A; add offline
 GPU oracle and armed mip capture

---
 rexlego/CMakeLists.txt                        |   2 +
 rexlego/src/gpu_native/format.cpp             |   1 +
 .../src/gpu_native/texture_alpha4_upload.h    |  42 +++++
 rexlego/src/gpu_native/texture_swizzle.h      |  36 +++++
 rexlego/src/gpu_native/textures.cpp           | 145 +++++++++++++++---
 tools/gpu-aot/README.md                       |   4 +
 tools/gpu-aot/audit_texture_instructions.py   | 119 ++++++++++++++
 .../gpu-aot/native-texture-format-20261004.md |  65 ++++++++
 .../native-validation-pending-20261003.md     |  12 +-
 tools/gpu-aot/native-work-20261004.md         |   4 +
 tools/gpu-aot/run_native_probe.ps1            |  16 +-
 tools/gpu-aot/test_texture_alpha4_upload.cpp  |  56 +++++++
 .../gpu-aot/test_texture_instruction_audit.py |  53 +++++++
 tools/gpu-aot/test_texture_load_gpu.cpp       | 141 +++++++++++++++++
 tools/gpu-aot/test_texture_load_gpu.py        |  30 ++++
 tools/gpu-aot/test_texture_swizzle.py         |  64 ++++++++
 tools/gpu-aot/test_texture_upload_capture.py  |  58 ++++++-
 17 files changed, 816 insertions(+), 32 deletions(-)
 create mode 100644 rexlego/src/gpu_native/texture_alpha4_upload.h
 create mode 100644 rexlego/src/gpu_native/texture_swizzle.h
 create mode 100644 tools/gpu-aot/audit_texture_instructions.py
 create mode 100644 tools/gpu-aot/native-texture-format-20261004.md
 create mode 100644 tools/gpu-aot/test_texture_alpha4_upload.cpp
 create mode 100644 tools/gpu-aot/test_texture_instruction_audit.py
 create mode 100644 tools/gpu-aot/test_texture_load_gpu.cpp
 create mode 100644 tools/gpu-aot/test_texture_load_gpu.py
 create mode 100644 tools/gpu-aot/test_texture_swizzle.py

diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt
index a649a52..1fc435a 100644
--- a/rexlego/CMakeLists.txt
+++ b/rexlego/CMakeLists.txt
@@ -280,6 +280,8 @@ if(LEGODIMENSIONS_NATIVE_GPU)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/pixel_shader_filter.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/pool_copy.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/sampler_state.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/texture_swizzle.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/texture_alpha4_upload.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/cheat_menu.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/draw.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/draw.h"
diff --git a/rexlego/src/gpu_native/format.cpp b/rexlego/src/gpu_native/format.cpp
index 19d4911..21fba29 100644
--- a/rexlego/src/gpu_native/format.cpp
+++ b/rexlego/src/gpu_native/format.cpp
@@ -107,6 +107,7 @@ plume::RenderFormat ConvertXenosTextureFormat(u32 xenos_format) {
     case TF::k_DXN:
       return RF::BC5_UNORM;
     case TF::k_DXT3A:
+      return RF::R8_UNORM;
     case TF::k_DXT5A:
       return RF::BC4_UNORM;
     case TF::k_16_FLOAT:
diff --git a/rexlego/src/gpu_native/texture_alpha4_upload.h b/rexlego/src/gpu_native/texture_alpha4_upload.h
new file mode 100644
index 0000000..38013a2
--- /dev/null
+++ b/rexlego/src/gpu_native/texture_alpha4_upload.h
@@ -0,0 +1,42 @@
+#pragma once
+#include <cstdint>
+#include <span>
+
+namespace legodimensions::gpu_native {
+
+// DXT3A stores sixteen independent four-bit alpha values in an eight-byte
+// block. It is not BC4, which stores endpoints and three-bit palette indices.
+// Offset returns a guest block byte address, including tiling/packed-mip offsets.
+template <class Offset>
+bool CopyTextureAlpha4Blocks(std::span<const uint8_t> source,
+                            std::span<uint8_t> destination,
+                            uint32_t width, uint32_t height, uint32_t depth,
+                            uint32_t pitch, uint32_t endian_xor, Offset offset) {
+  if (!width || !height || !depth || pitch < width || endian_xor > 3 ||
+      uint64_t(pitch) * height > destination.size() / depth) return false;
+  for (uint32_t z = 0; z < depth; ++z) {
+    for (uint32_t by = 0; by < (uint64_t(height) + 3) / 4; ++by) {
+      for (uint32_t bx = 0; bx < (uint64_t(width) + 3) / 4; ++bx) {
+        const int64_t start = offset(bx, by, z);
+        if (start < 0) return false;
+        uint8_t block[8];
+        for (uint32_t byte = 0; byte < 8; ++byte) {
+          const uint64_t at = (uint64_t(start) + byte) ^ endian_xor;
+          if (at >= source.size()) return false;
+          block[byte] = source[size_t(at)];
+        }
+        for (uint32_t y = 0; y < 4 && uint64_t(by) * 4 + y < height; ++y) {
+          for (uint32_t x = 0; x < 4 && uint64_t(bx) * 4 + x < width; ++x) {
+            const uint32_t texel = y * 4 + x;
+            const uint8_t alpha = (block[texel / 2] >> ((texel & 1) * 4)) & 15;
+            const uint64_t at = (uint64_t(z) * height + uint64_t(by) * 4 + y) * pitch + uint64_t(bx) * 4 + x;
+            destination[size_t(at)] = alpha * 17;
+          }
+        }
+      }
+    }
+  }
+  return true;
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/texture_swizzle.h b/rexlego/src/gpu_native/texture_swizzle.h
new file mode 100644
index 0000000..02e6e34
--- /dev/null
+++ b/rexlego/src/gpu_native/texture_swizzle.h
@@ -0,0 +1,36 @@
+#pragma once
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+
+// Xenos expands scalar/two-channel formats before applying fetch.swizzle.
+// Desktop R/RG and BC4/BC5 views instead fill missing channels with 0/1.
+// Values are six-bit Xenos TextureFormat IDs, not full D3DFORMAT words.
+constexpr uint32_t TextureFormatChannelSwizzle(uint32_t format) {
+  switch (format) {
+    case 2: case 8: case 9:             // 8, 8_A, 8_B
+    case 22: case 23:                  // D24 depth sampling
+    case 30: case 36: case 58: case 59:  // 16_FLOAT, 32_FLOAT, DXT3A, DXT5A
+      return 0;                       // RRRR
+    case 10: case 13: case 25:          // 8_8, 16_16_EDRAM, 16_16
+    case 31: case 37: case 49:          // 16_16_FLOAT, 32_32_FLOAT, DXN
+      return (1u << 3) | (1u << 6) | (1u << 9);  // RGGG
+    default:
+      return 0x688;                   // RGBA, existing supported four-channel formats
+  }
+}
+
+constexpr uint32_t NativeTextureSwizzle(uint32_t format, uint32_t guest) {
+  const uint32_t channels = TextureFormatChannelSwizzle(format);
+  uint32_t result = 0;
+  for (uint32_t lane = 0; lane < 4; ++lane) {
+    const uint32_t requested = (guest >> (lane * 3)) & 7;
+    // Xenos/SDK sanitizes reserved selectors 6/7 to constant zero/one.
+    const uint32_t selected = requested < 4 ? (channels >> (requested * 3)) & 7
+        : requested & 5;
+    result |= selected << (lane * 3);
+  }
+  return result;
+}
+
+}  // namespace legodimensions::gpu_native
diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index 90c9dad..a1a972f 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -32,8 +32,10 @@
 #include "gpu_native/shaders.h"
 #include "gpu_native/format.h"
 #include "gpu_native/texture_number.h"
+#include "gpu_native/texture_swizzle.h"
 #include "gpu_native/texture_upload_cache.h"
 #include "gpu_native/texture_volume_upload.h"
+#include "gpu_native/texture_alpha4_upload.h"
 #include "gpu_native/long_probe.h"
 #include "gpu_native/memory_watch.h"
 #include "gpu_native/depth_alias.h"
@@ -282,9 +284,10 @@ std::shared_ptr<TextureResource> AdoptTexture(u32 guest_address) {
         plume::RenderSwizzle::ZERO, plume::RenderSwizzle::ONE,
         plume::RenderSwizzle::ZERO, plume::RenderSwizzle::ZERO};
     const u32 swizzle = resource->guest_fetch.swizzle;
+    const u32 host_swizzle = NativeTextureSwizzle(resource->guest_format, swizzle);
     view_desc.componentMapping = plume::RenderComponentMapping(
-        swizzles[swizzle & 7], swizzles[(swizzle >> 3) & 7],
-        swizzles[(swizzle >> 6) & 7], swizzles[(swizzle >> 9) & 7]);
+        swizzles[host_swizzle & 7], swizzles[(host_swizzle >> 3) & 7],
+        swizzles[(host_swizzle >> 6) & 7], swizzles[(host_swizzle >> 9) & 7]);
     view_desc.format = resource->format;
     view_desc.mipLevels = resource->levels;
     view_desc.dimension =
@@ -467,35 +470,81 @@ u32 LevelOffset(const TextureResource& resource, u32 level) {
 
 // Opt-in diagnostic: inspect the actual base-level bytes uploaded to D3D12,
 // independently of draw geometry, shaders, swizzle and sampler state.
-void DumpTextureUpload(const TextureResource& resource, const u8* data) {
-  static const std::filesystem::path root = [] {
+struct TextureCaptureState {
+  std::filesystem::path root, trigger;
+  bool all_levels = false, armed = false;
+  u32 checked_frame = ~u32{0};
+  u64 written_bytes = 0;
+  std::unordered_set<u32> dumped;
+};
+
+TextureCaptureState& TextureCapture() {
+  static TextureCaptureState state = [] {
+    TextureCaptureState result;
     char* value = nullptr;
     size_t length = 0;
     _dupenv_s(&value, &length, "LEGO_DUMP_TEXTURE_UPLOADS");
-    std::filesystem::path result(value ? value : "");
+    result.root = value ? value : "";
+    std::free(value);
+    value = nullptr;
+    _dupenv_s(&value, &length, "LEGO_DUMP_TEXTURE_UPLOADS_TRIGGER");
+    result.trigger = value ? value : "";
     std::free(value);
+    result.all_levels = std::getenv("LEGO_DUMP_TEXTURE_UPLOADS_ALL_LEVELS") != nullptr;
     return result;
   }();
-  if (root.empty()) return;
+  return state;
+}
+
+// The renderer recording lock serializes capture state, just like CPU upload.
+// Disabled captures do no filesystem work. The trigger is checked once/frame.
+bool TextureUploadCapturePending(const TextureResource& resource) {
+  auto& state = TextureCapture();
+  if (state.root.empty() || state.dumped.size() >= (state.all_levels ? 32u : 8u) ||
+      state.written_bytes >= 128ull * 1024 * 1024 || state.dumped.contains(resource.guest_address)) return false;
+  if (!state.trigger.empty() && state.checked_frame != g_probe_frame.load()) {
+    state.checked_frame = g_probe_frame.load();
+    std::error_code error;
+    state.armed = std::filesystem::is_regular_file(state.trigger, error) && !error;
+  }
+  if (!state.trigger.empty() && !state.armed) return false;
+  if (state.all_levels && (u32(resource.guest_fetch.dimension) != 1 || resource.levels > 16)) return false;
+  if (resource.format != plume::RenderFormat::BC1_UNORM &&
+      resource.format != plume::RenderFormat::BC3_UNORM &&
+      resource.format != plume::RenderFormat::R8G8B8A8_UNORM) return false;
   static const bool logos_only = std::getenv("LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY") != nullptr;
   if (logos_only) {
     const u64 vs = BoundShaderHash(ShaderStage::kVertex);
     const u64 ps = BoundShaderHash(ShaderStage::kPixel);
     if (!((vs == 0x91007ACB3E640E3Dull && ps == 0xEEE573BE160E037Dull) ||
-          (vs == 0xC3958E2D1B795ED9ull && ps == 0x0C1840BF35E84F2Full))) return;
+          (vs == 0xC3958E2D1B795ED9ull && ps == 0x0C1840BF35E84F2Full))) return false;
   }
+  return true;
+}
+
+void DumpTextureUpload(const TextureResource& resource, const u8* data,
+                       std::span<const u8> base = {}, std::span<const u8> mips = {}) {
+  if (!TextureUploadCapturePending(resource)) return;
+  auto& state = TextureCapture();
+  const auto& root = state.root;
   const bool bc1 = resource.format == plume::RenderFormat::BC1_UNORM;
   const bool bc3 = resource.format == plume::RenderFormat::BC3_UNORM;
   const bool compressed = bc1 || bc3;
   if (!compressed && resource.format != plume::RenderFormat::R8G8B8A8_UNORM) return;
-  static std::mutex dump_mutex;
-  static std::unordered_set<u32> dumped;
-  std::lock_guard lock(dump_mutex);
-  if (dumped.size() >= 8 || !dumped.insert(resource.guest_address).second) return;
+  state.dumped.insert(resource.guest_address);
+  const u32 levels = state.all_levels ? resource.levels : 1;
+  u64 payload_size = 128;
+  for (u32 level = 0; level < levels; ++level) {
+    const u32 w = std::max(1u, resource.width >> level), h = std::max(1u, resource.height >> level);
+    payload_size += u64(compressed ? ((w + 3) / 4) * (bc1 ? 8 : 16) : w * 4) * RowCount(resource.format, h);
+  }
+  if (state.all_levels) payload_size += base.size() + mips.size();
+  if (payload_size > 128ull * 1024 * 1024 - state.written_bytes) return;
+  state.written_bytes += payload_size;
   std::error_code error;
   std::filesystem::create_directories(root, error);
   if (error) return;
-  // Legacy DDS, one mip, little-endian host. Strip D3D12 row padding.
+  // Legacy DDS, little-endian host. Strip row padding and 512-byte mip gaps.
   u32 header[32]{};
   header[0] = 0x20534444; header[1] = 124;
   header[2] = 0x1007 | (compressed ? 0x80000 : 0x8);
@@ -510,18 +559,43 @@ void DumpTextureUpload(const TextureResource& resource, const u8* data) {
     header[25] = 0xFF0000; header[26] = 0xFF000000;
   }
   header[27] = 0x1000;
+  if (levels > 1) { header[2] |= 0x20000; header[7] = levels; header[27] |= 0x400008; }
   const std::string name = std::to_string(resource.guest_address);
   std::ofstream file(root / (name + ".dds"), std::ios::binary);
   file.write(reinterpret_cast<const char*>(header), sizeof(header));
-  for (u32 row = 0; row < rows; ++row)
-    file.write(reinterpret_cast<const char*>(data + row * RowPitch(resource.format, resource.width)), row_bytes);
+  u64 offset = 0;
+  for (u32 level = 0; level < levels; ++level) {
+    offset = (offset + 511) & ~u64{511};
+    const u32 w = std::max(1u, resource.width >> level), h = std::max(1u, resource.height >> level);
+    const u32 bytes = compressed ? ((w + 3) / 4) * (bc1 ? 8 : 16) : w * 4;
+    const u32 level_rows = RowCount(resource.format, h), pitch = RowPitch(resource.format, w);
+    for (u32 row = 0; row < level_rows; ++row)
+      file.write(reinterpret_cast<const char*>(data + offset + u64(row) * pitch), bytes);
+    offset += u64(pitch) * level_rows;
+  }
+  if (state.all_levels) {
+    for (const auto& item : {std::pair{".guest-base.bin", base}, std::pair{".guest-mips.bin", mips}}) {
+      if (item.second.empty()) continue;
+      std::ofstream raw(root / (name + item.first), std::ios::binary);
+      raw.write(reinterpret_cast<const char*>(item.second.data()), item.second.size());
+    }
+  }
   std::ofstream metadata(root / (name + ".txt"));
   const auto& fetch = resource.guest_fetch;
   metadata << "guest=" << std::hex << resource.guest_address
            << " base=" << (fetch.base_address << 12) << " mip=" << (fetch.mip_address << 12)
            << " swizzle=" << fetch.swizzle << std::dec << " endian=" << u32(fetch.endianness)
            << " pitch=" << (fetch.pitch << 5) << " tiled=" << fetch.tiled
-           << " packed=" << fetch.packed_mips << '\n';
+           << " packed=" << fetch.packed_mips << " levels=" << levels
+           << " width=" << resource.width << " height=" << resource.height
+           << " frame=" << g_probe_frame.load() << std::hex
+           << " VS=" << BoundShaderHash(ShaderStage::kVertex)
+           << " PS=" << BoundShaderHash(ShaderStage::kPixel) << '\n';
+  std::array<u32, 6> words{};
+  std::memcpy(words.data(), &fetch, sizeof(words));
+  metadata << "fetch=";
+  for (const auto word : words) metadata << word << ',';
+  metadata << '\n';
   REXLOG_INFO("Native GPU: captured texture upload {} ({}x{}, format={})",
               name, resource.width, resource.height, resource.guest_format);
 }
@@ -985,7 +1059,8 @@ bool UploadTextureResource(u32 guest_address,
     std::lock_guard lock(resource->mutex);
     if (resource->resolved_on_host) return true;
     static const bool audit_texture_watch = std::getenv("LEGO_NATIVE_AUDIT_TEXTURE_WATCH") != nullptr;
-    const bool watch_hit = !require_content_hash && resource->guest_uploaded &&
+    const bool capture_pending = TextureUploadCapturePending(*resource);
+    const bool watch_hit = !capture_pending && !require_content_hash && resource->guest_uploaded &&
         CpuMemoryUnchanged(resource->cpu_stamp);
     if (watch_hit && !audit_texture_watch) {
       if (timer.enabled) ++g_upload_timing.source_hits;
@@ -1000,9 +1075,10 @@ bool UploadTextureResource(u32 guest_address,
         fetch.stacked || !fetch.base_address)
       return false;
     const u32 faces = cube ? 6 : 1;
-    const u32 block_bytes = plume::RenderFormatSize(resource->format);
-    const u32 block_width = plume::RenderFormatBlockWidth(resource->format);
     const auto* format_info = rex::graphics::FormatInfo::Get(fetch.format);
+    const bool alpha4 = fetch.format == rex::graphics::xenos::TextureFormat::k_DXT3A;
+    const u32 block_bytes = alpha4 ? 8 : plume::RenderFormatSize(resource->format);
+    const u32 block_width = alpha4 ? 4 : plume::RenderFormatBlockWidth(resource->format);
     if (!block_bytes || !block_width ||
         format_info->bits_per_pixel * format_info->block_width * format_info->block_height != block_bytes * 8)
       return false;
@@ -1032,7 +1108,7 @@ bool UploadTextureResource(u32 guest_address,
           "base=", u32(fetch.base_address<<12), "mip=", u32(fetch.mip_address<<12));
     // Never read an unvalidated volume extent. Volumes may have padding
     // between Z slices, so validating just width*height*depth isn't enough.
-    if (volume && !source_key_valid) return false;
+    if ((volume || alpha4 || capture_pending) && !source_key_valid) return false;
     if (source_key_valid) {
       const auto hash_start = timer.enabled ? std::chrono::steady_clock::now()
                                             : std::chrono::steady_clock::time_point{};
@@ -1058,7 +1134,7 @@ bool UploadTextureResource(u32 guest_address,
         g_upload_timing.source_ms += std::chrono::duration<double, std::milli>(
             std::chrono::steady_clock::now() - source_start).count();
       }
-      if (resource->guest_uploaded && resource->upload_source_key_valid &&
+      if (!capture_pending && resource->guest_uploaded && resource->upload_source_key_valid &&
           resource->upload_source_key == source_key) {
         if (timer.enabled) ++g_upload_timing.source_hits;
         resource->cpu_stamp = std::move(cpu_stamp);
@@ -1107,6 +1183,25 @@ bool UploadTextureResource(u32 guest_address,
       const u32 width = MipDimension(resource->width, level);
       const u32 height = MipDimension(resource->height, level);
       const u32 pitch = RowPitch(resource->format, width);
+      if (alpha4) {
+        const bool copied = CopyTextureAlpha4Blocks(
+            {source, guest_level.array_slice_data_extent_bytes},
+            {mapped + destination_offset, LevelSize(*resource, level)},
+            width, height, volume ? MipDimension(resource->depth, level) : 1,
+            pitch, endian_xor, [&](u32 x, u32 y, u32 z) -> int64_t {
+              if (fetch.tiled) {
+                if (volume) return tu::GetTiledOffset3D(x + offset_x, y + offset_y, z + offset_z,
+                    guest_level.row_pitch_bytes / 8, guest_level.z_slice_stride_block_rows, 3);
+                return tu::GetTiledOffset2D(x + offset_x, y + offset_y,
+                    guest_level.row_pitch_bytes / 8, 3);
+              }
+              return (u64(z + offset_z) * guest_level.z_slice_stride_block_rows + y + offset_y) *
+                  guest_level.row_pitch_bytes + u64(x + offset_x) * 8;
+            });
+        if (!copied) return false;
+        destination_offset += LevelSize(*resource, level);
+        continue;
+      }
       if (volume) {
         const u32 block_height = format_info->block_height;
         const bool copied = CopyTextureVolumeBlocks(
@@ -1141,7 +1236,9 @@ bool UploadTextureResource(u32 guest_address,
       }
       destination_offset += LevelSize(*resource, level);
     }
-    DumpTextureUpload(*resource, mapped);
+    DumpTextureUpload(*resource, mapped,
+        source_key_valid ? std::span<const u8>(base_bytes, layout.base.level_data_extent_bytes) : std::span<const u8>{},
+        source_key_valid ? std::span<const u8>(mip_bytes, mip_size) : std::span<const u8>{});
     if (timer.enabled) g_upload_timing.converted_bytes += upload_size;
     const u64 content_hash = XXH3_64bits(mapped, upload_size);
     if (LongProbeEnabled() && (!resource->guest_uploaded || resource->guest_content_hash != content_hash))
@@ -1166,6 +1263,10 @@ bool UploadTextureResource(u32 guest_address,
     commands->barriers(plume::RenderBarrierStage::COPY,
         plume::RenderTextureBarrier(resource->texture.get(), plume::RenderTextureLayout::COPY_DEST));
     destination_offset = 0;
+    // Copy footprints describe decoded host storage, not the guest blocks
+    // (DXT3A expands eight-byte blocks into individual R8 pixels).
+    const u32 host_block_bytes = plume::RenderFormatSize(resource->format);
+    const u32 host_block_width = plume::RenderFormatBlockWidth(resource->format);
     for (u32 subresource = 0; subresource < faces * resource->levels; ++subresource) {
       const u32 face = subresource / resource->levels;
       const u32 level = subresource % resource->levels;
@@ -1176,7 +1277,7 @@ bool UploadTextureResource(u32 guest_address,
           plume::RenderTextureCopyLocation::Subresource(resource->texture.get(), level, face),
           plume::RenderTextureCopyLocation::PlacedFootprint(upload.get(), resource->format,
               width, height, volume ? MipDimension(resource->depth, level) : 1,
-              RowPitch(resource->format, width) / block_bytes * block_width,
+              RowPitch(resource->format, width) / host_block_bytes * host_block_width,
               destination_offset));
       destination_offset += LevelSize(*resource, level);
     }
diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index c3b306d..5cc9280 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -550,3 +550,7 @@ or failed resize is retried even when native framebuffers are empty.
 `test_resize_fences.py` executes the actual wait/resize bodies with deterministic
 fences/events and checks all slots, removal, partial retries and backbuffer
 restoration. These offline checks do not confirm real window-mode recovery.
+
+## Offline format and scene capture verification (October4)
+
+See native-texture-format-20261004.md. Adopted scalar/RG views now compose Xenos channel replication; DXT3A expands its four-bit alpha blocks to R8. Tests include102400 swizzle cases,6144 alpha cases and48 comparisons against actual SDK D3D12 loader bytecode without launching a game. TextureForensics prepares bounded, late-triggered full-mip/raw-backing captures for a future authorized scene. These checks do not establish a repair for LOTR stripes or blank icons. Production remains624-byte ABI; the rejected LOD candidate stays rolled back.
diff --git a/tools/gpu-aot/audit_texture_instructions.py b/tools/gpu-aot/audit_texture_instructions.py
new file mode 100644
index 0000000..aad81e4
--- /dev/null
+++ b/tools/gpu-aot/audit_texture_instructions.py
@@ -0,0 +1,119 @@
+"""Inventory actual fetch instructions in local Xenos containers, without a game.
+
+Uses EXEC instruction sequences, not arbitrary byte pattern matches. Game input
+and generated reports remain local. This is coverage evidence, not a pixel test.
+"""
+import argparse
+from collections import Counter
+import hashlib
+import json
+from pathlib import Path
+import struct
+
+EXEC = {1, 2, 3, 4, 5, 6, 13, 14}
+
+def fetches(data):
+    flags, virtual, physical = struct.unpack_from('>III', data)
+    if flags not in (0x102A1100, 0x102A1101):
+        raise ValueError('Not a vertex/pixel Xenos container')
+    if len(data) != virtual + 4 + physical:
+        raise ValueError('Container length mismatch')
+    header = struct.unpack_from('>I', data, 24)[0]
+    offset, size = struct.unpack_from('>II', data, header)
+    if size % 12 or offset + size > physical:
+        raise ValueError('Invalid microcode extent')
+    words = struct.unpack_from('>' + 'I' * (size // 4), data, virtual + 4 + offset)
+    boundary = len(words) // 3
+    pos = 0
+    seen = set()
+    result = []
+    while pos < boundary:
+        w0, w1, w2 = words[pos * 3:pos * 3 + 3]
+        for cf in (w0 | ((w1 & 65535) << 32), (w1 >> 16) | (w2 << 16)):
+            opcode = (cf >> 44) & 15
+            if opcode not in EXEC:
+                continue
+            address, count, sequence = cf & 4095, (cf >> 12) & 7, (cf >> 16) & 4095
+            boundary = min(boundary, address)
+            if address + count > len(words) // 3:
+                raise ValueError('EXEC outside microcode')
+            for i in range(count):
+                at = address + i
+                if (sequence >> (i * 2)) & 1 and at not in seen:
+                    seen.add(at)
+                    a, b, c = words[at * 3:at * 3 + 3]
+                    op = a & 31
+                    if op == 0:
+                        continue  # Vertex fetch, not a texture operation.
+                    result.append({'instruction':at, 'opcode':op,
+                        'slot':(a >> 20) & 31, 'denormalized':bool((a >> 25) & 1),
+                        'dimension':(c >> 14) & 3,
+                        'mag_filter':(b >> 12) & 3, 'min_filter':(b >> 14) & 3,
+                        'mip_filter':(b >> 16) & 3,
+                        'computed_lod':bool((b >> 28) & 1),
+                        'register_lod':bool((b >> 29) & 1),
+                        'register_gradients':bool(c & 1),
+                        'instruction_lod_bias':(((c >> 2) & 127) - (128 if c & 256 else 0)) / 16})
+        pos += 1
+    return flags & 1, result
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('input', type=Path)
+    p.add_argument('output', type=Path)
+    a = p.parse_args()
+    counts = Counter()
+    shaders = []
+    containers = []
+    paths = sorted(a.input.glob('*.bin'))
+    if paths:
+        containers = [(path.name, path.read_bytes()) for path in paths]
+    else:
+        seen = set()
+        for path in sorted(a.input.rglob('*.360_shaders')):
+            asset = path.read_bytes()
+            for signature in (b'\x10\x2a\x11\x00', b'\x10\x2a\x11\x01'):
+                at = asset.find(signature)
+                while at != -1:
+                    if at + 36 <= len(asset):
+                        _, virtual, physical = struct.unpack_from('>III', asset, at)
+                        end = at + virtual + 4 + physical
+                        if 36 <= virtual <= 16*1024*1024 and physical >= 12 and end <= len(asset):
+                            data = asset[at:end]
+                            key = hashlib.sha256(data).digest()
+                            if key not in seen:
+                                try:
+                                    fetches(data)
+                                except (ValueError, struct.error):
+                                    pass
+                                else:
+                                    seen.add(key)
+                                    containers.append((str(path.relative_to(a.input))+f'@{at:X}', data))
+                    at = asset.find(signature, at+1)
+    if not containers:
+        raise ValueError('No shader containers found')
+    for name, data in containers:
+        stage, records = fetches(data)
+        counts['containers'] += 1
+        interesting = []
+        for record in records:
+            op = record['opcode']
+            counts[f'opcode_{op}'] += 1
+            if op != 1:
+                continue
+            for field in ['denormalized', 'register_gradients']:
+                if record[field]:
+                    counts[field] += 1
+            for field in ['mag_filter', 'min_filter', 'mip_filter']:
+                counts[f'{field}_{record[field]}'] += 1
+            if record['denormalized'] or record['register_gradients'] or record['mip_filter'] != 3:
+                interesting.append(record)
+        if interesting:
+            shaders.append({'container':name, 'stage':'VS' if stage else 'PS',
+                            'fetches':interesting})
+    a.output.parent.mkdir(parents=True, exist_ok=True)
+    a.output.write_text(json.dumps({'counts':dict(counts), 'shaders':shaders}, indent=2)+'\n')
+    print(json.dumps(dict(counts), sort_keys=True))
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/gpu-aot/native-texture-format-20261004.md b/tools/gpu-aot/native-texture-format-20261004.md
new file mode 100644
index 0000000..9aa1aae
--- /dev/null
+++ b/tools/gpu-aot/native-texture-format-20261004.md
@@ -0,0 +1,65 @@
+# Offline texture format investigation, October4
+
+Latest owner instruction: keep working offline; no game launch. Future authorized
+probes use monitor3. The rejected explicit-LOD candidate remains rolled back,
+with the production624-byte shared ABI and the existing250 runtime shaders.
+
+Two concrete mismatches were found against the read-only SDK texture code:
+
+- Scalar formats expand to RRRR and two-channel formats to RGGG before the
+  guest fetch swizzle. Desktop R/RG and BC4/BC5 defaults instead fill absent
+  channels with0/1. Adopted texture views now compose format replication with
+  guest swizzling, retaining the separate logical-RGBA color-resolve view.
+- DXT3A contains sixteen independent four-bit alpha values in each eight-byte
+  block. Treating it as BC4 incorrectly interprets those bytes as palette
+  endpoints/indices. It now expands into R8 storage, including packed mips,
+  endian conversion and volume slices. Copy footprints use host R8 sizes.
+
+These are format fixes, not an established explanation for the owner's visual
+failures. In particular, common BC1/BC3 four-channel scenery is unaffected by
+channel replication, and DXT3A was not observed in the latest manual-run log.
+
+Verification on RTX4060, without a game, window or swap chain:
+
+- `test_texture_swizzle.py OUTPUT`:102400 compositions across25 formats
+  against the actual SDK format table and GuestToHostSwizzle function body.
+- `test_texture_alpha4_upload.cpp`:6144 cases, all four endian modes, odd
+  edges, padded rows/slices, packed/permuted offsets and invalid bounds.
+- `test_texture_load_gpu.py OUTPUT`:48 comparisons against the SDK's actual
+  D3D12 texture_load_64bpb and texture_load_dxt3a compute bytecode, covering
+  linear/tiled32x32,128x64,1024x512 and four endian modes. Packed offset testing
+  uses CPU extraction; the GPU tiled loader decodes the tile from its origin.
+  This verifies byte decoding, not material sampling or game pixels.
+- Native Release build passes. Build proof and all local test payloads stay
+  under ignored `session-20261004`; SDK/runtime/plugin/config were preserved.
+
+Instruction audit uses actual EXEC sequences rather than byte-pattern matches.
+Runtime250 contains563 texture fetches,41 SetLOD and5 GetWeights operations.
+The scanned main asset corpus9289 contains31847 texture fetches,1110 SetLOD,
+100 GetGradients and702 GetWeights. All audited texture fetches use normalized
+coordinates, fetch-constant filters and no register-gradient flag. This does
+not establish that the omitted non-fetch operations are harmless, and does
+not justify reinstalling the rejected LOD candidate. The EXEC fixture checks
+paired/conditional sequences, deduplication, ALU/vertex exclusion and extents.
+
+## Prepared scene-specific evidence
+
+`run_native_probe.ps1 -LongProbe -PortraitTrace -TextureForensics -CaptureMissing`
+prepares a disabled-until-triggered capture. It does not arm at startup. Once
+the owner reaches the affected scene in an authorized monitor3 run, create
+the fresh process JSON's textureCaptureTrigger file. The renderer will decode
+already-cached textures once for capture without forcing an unchanged GPU
+upload. Each record contains all mip DDS pixels, exact base/mip guest bytes,
+six fetch words, shader identities and frame. Captures are restricted to2D,
+32 textures and128MiB; no filesystem checks occur when capture is disabled.
+The current DDS writer supports BC1, BC3 and RGBA8. It does not dump every
+possible texture format. Existing first-eight startup captures were inadequate
+to identify the LOTR striped texture's mip bytes.
+
+`test_texture_upload_capture.py OUTPUT` executes the actual capture functions
+and checks late arming, cached capture eligibility, seven mip levels, stripped
+row/mip padding, exact guest backing, legacy logo filtering and capture limits.
+
+Portraits, vehicle icons, button prompts, LOTR artifacts, missing shaders and
+three-level/cutscene correctness remain open. Internal rendering remains720p;
+actual1080p and stable60FPS are not achieved or measured by these tests.
diff --git a/tools/gpu-aot/native-validation-pending-20261003.md b/tools/gpu-aot/native-validation-pending-20261003.md
index 2ba598d..f715e88 100644
--- a/tools/gpu-aot/native-validation-pending-20261003.md
+++ b/tools/gpu-aot/native-validation-pending-20261003.md
@@ -1,7 +1,10 @@
 # Required runtime validation after October 3 offline fixes
 
-October4: the owner authorized resuming game tests and requested monitor3 for
-all current probes, superseding the earlier no-launch/monitor2 instruction.
+Latest October4 instruction: **keep working offline; no launch yet**. Earlier
+launch authorization has been revoked. Monitor3 applies to future authorized
+probes. The owner rejected the combined explicit-LOD candidate; production
+has returned to the coupled624-byte host/shader ABI. Texture format fixes and
+late-trigger mip forensics have passed offline checks, but are visually untested.
 Use questions for user-assisted hub readiness; no activation or injected input.
 Keep the exact executable,
 SDK/runtime/plugin hashes and configuration with every probe; close the exact
@@ -19,9 +22,10 @@ Once launches are authorized:
 2. Slot2 controllable Vorton, matching native/emulated character and camera.
    Check head, body, hands, HUD icons, portrait, effects, shadows and portals.
    Preserve the override material/texture identity if the portrait remains blank.
-   October4: use -LongProbe -PortraitTrace to correlate the actual mask's
+   October4: use -LongProbe -PortraitTrace -TextureForensics -CaptureMissing to correlate the actual mask's
    NuTexture/backend/active D3D header with draw shader/texture/constant captures. See
-   native-work-20261004.md. Keep this separate from clean FPS measurements.
+   native-work-20261004.md. Arm the fresh process record's textureCaptureTrigger
+   only after the owner reaches the affected scene. Keep this separate from clean FPS measurements.
 3. Audit buffer watches in hub and three story levels, loading/cutscenes and
    mesh-pool relocations. Require zero stale-clean errors. Until then buffer
    watches stay OFF by default, independent of the existing texture watch.
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index 02747bf..d3847c0 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -200,3 +200,7 @@ conversion or UVs. Existing texture-upload dumps contain only the first eight
 base textures, so they cannot establish the affected LOTR mip contents. Do not
 claim portraits, prompts or LOTR are repaired. No launch, FPS measurement,
 SDK/Plume change or hotfix change is authorized/implied by this rollback.
+
+## Offline texture format and scene capture follow-up
+
+See native-texture-format-20261004.md for tested format replication/DXT3A fixes, actual SDK GPU oracle results, instruction audit and late-trigger mip capture. Owner explicitly reaffirmed no launch. Visual failures and native1080p/60FPS remain unresolved.
diff --git a/tools/gpu-aot/run_native_probe.ps1 b/tools/gpu-aot/run_native_probe.ps1
index f356aad..6f9b650 100644
--- a/tools/gpu-aot/run_native_probe.ps1
+++ b/tools/gpu-aot/run_native_probe.ps1
@@ -15,6 +15,7 @@ param(
     [switch]$Stencil,
     [switch]$MeshTrace,
     [switch]$LogoUploads,
+    [switch]$TextureForensics,
     [switch]$Viewport,
     [switch]$BufferWindows,
     [switch]$DepthAlias,
@@ -27,9 +28,11 @@ param(
 )
 $ErrorActionPreference = 'Stop'
 if ($PortraitTrace -and !$LongProbe) { throw 'PortraitTrace requires LongProbe for correlated draw/constants capture' }
+if ($TextureForensics -and !$LongProbe) { throw 'TextureForensics requires LongProbe for correlated shader/draw evidence' }
+if ($TextureForensics -and $LogoUploads) { throw 'TextureForensics and LogoUploads use separate capture filters' }
 if ($ConfiguredRenderer -and $Pm4Reference) { throw 'Select configured renderer or explicit PM4 reference, not both' }
 if ($DepthAlias -and $DisableDepthAlias) { throw 'DepthAlias and DisableDepthAlias are mutually exclusive' }
-if (($Pm4Reference -or $ConfiguredRenderer) -and ($Stencil -or $MeshTrace -or $LogoUploads -or $Viewport -or $BufferWindows -or $BufferWatch -or $AuditBufferWatch -or $DepthAlias -or $DisableDepthAlias -or $LongProbe -or $LogoCaptureRenderDocDll)) {
+if (($Pm4Reference -or $ConfiguredRenderer) -and ($Stencil -or $MeshTrace -or $LogoUploads -or $TextureForensics -or $Viewport -or $BufferWindows -or $BufferWatch -or $AuditBufferWatch -or $DepthAlias -or $DisableDepthAlias -or $LongProbe -or $LogoCaptureRenderDocDll)) {
     throw 'PM4 reference mode uses original XDK/SDK objects; native-only diagnostic/candidate switches must be disabled'
 }
 $projectRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
@@ -57,7 +60,8 @@ $keys = @('SDL_WINDOW_ACTIVATE_WHEN_SHOWN','LEGO_NATIVE_TIMING','LEGO_NATIVE_ALL
     'LEGO_NATIVE_NO_MEMORY_WATCH','LEGO_NATIVE_STATIC_TEXTURE_WATCH','LEGO_NATIVE_AUDIT_TEXTURE_WATCH','LEGO_NATIVE_BUFFER_WATCH','LEGO_NATIVE_AUDIT_BUFFER_WATCH','LEGO_NATIVE_NO_DRAW_ARENA',
     'LEGO_NATIVE_FRAME_METRICS','LEGO_NATIVE_ASYNC_CPU_RESOURCES','LEGO_NATIVE_STENCIL','LEGO_NATIVE_MESH_TRACE',
     'LEGO_NATIVE_VIEWPORT','LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY','LEGO_NATIVE_BUFFER_WINDOWS','LEGO_NATIVE_PM4_REFERENCE',
-    'LEGO_NATIVE_RENDERDOC_DLL','LEGO_NATIVE_RENDERDOC_CAPTURE','LEGO_NATIVE_DEPTH_ALIAS','LEGO_NATIVE_PORTRAIT_TRACE')
+    'LEGO_NATIVE_RENDERDOC_DLL','LEGO_NATIVE_RENDERDOC_CAPTURE','LEGO_NATIVE_DEPTH_ALIAS','LEGO_NATIVE_PORTRAIT_TRACE',
+    'LEGO_DUMP_TEXTURE_UPLOADS_ALL_LEVELS','LEGO_DUMP_TEXTURE_UPLOADS_TRIGGER')
 $saved = @{}
 foreach ($key in $keys) {
     $saved[$key] = [Environment]::GetEnvironmentVariable($key,'Process')
@@ -97,6 +101,12 @@ try {
         $env:LEGO_DUMP_TEXTURE_UPLOADS = Join-Path $runRoot "logo-uploads-$Label"
         $env:LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY = '1'
     }
+    if ($TextureForensics) {
+        $env:LEGO_DUMP_TEXTURE_UPLOADS = Join-Path $runRoot "texture-forensics-$Label"
+        New-Item -ItemType Directory -Path $env:LEGO_DUMP_TEXTURE_UPLOADS | Out-Null
+        $env:LEGO_DUMP_TEXTURE_UPLOADS_ALL_LEVELS = '1'
+        $env:LEGO_DUMP_TEXTURE_UPLOADS_TRIGGER = Join-Path $env:LEGO_DUMP_TEXTURE_UPLOADS 'capture.trigger'
+    }
     if ($Mode -eq 'timing') { $env:LEGO_NATIVE_FRAME_METRICS = Join-Path $runRoot "$Label-frames.csv" }
     if ($LongProbe) {
         $env:LEGO_NATIVE_TIMING = '1'
@@ -140,6 +150,8 @@ try {
         stencil=[bool]$Stencil;meshTrace=[bool]$MeshTrace;
         logoUploads=[bool]$LogoUploads;viewport=[bool]$Viewport;bufferWindows=[bool]$BufferWindows;
         nativeDepthAliasEnabled=![bool]$DisableDepthAlias;
+        textureForensics=[bool]$TextureForensics;
+        textureCaptureTrigger=$env:LEGO_DUMP_TEXTURE_UPLOADS_TRIGGER;
         pm4Reference=[bool]$Pm4Reference;
         configuredRenderer=[bool]$ConfiguredRenderer;
         configSnapshot=$configSnapshot;environmentOverrides=$nativeEnvironment;
diff --git a/tools/gpu-aot/test_texture_alpha4_upload.cpp b/tools/gpu-aot/test_texture_alpha4_upload.cpp
new file mode 100644
index 0000000..e6b31e6
--- /dev/null
+++ b/tools/gpu-aot/test_texture_alpha4_upload.cpp
@@ -0,0 +1,56 @@
+#include "../../rexlego/src/gpu_native/texture_alpha4_upload.h"
+#include <cassert>
+#include <iostream>
+#include <vector>
+using legodimensions::gpu_native::CopyTextureAlpha4Blocks;
+
+int main() {
+  uint32_t cases = 0;
+  for (uint32_t width : {1u,2u,3u,4u,5u,16u,31u,32u})
+  for (uint32_t height : {1u,3u,4u,5u,16u,32u})
+  for (uint32_t depth : {1u,3u})
+  for (uint32_t endian = 0; endian < 4; ++endian)
+  for (uint32_t phase = 0; phase < 16; ++phase) {
+    constexpr uint32_t pitch = 256;
+    const uint32_t blocks_x = (width+3)/4, blocks_y = (height+3)/4;
+    // Include a packed-mip start, row padding and Z padding independently of
+    // host row/slice pitch. Permute adjacent blocks to exercise offset use.
+    auto offset = [=](uint32_t x,uint32_t y,uint32_t z)->int64_t {
+      return 128 + uint64_t(z)*2048 + uint64_t(y)*128 + (x^1u)*8;
+    };
+    std::vector<uint8_t> source(size_t(offset(blocks_x-1,blocks_y-1,depth-1))+32,0xCD);
+    // x^1 may address one block beyond the final logical X.
+    source.resize(128+depth*2048,0xCD);
+    std::vector<uint8_t> result(pitch*height*depth,0xFE);
+    for(uint32_t z=0;z<depth;++z)
+    for(uint32_t y=0;y<blocks_y;++y)
+    for(uint32_t x=0;x<blocks_x;++x)
+    for(uint32_t t=0;t<16;++t){
+      const uint8_t nibble = uint8_t((phase+t+x*3+y*5+z*7)&15);
+      const size_t at=(size_t(offset(x,y,z))+t/2)^endian;
+      const uint32_t shift=(t&1)*4;
+      source[at]=uint8_t((source[at]&~(15<<shift))|(nibble<<shift));
+    }
+    assert(CopyTextureAlpha4Blocks(source,result,width,height,depth,pitch,endian,offset));
+    for(uint32_t z=0;z<depth;++z)
+    for(uint32_t y=0;y<height;++y){
+      for(uint32_t x=0;x<width;++x){
+        const uint32_t t=(y%4)*4+x%4;
+        const uint8_t expected=uint8_t(((phase+t+(x/4)*3+(y/4)*5+z*7)&15)*17);
+        assert(result[(z*height+y)*pitch+x]==expected);
+      }
+      assert(result[(z*height+y)*pitch+width]==0xFE);
+    }
+    ++cases;
+  }
+  std::vector<uint8_t> block(8), output(16,0xFE);
+  auto linear=[](auto,auto,auto){return int64_t{0};};
+  assert(!CopyTextureAlpha4Blocks(std::span<const uint8_t>(block).first(7),output,4,4,1,4,0,linear));
+  assert(!CopyTextureAlpha4Blocks(block,std::span<uint8_t>(output).first(15),4,4,1,4,0,linear));
+  assert(!CopyTextureAlpha4Blocks(block,output,4,4,1,3,0,linear));
+  assert(!CopyTextureAlpha4Blocks(block,output,4,4,1,4,4,linear));
+  assert(!CopyTextureAlpha4Blocks(block,output,4,4,1,4,0,[](auto,auto,auto){return int64_t{-1};}));
+  assert(!CopyTextureAlpha4Blocks(block,output,4,4,1,4,0,[](auto,auto,auto){return INT64_MAX;}));
+  assert(!CopyTextureAlpha4Blocks(block,output,UINT32_MAX,UINT32_MAX,UINT32_MAX,UINT32_MAX,0,linear));
+  std::cout<<"PASS: "<<cases<<" alpha4 cases, endian modes, packed offsets, odd edges, padded slices and bounded reads\n";
+}
diff --git a/tools/gpu-aot/test_texture_instruction_audit.py b/tools/gpu-aot/test_texture_instruction_audit.py
new file mode 100644
index 0000000..3f597e2
--- /dev/null
+++ b/tools/gpu-aot/test_texture_instruction_audit.py
@@ -0,0 +1,53 @@
+"""Exercise EXEC traversal independently of real game shader assets."""
+import struct
+import importlib.util
+from pathlib import Path
+
+spec = importlib.util.spec_from_file_location('texture_audit',
+    Path(__file__).with_name('audit_texture_instructions.py'))
+audit = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(audit)
+fetches = audit.fetches
+
+
+def container(instructions):
+    code = b''.join(struct.pack('>III', *words) for words in instructions)
+    header = bytearray(36)
+    struct.pack_into('>III', header, 0, 0x102A1100, 36, len(code))
+    struct.pack_into('>I', header, 24, 28)
+    struct.pack_into('>II', header, 28, 0, len(code))
+    return bytes(header) + bytes(4) + code
+
+
+def pair(a, b):
+    return a & 0xffffffff, (a >> 32) | ((b & 65535) << 16), b >> 16
+
+
+def execute(address, count, sequence, opcode=1):
+    return (opcode << 44) | (sequence << 16) | (count << 12) | address
+
+
+# Both 48-bit halves execute, and duplicate EXEC references are deduplicated.
+# Instruction 3 resembles a texture fetch, but EXEC identifies it as ALU.
+texture = (1 | (7 << 20) | (1 << 25), (3 << 12) | (2 << 14) | (1 << 16),
+           (2 << 14) | 1 | (0x78 << 2))
+data = container([pair(execute(2, 3, 0b010001), execute(2, 1, 1, 3)),
+                  pair(execute(5, 1, 1, 13), 0),
+                  texture, texture, (0, 0, 0), (24, 0, 0)])
+stage, records = fetches(data)
+assert stage == 0
+assert [r['instruction'] for r in records] == [2, 5]
+assert [r['opcode'] for r in records] == [1, 24]
+assert records[0]['slot'] == 7 and records[0]['denormalized']
+assert records[0]['dimension'] == 2 and records[0]['register_gradients']
+assert records[0]['instruction_lod_bias'] == -0.5
+assert [records[0][f] for f in ('mag_filter', 'min_filter', 'mip_filter')] == [3, 2, 1]
+
+for broken in (data[:-1], container([pair(execute(2, 1, 1), 0)])):
+    try:
+        fetches(broken)
+    except ValueError:
+        pass
+    else:
+        raise AssertionError('Malformed extent accepted')
+print('PASS: paired EXEC, conditional EXEC, deduplication, ALU/vertex exclusion, fields and malformed extents')
diff --git a/tools/gpu-aot/test_texture_load_gpu.cpp b/tools/gpu-aot/test_texture_load_gpu.cpp
new file mode 100644
index 0000000..042e92e
--- /dev/null
+++ b/tools/gpu-aot/test_texture_load_gpu.cpp
@@ -0,0 +1,141 @@
+// Standalone D3D12 numeric oracle. No game, window, swap chain or user input.
+#include <d3d12.h>
+#include <dxgi1_6.h>
+#include <wrl/client.h>
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <vector>
+#include "../../rexlego/src/gpu_native/texture_alpha4_upload.h"
+#include "../../rexglue-sdk/src/graphics/shaders/bytecode/d3d12_5_1/texture_load_dxt3a_cs.h"
+#include "../../rexglue-sdk/src/graphics/shaders/bytecode/d3d12_5_1/texture_load_64bpb_cs.h"
+#include "oracle_tiled_offsets.h"
+using Microsoft::WRL::ComPtr;
+using namespace legodimensions::gpu_native;
+
+int main() {
+  ComPtr<IDXGIFactory4> factory; assert(SUCCEEDED(CreateDXGIFactory1(IID_PPV_ARGS(&factory))));
+  ComPtr<IDXGIAdapter1> adapter; assert(SUCCEEDED(factory->EnumAdapters1(0,&adapter)));
+  DXGI_ADAPTER_DESC1 adapter_desc; assert(SUCCEEDED(adapter->GetDesc1(&adapter_desc)));
+  std::wcout<<adapter_desc.Description<<L"\n";
+  ComPtr<ID3D12Device> device;
+  assert(SUCCEEDED(D3D12CreateDevice(adapter.Get(),D3D_FEATURE_LEVEL_11_0,IID_PPV_ARGS(&device))));
+  D3D12_DESCRIPTOR_RANGE ranges[2]{};
+  ranges[0].RangeType=D3D12_DESCRIPTOR_RANGE_TYPE_SRV; ranges[0].NumDescriptors=1;
+  ranges[1].RangeType=D3D12_DESCRIPTOR_RANGE_TYPE_UAV; ranges[1].NumDescriptors=1;
+  D3D12_ROOT_PARAMETER params[3]{};
+  params[0].ParameterType=D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS;
+  params[0].Constants.Num32BitValues=10;
+  for(int i=0;i<2;++i){
+    params[i+1].ParameterType=D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE;
+    params[i+1].DescriptorTable={1,&ranges[i]};
+  }
+  D3D12_ROOT_SIGNATURE_DESC desc{3,params,0,nullptr,D3D12_ROOT_SIGNATURE_FLAG_NONE};
+  ComPtr<ID3DBlob> serialized,errors;
+  assert(SUCCEEDED(D3D12SerializeRootSignature(&desc,D3D_ROOT_SIGNATURE_VERSION_1,&serialized,&errors)));
+  ComPtr<ID3D12RootSignature> root;
+  assert(SUCCEEDED(device->CreateRootSignature(0,serialized->GetBufferPointer(),serialized->GetBufferSize(),IID_PPV_ARGS(&root))));
+  D3D12_COMMAND_QUEUE_DESC queue_desc{};
+  ComPtr<ID3D12CommandQueue> queue; assert(SUCCEEDED(device->CreateCommandQueue(&queue_desc,IID_PPV_ARGS(&queue))));
+  ComPtr<ID3D12Fence> fence; assert(SUCCEEDED(device->CreateFence(0,D3D12_FENCE_FLAG_NONE,IID_PPV_ARGS(&fence))));
+  HANDLE event=CreateEvent(nullptr,FALSE,FALSE,nullptr); assert(event);
+  uint64_t fence_value=0;
+  auto buffer=[&](size_t size,D3D12_HEAP_TYPE type,D3D12_RESOURCE_STATES state,bool uav){
+    D3D12_HEAP_PROPERTIES heap{};heap.Type=type;
+    D3D12_RESOURCE_DESC r{};r.Dimension=D3D12_RESOURCE_DIMENSION_BUFFER;
+    r.Width=size;r.Height=1;r.DepthOrArraySize=1;r.MipLevels=1;r.SampleDesc.Count=1;
+    r.Layout=D3D12_TEXTURE_LAYOUT_ROW_MAJOR;if(uav)r.Flags=D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
+    ComPtr<ID3D12Resource> result;
+    assert(SUCCEEDED(device->CreateCommittedResource(&heap,D3D12_HEAP_FLAG_NONE,&r,state,nullptr,IID_PPV_ARGS(&result))));
+    return result;
+  };
+  uint32_t tested=0;
+  for(const auto dimensions : {std::pair{32u,32u},std::pair{128u,64u},std::pair{1024u,512u}})
+  for(bool alpha : {false,true})
+  for(uint32_t tiled=0;tiled<2;++tiled)
+  for(uint32_t endian=0;endian<4;++endian){
+    const uint32_t width=dimensions.first,height=dimensions.second;
+    const uint32_t pitch=((alpha?width:(width/4)*8)+255)&~255u;
+    const uint32_t guest_pitch_blocks=((width/4)+31)&~31u;
+    const uint32_t source_size=2*1024*1024,output_size=pitch*height;
+    const uint32_t xor_value=endian==1?1:endian==2?3:endian==3?2:0;
+    const uint32_t packed_x=4,packed_y=4;
+    auto offset=[=](uint32_t x,uint32_t y,uint32_t)->int64_t{
+      return tiled?GetTiledOffset2D(x+packed_x,y+packed_y,guest_pitch_blocks,3)
+          : (y+packed_y)*guest_pitch_blocks*8+(x+packed_x)*8;
+    };
+    std::vector<uint8_t> source(source_size,0),expected(output_size,0);
+    for(uint32_t y=0;y<height/4;++y)
+    for(uint32_t x=0;x<width/4;++x)
+    for(uint32_t b=0;b<8;++b)
+      source[(size_t(offset(x,y,0))+b)^xor_value]=uint8_t(b*31+x*13+y*7);
+    if(alpha){
+      assert(CopyTextureAlpha4Blocks(source,expected,width,height,1,pitch,xor_value,offset));
+    }else{
+      for(uint32_t y=0;y<height/4;++y)
+      for(uint32_t x=0;x<width/4;++x)
+      for(uint32_t b=0;b<8;++b)
+        expected[y*pitch+x*8+b]=source[(size_t(offset(x,y,0))+b)^xor_value];
+    }
+    // GPU loader starts at packed-mip offsets via the guest byte offset. A
+    // tiled offset isn't linear in X/Y, so cover packed positions separately
+    // in CPU tests and compare a zero-origin tiled texture here.
+    if(tiled){
+      std::vector<uint8_t> origin(source_size,0);
+      for(uint32_t y=0;y<height/4;++y)
+      for(uint32_t x=0;x<width/4;++x)
+      for(uint32_t b=0;b<8;++b)
+        origin[(size_t(GetTiledOffset2D(x,y,guest_pitch_blocks,3))+b)^xor_value]=source[(size_t(offset(x,y,0))+b)^xor_value];
+      source=std::move(origin);
+    }
+    auto input=buffer(source_size,D3D12_HEAP_TYPE_UPLOAD,D3D12_RESOURCE_STATE_GENERIC_READ,false);
+    auto output=buffer(output_size,D3D12_HEAP_TYPE_DEFAULT,D3D12_RESOURCE_STATE_UNORDERED_ACCESS,true);
+    auto readback=buffer(output_size,D3D12_HEAP_TYPE_READBACK,D3D12_RESOURCE_STATE_COPY_DEST,false);
+    void* mapped;D3D12_RANGE no_read{0,0};assert(SUCCEEDED(input->Map(0,&no_read,&mapped)));
+    std::memcpy(mapped,source.data(),source.size());input->Unmap(0,nullptr);
+    D3D12_DESCRIPTOR_HEAP_DESC heap_desc{D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV,2,D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE,0};
+    ComPtr<ID3D12DescriptorHeap> heap;assert(SUCCEEDED(device->CreateDescriptorHeap(&heap_desc,IID_PPV_ARGS(&heap))));
+    const auto step=device->GetDescriptorHandleIncrementSize(heap_desc.Type);
+    auto cpu=heap->GetCPUDescriptorHandleForHeapStart();
+    D3D12_SHADER_RESOURCE_VIEW_DESC srv{};srv.Format=DXGI_FORMAT_R32G32B32A32_UINT;
+    srv.ViewDimension=D3D12_SRV_DIMENSION_BUFFER;srv.Shader4ComponentMapping=D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING;
+    srv.Buffer.NumElements=source_size/16;device->CreateShaderResourceView(input.Get(),&srv,cpu);
+    cpu.ptr+=step;
+    D3D12_UNORDERED_ACCESS_VIEW_DESC uav{};uav.Format=srv.Format;uav.ViewDimension=D3D12_UAV_DIMENSION_BUFFER;
+    uav.Buffer.NumElements=output_size/16;device->CreateUnorderedAccessView(output.Get(),nullptr,&uav,cpu);
+    D3D12_COMPUTE_PIPELINE_STATE_DESC pipeline_desc{};pipeline_desc.pRootSignature=root.Get();
+    pipeline_desc.CS=alpha?D3D12_SHADER_BYTECODE{texture_load_dxt3a_cs,sizeof(texture_load_dxt3a_cs)}
+        :D3D12_SHADER_BYTECODE{texture_load_64bpb_cs,sizeof(texture_load_64bpb_cs)};
+    ComPtr<ID3D12PipelineState> pipeline;assert(SUCCEEDED(device->CreateComputePipelineState(&pipeline_desc,IID_PPV_ARGS(&pipeline))));
+    ComPtr<ID3D12CommandAllocator> allocator;assert(SUCCEEDED(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT,IID_PPV_ARGS(&allocator))));
+    ComPtr<ID3D12GraphicsCommandList> commands;
+    assert(SUCCEEDED(device->CreateCommandList(0,D3D12_COMMAND_LIST_TYPE_DIRECT,allocator.Get(),pipeline.Get(),IID_PPV_ARGS(&commands))));
+    commands->SetComputeRootSignature(root.Get());ID3D12DescriptorHeap* heaps[]={heap.Get()};commands->SetDescriptorHeaps(1,heaps);
+    uint32_t constants[]={tiled|(endian<<2),tiled?0u:uint32_t(offset(0,0,0)),
+        tiled?guest_pitch_blocks:guest_pitch_blocks*8,32,width/4,height/4,1,0,pitch,height};
+    commands->SetComputeRoot32BitConstants(0,10,constants,0);
+    auto gpu=heap->GetGPUDescriptorHandleForHeapStart();commands->SetComputeRootDescriptorTable(1,gpu);
+    gpu.ptr+=step;commands->SetComputeRootDescriptorTable(2,gpu);
+    commands->Dispatch(((width/4)+15)/16,((height/4)+31)/32,1);
+    D3D12_RESOURCE_BARRIER barrier{};barrier.Type=D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
+    barrier.Transition={output.Get(),D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,D3D12_RESOURCE_STATE_UNORDERED_ACCESS,D3D12_RESOURCE_STATE_COPY_SOURCE};
+    commands->ResourceBarrier(1,&barrier);commands->CopyBufferRegion(readback.Get(),0,output.Get(),0,output_size);
+    assert(SUCCEEDED(commands->Close()));ID3D12CommandList* lists[]={commands.Get()};queue->ExecuteCommandLists(1,lists);
+    assert(SUCCEEDED(queue->Signal(fence.Get(),++fence_value)));
+    assert(SUCCEEDED(fence->SetEventOnCompletion(fence_value,event)));
+    assert(WaitForSingleObject(event,30000)==WAIT_OBJECT_0);
+    D3D12_RANGE read_range{0,output_size};assert(SUCCEEDED(readback->Map(0,&read_range,&mapped)));
+    const uint32_t rows=alpha?height:height/4,bytes=alpha?width:(width/4)*8;
+    for(uint32_t y=0;y<rows;++y)for(uint32_t x=0;x<bytes;++x){
+      const auto actual=static_cast<uint8_t*>(mapped)[y*pitch+x];
+      if(actual!=expected[y*pitch+x]){
+        std::cerr<<"mismatch alpha="<<alpha<<" tiled="<<tiled<<" endian="<<endian<<" x="<<x<<" y="<<y
+            <<" expected="<<unsigned(expected[y*pitch+x])<<" got="<<unsigned(actual)<<"\n";
+        return 1;
+      }
+    }
+    readback->Unmap(0,&no_read);++tested;
+  }
+  CloseHandle(event);
+  std::cout<<"PASS: "<<tested<<" actual SDK D3D12 texture-loader comparisons; no game launched\n";
+}
diff --git a/tools/gpu-aot/test_texture_load_gpu.py b/tools/gpu-aot/test_texture_load_gpu.py
new file mode 100644
index 0000000..37ba33a
--- /dev/null
+++ b/tools/gpu-aot/test_texture_load_gpu.py
@@ -0,0 +1,30 @@
+"""Run actual SDK texture loader bytecode on D3D12 with synthetic input only."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+
+p=argparse.ArgumentParser(description=__doc__)
+p.add_argument('output',type=Path)
+a=p.parse_args();a.output.mkdir(parents=True,exist_ok=True)
+root=Path(__file__).resolve().parents[2]
+s=(root/'rexglue-sdk/src/graphics/pipeline/texture/util.cpp').read_text()
+start=s.index('int32_t GetTiledOffset2D(')
+end=s.index('\n}\n',start)+3
+body=s[start:end].replace('rex::align(pitch, xenos::kTextureTileWidthHeight)','((pitch + 31u) & ~31u)')
+(a.output/'oracle_tiled_offsets.h').write_text('#include <cstdint>\n'+body)
+exe=a.output/'test.exe'
+subprocess.run(['clang++','-std=c++20','-O2','-I'+str(a.output.resolve()),
+    str(root/'tools/gpu-aot/test_texture_load_gpu.cpp'),'-ld3d12','-ldxgi','-o',str(exe)],check=True)
+result = subprocess.run([str(exe.resolve())], check=True, capture_output=True, text=True)
+print(result.stdout, end='')
+(a.output/'gpu-result.txt').write_text(result.stdout + result.stderr)
+(a.output/'verification.json').write_text(json.dumps({
+    'passed': True,
+    'output': result.stdout.strip(),
+    'oracle_headers': {name: hashlib.sha256((root/'rexglue-sdk/src/graphics/shaders/bytecode/d3d12_5_1'/name).read_bytes()).hexdigest()
+        for name in ('texture_load_64bpb_cs.h', 'texture_load_dxt3a_cs.h')},
+    'game_launched': False,
+    'game_visual_validation': False,
+}, indent=2) + '\n')
diff --git a/tools/gpu-aot/test_texture_swizzle.py b/tools/gpu-aot/test_texture_swizzle.py
new file mode 100644
index 0000000..ea38432
--- /dev/null
+++ b/tools/gpu-aot/test_texture_swizzle.py
@@ -0,0 +1,64 @@
+"""Compare native channel composition against the actual SDK table/function.
+
+No SDK files are modified, no game runs, and no game data is required.
+"""
+import argparse
+import json
+from pathlib import Path
+import re
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+a.output.mkdir(parents=True, exist_ok=True)
+root = Path(__file__).resolve().parents[2]
+sdk = root/'rexglue-sdk'
+ids = {name:int(value) for name,value in re.findall(r'(k_\w+) = (\d+)',
+    (sdk/'include/rex/graphics/xenos.h').read_text().split('enum class TextureFormat')[1].split('};')[0])}
+table = (sdk/'src/graphics/d3d12/texture_cache.cpp').read_text()
+oracle = {}
+for name,entry in re.findall(r'// (k_\w+)\s*\n(?:\s*//[^\n]*\n)*\s*\{([^}]+)\}',table):
+    match = re.search(r'XE_GPU_TEXTURE_SWIZZLE_([RGBA]{4})\s*$',entry)
+    if name in ids and match:
+        oracle[ids[name]] = sum('RGBA'.index(c) << (3*i) for i,c in enumerate(match[1]))
+formats = [2,8,9,10,13,22,23,25,30,31,36,37,49,58,59,
+           6,14,18,19,20,21,26,32,38,54]
+assert all(f in oracle for f in formats), [f for f in formats if f not in oracle]
+s = (sdk/'src/graphics/pipeline/texture/cache.cpp').read_text()
+start = s.index('uint32_t TextureCache::GuestToHostSwizzle(')
+end = s.index('\n}\n',start)+3
+function = s[start:end].replace('TextureCache::GuestToHostSwizzle','OracleSwizzle')
+function = function.replace('xenos::XE_GPU_TEXTURE_SWIZZLE_0','4u')
+cases = ','.join(f'{{{f},{oracle[f]}}}' for f in formats)
+cpp = a.output/'test.cpp'
+cpp.write_text('''#include "texture_swizzle.h"
+#include <cassert>
+#include <cstdint>
+#include <iostream>
+using namespace legodimensions::gpu_native;
+'''+function+'''
+int main(){
+ struct Case{uint32_t format,channels;};
+ Case cases[]={'''+cases+'''};
+ for(auto c:cases){
+  assert(TextureFormatChannelSwizzle(c.format)==c.channels);
+  for(uint32_t guest=0;guest<4096;++guest)
+   assert(NativeTextureSwizzle(c.format,guest)==OracleSwizzle(guest,c.channels));
+ }
+ assert(NativeTextureSwizzle(2,0x688)==0); // Scalar alpha was previously filled with 1.
+ assert(NativeTextureSwizzle(49,0x688)==0x248); // DXN blue/alpha replicate G.
+ std::cout<<"PASS: actual SDK composition and format table, all 4096 fetch swizzles per format\\n";
+}
+''')
+exe = a.output/'test.exe'
+subprocess.run(['clang++','-std=c++20','-O2','-I'+str(root/'rexlego/src/gpu_native'),
+                str(cpp),'-o',str(exe)],check=True)
+subprocess.run([str(exe.resolve())],check=True)
+draw = (root/'rexlego/src/gpu_native/textures.cpp').read_text()
+assert 'NativeTextureSwizzle(resource->guest_format, swizzle)' in draw
+assert 'swizzles[(host_swizzle >> 9) & 7]' in draw
+(a.output/'verification.json').write_text(json.dumps({'passed':True,
+    'formats':formats,'composition_cases':len(formats)*4096,
+    'oracle':'actual read-only SDK format table and GuestToHostSwizzle body',
+    'game_visual_validation':False},indent=2)+'\n')
diff --git a/tools/gpu-aot/test_texture_upload_capture.py b/tools/gpu-aot/test_texture_upload_capture.py
index b69cc1c..c3e9875 100644
--- a/tools/gpu-aot/test_texture_upload_capture.py
+++ b/tools/gpu-aot/test_texture_upload_capture.py
@@ -11,32 +11,64 @@
 a.output.mkdir(parents=True,exist_ok=True)
 root=Path(__file__).resolve().parents[2]
 s=(root/'rexlego/src/gpu_native/textures.cpp').read_text()
-start=s.index('void DumpTextureUpload(')
+start=s.index('struct TextureCaptureState {')
 end=s.index('\nbool EnsureMirror',start)
 function=s[start:end]
 cpp=a.output/'test.cpp'
 cpp.write_text(r'''
 #include <cstdint>
+#include <atomic>
+#include <array>
 #include <cstdlib>
 #include <filesystem>
 #include <fstream>
 #include <mutex>
+#include <span>
 #include <string>
 #include <unordered_set>
 #include <vector>
 using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;
+std::atomic<u32> g_probe_frame{0};
 namespace plume {enum class RenderFormat {BC1_UNORM,BC3_UNORM,R8G8B8A8_UNORM};}
 enum class ShaderStage {kVertex,kPixel};
 u64 vs=0,ps=0;
 u64 BoundShaderHash(ShaderStage stage){return stage==ShaderStage::kVertex?vs:ps;}
-struct Fetch {u32 base_address=0,mip_address=0,swizzle=0x688,endianness=0,pitch=1,tiled=1,packed_mips=0;};
-struct TextureResource {plume::RenderFormat format;u32 guest_address,width=5,height=7,guest_format=20;Fetch guest_fetch;};
+struct Fetch {u32 base_address=0,mip_address=0,swizzle=0x688,endianness=0,pitch=1,tiled=1,packed_mips=0,dimension=1;};
+struct TextureResource {plume::RenderFormat format;u32 guest_address,width=5,height=7,guest_format=20,levels=1;Fetch guest_fetch;};
 u32 RowPitch(plume::RenderFormat,u32){return 256;}
 u32 RowCount(plume::RenderFormat f,u32 h){return f==plume::RenderFormat::R8G8B8A8_UNORM?h:(h+3)/4;}
 #define REXLOG_INFO(...) ((void)0)
 '''+function+r'''
 int main(int argc,char**argv){
  _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS",argv[1]);
+ if(argc>2){
+  _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS_ALL_LEVELS","1");
+  _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS_TRIGGER",argv[2]);
+  TextureResource r{plume::RenderFormat::BC1_UNORM,42,64,16,18,7};
+  std::vector<u8> data(8192,0xCD),base(4096,0xAB),mips(2048,0xEF);
+  u32 offset=0;
+  for(u32 level=0;level<7;++level){
+   offset=(offset+511)&~511u;
+   u32 w=std::max(1u,64u>>level),h=std::max(1u,16u>>level);
+   u32 rows=RowCount(r.format,h),bytes=((w+3)/4)*8;
+   for(u32 row=0;row<rows;++row)for(u32 b=0;b<bytes;++b)data[offset+row*256+b]=u8(level*23+row*7+b);
+   offset+=256*rows;
+  }
+  if(TextureUploadCapturePending(r))return 1;
+  DumpTextureUpload(r,data.data(),base,mips);
+  if(std::filesystem::exists(std::filesystem::path(argv[1])/"42.dds"))return 2;
+  std::ofstream(argv[2])<<"armed";
+  ++g_probe_frame;
+  if(!TextureUploadCapturePending(r))return 3;
+  DumpTextureUpload(r,data.data(),base,mips);
+  if(TextureUploadCapturePending(r))return 4;
+  r.guest_address=43;r.guest_fetch.dimension=2;
+  if(TextureUploadCapturePending(r))return 5;
+  r.guest_fetch.dimension=1;
+  TextureCapture().written_bytes=128ull*1024*1024;
+  if(TextureUploadCapturePending(r))return 6;
+  return 0;
+ }
  _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS_LOGOS_ONLY","1");
  std::vector<u8> data(256*7);
  for(size_t i=0;i<data.size();++i)data[i]=u8(i/256*31+i%256);
@@ -61,7 +93,25 @@
     assert len(b)==128+row_bytes*rows
     expected=bytes((row*31+column)&255 for row in range(rows) for column in range(row_bytes))
     assert b[128:]==expected
+all_data=a.output/'all-mips'
+trigger=a.output/'arm-capture'
+trigger.unlink(missing_ok=True)
+subprocess.run([str(exe.resolve()),str(all_data.resolve()),str(trigger.resolve())],check=True)
+b=(all_data/'42.dds').read_bytes()
+assert struct.unpack_from('<I',b,28)[0]==7
+expected=bytearray()
+for level in range(7):
+    w,h=max(1,64>>level),max(1,16>>level)
+    for row in range((h+3)//4):
+        expected.extend((level*23+row*7+column)&255 for column in range(((w+3)//4)*8))
+assert b[128:]==expected
+assert (all_data/'42.guest-base.bin').read_bytes()==bytes([0xAB])*4096
+assert (all_data/'42.guest-mips.bin').read_bytes()==bytes([0xEF])*2048
+assert 'levels=7' in (all_data/'42.txt').read_text()
+assert not (all_data/'43.dds').exists()
 (a.output/'verification.json').write_text(json.dumps({'passed':True,'formats':['BC3/DXT5','BC1/DXT1','RGBA8'],
-    'checks':['odd dimensions','256-byte row padding stripped','exact payload preserved','both TT shader variants','unrelated shader filtered'],
+    'checks':['odd dimensions','256-byte row padding stripped','exact payload preserved','both TT shader variants','unrelated shader filtered',
+              'late trigger','cached-upload capture request','all seven packed-tail mip sizes','512-byte mip gaps stripped',
+              'exact raw guest backing','volume refused','128MiB budget'],
     'game_launched':False},indent=2)+'\n')
 print('PASS: DXT5/DXT1/RGBA capture preserves exact rows and filters unrelated shaders')

From cbd2c399d59f6874f595be55f043bf3468a3bede Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 15:12:00 +0300
Subject: [PATCH 40/44] Prepare isolated texture gradient translation and audit
 affected LOTR materials

---
 tools/gpu-aot/audit_texture_instructions.py   |  1 +
 .../gpu-aot/native-texture-format-20261004.md | 24 ++++++
 tools/gpu-aot/prepare_gradients_candidate.py  | 73 +++++++++++++++++
 tools/gpu-aot/test_texture_gradients.py       | 79 +++++++++++++++++++
 4 files changed, 177 insertions(+)
 create mode 100644 tools/gpu-aot/prepare_gradients_candidate.py
 create mode 100644 tools/gpu-aot/test_texture_gradients.py

diff --git a/tools/gpu-aot/audit_texture_instructions.py b/tools/gpu-aot/audit_texture_instructions.py
index aad81e4..b1d62fe 100644
--- a/tools/gpu-aot/audit_texture_instructions.py
+++ b/tools/gpu-aot/audit_texture_instructions.py
@@ -100,6 +100,7 @@ def main():
             op = record['opcode']
             counts[f'opcode_{op}'] += 1
             if op != 1:
+                interesting.append(record)
                 continue
             for field in ['denormalized', 'register_gradients']:
                 if record[field]:
diff --git a/tools/gpu-aot/native-texture-format-20261004.md b/tools/gpu-aot/native-texture-format-20261004.md
index 9aa1aae..b79926e 100644
--- a/tools/gpu-aot/native-texture-format-20261004.md
+++ b/tools/gpu-aot/native-texture-format-20261004.md
@@ -63,3 +63,27 @@ row/mip padding, exact guest backing, legacy logo filtering and capture limits.
 Portraits, vehicle icons, button prompts, LOTR artifacts, missing shaders and
 three-level/cutscene correctness remain open. Internal rendering remains720p;
 actual1080p and stable60FPS are not achieved or measured by these tests.
+
+## Isolated gradient instruction correction
+
+The audit now records non-sampling texture operations with their asset paths.
+The production compiler silently skips GetTextureGradients (opcode18). The
+read-only SDK D3D12 translator computes coarse derivatives in the order
+dx(source.x),dy(source.x),dx(source.y),dy(source.y), independent of a texture.
+An isolated compiler prepared by `prepare_gradients_candidate.py` adds this
+operation with source/destination swizzles, destination0/1/keep and instruction
+predication. It refuses vertex and relative-address forms rather than silently
+emitting an incorrect result. Shared ABI remains624; no LOD change is included.
+
+`test_texture_gradients.py COMPILER COMMON OUTPUT` passes50 actual synthetic
+microcode/HLSL/validated-DXIL fixtures and three unsupported-mode refusals.
+Source r7 is initialized from SV_Position so XY derivatives vary. All89
+affected actual asset containers compile, with all100 gradient assignments
+present in emitted HLSL. Four belong to hub_lotr_05; this does not prove these
+are the draws in the owner's failed screenshot. Avengers avgref has no PC
+signature twin for the queried LOTR representative157; no substitute shader
+was copied or treated as ground truth. Candidate outputs remain ignored under
+session-20261004/gradients-candidate and are **not installed** into production.
+All250 captured runtime shaders also compile with byte-identical emitted HLSL
+against the prior production compiler. None uses GetTextureGradients, so this
+is a useful check that the isolated change leaves those existing draws alone.
diff --git a/tools/gpu-aot/prepare_gradients_candidate.py b/tools/gpu-aot/prepare_gradients_candidate.py
new file mode 100644
index 0000000..1f6368e
--- /dev/null
+++ b/tools/gpu-aot/prepare_gradients_candidate.py
@@ -0,0 +1,73 @@
+"""Prepare isolated GetTextureGradients translation without installing shaders."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('base_source', type=Path)
+p.add_argument('scratch', type=Path)
+a = p.parse_args()
+source = a.scratch.resolve()/'source'
+source.mkdir(parents=True, exist_ok=True)
+manifest = {}
+for path in a.base_source.iterdir():
+    if path.suffix in ('.cpp', '.h') or path.name == 'CMakeLists.txt':
+        shutil.copyfile(path, source/path.name)
+        manifest[path.name] = hashlib.sha256(path.read_bytes()).hexdigest()
+compiler = (source/'shader_recompiler.cpp').read_text()
+
+def replace_once(old, new):
+    global compiler
+    if compiler.count(old) != 1:
+        raise ValueError('Expected one source anchor: ' + old[:100])
+    compiler = compiler.replace(old, new)
+
+replace_once('if (instr.opcode != FetchOpcode::TextureFetch && instr.opcode != FetchOpcode::GetTextureWeights)\n        return;',
+    'if (instr.opcode != FetchOpcode::TextureFetch && instr.opcode != FetchOpcode::GetTextureWeights &&\n'
+    '        instr.opcode != FetchOpcode::GetTextureGradients)\n        return;')
+replace_once('    std::string constName;\n', '''    if (instr.opcode == FetchOpcode::GetTextureGradients)
+    {
+        if (!isPixelShader || instr.srcRegisterAm || instr.dstRegisterAm)
+            throw std::runtime_error("Gradient candidate: vertex/relative addressing unsupported");
+        bool writesGradient = false;
+        for (size_t lane = 0; lane < 4; ++lane)
+            writesGradient |= getDestSwizzle(instr.dstSwizzle, lane) <= FetchDestinationSwizzle::W;
+        if (writesGradient)
+        {
+            indent();
+            print("r{}.", instr.dstRegister);
+            printDstSwizzle(instr.dstSwizzle, false);
+            out += " = float4(ddx_coarse(";
+            printSrcRegister(1);
+            out += "), ddy_coarse(";
+            printSrcRegister(1);
+            out += "), ddx_coarse(";
+            print("r{}.{}", instr.srcRegister, SWIZZLES[(instr.srcSwizzle >> 2) & 3]);
+            out += "), ddy_coarse(";
+            print("r{}.{}", instr.srcRegister, SWIZZLES[(instr.srcSwizzle >> 2) & 3]);
+            out += ")).";
+            printDstSwizzle(instr.dstSwizzle, true);
+            out += ";\\n";
+        }
+        printDstSwizzle01(instr.dstRegister, instr.dstSwizzle);
+        if (instr.isPredicated)
+        {
+            --indentation;
+            indent();
+            out += "}\\n";
+        }
+        return;
+    }
+
+    std::string constName;
+''')
+(source/'shader_recompiler.cpp').write_text(compiler)
+cmake = (source/'CMakeLists.txt').read_text().replace('alpha_compiler', 'gradient_compiler')
+(source/'CMakeLists.txt').write_text(cmake)
+(a.scratch/'source-proof.json').write_text(json.dumps({
+    'base_sha256': manifest, 'shared_bytes': 624,
+    'semantics': 'coarse derivatives in dx(x),dy(x),dx(y),dy(y) order; source/destination swizzle and predication',
+    'installed': False, 'game_launched': False}, indent=2)+'\n')
+print(source)
diff --git a/tools/gpu-aot/test_texture_gradients.py b/tools/gpu-aot/test_texture_gradients.py
new file mode 100644
index 0000000..7a1d407
--- /dev/null
+++ b/tools/gpu-aot/test_texture_gradients.py
@@ -0,0 +1,79 @@
+"""Synthetic microcode -> HLSL -> validated DXIL gradient candidate checks."""
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import struct
+import subprocess
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('compiler', type=Path)
+p.add_argument('common', type=Path)
+p.add_argument('scratch', type=Path)
+a = p.parse_args()
+
+def fixture(swizzle=4, dst=0x688, predicate=None, relative=0, vertex=False):
+    data = bytearray(164)
+    struct.pack_into('>9I', data, 0, 0x102A1100 | int(vertex), 128, 36, 0, 36, 0, 80, 0, 0)
+    struct.pack_into('>8I', data, 36, 32, 28, 0, 0, 0, 0, 0, 0)
+    # r7 receives SV_Position, so non-predicated XY derivative fixtures operate
+    # on a varying input rather than only compiling derivatives of zero.
+    struct.pack_into('>8I', data, 80, 0, 36, 0, 7 << 8, 0, 0, 0, 1)
+    struct.pack_into('>3I', data, 128, 1 | (2 << 12) | (1 << 16), 2 << 12, 0)
+    # getGradients r2, r7.<two source components>; no sampler needed.
+    struct.pack_into('>3I', data, 140, 18 | (7 << 5) | (2 << 12) | (swizzle << 26) | relative,
+                     dst | (int(predicate is not None) << 31),
+                     (1 << 14) | (int(bool(predicate)) << 31))
+    struct.pack_into('>3I', data, 152, (62 if vertex else 0) | (1 << 15) | (15 << 16), 0,
+                     2 | (2 << 8) | (2 << 16) | (2 << 24) | (7 << 29))
+    return data
+
+inputs = a.scratch/'input'
+hlsl = a.scratch/'hlsl'
+inputs.mkdir(parents=True, exist_ok=True)
+cases = {}
+for swizzle in range(16):
+    for predicate in (None, False, True):
+        name = f'swizzle{swizzle}_predicate{predicate}'
+        cases[name] = (swizzle, predicate)
+        (inputs/(name+'.bin')).write_bytes(fixture(swizzle=swizzle, predicate=predicate))
+for name, dst in [('reversed', 0x053), ('constants', 4 | (5 << 3) | (7 << 6) | (7 << 9))]:
+    (inputs/(name+'.bin')).write_bytes(fixture(dst=dst))
+env = os.environ.copy()
+env.update(XENOS_RECOMP_DXIL_ONLY='1', XENOS_RECOMP_LEGO_NATIVE_SCALE='1', XENOS_RECOMP_VERBOSE_DXC='1')
+env.pop('XENOS_RECOMP_ONLY_HASH', None)
+
+def run(directory, output, hlsl_dir=None):
+    cmd = [str(a.compiler.resolve()), str(directory.resolve()), str(output.resolve()), str(a.common.resolve())]
+    if hlsl_dir is not None:
+        cmd.append(str(hlsl_dir.resolve()))
+    return subprocess.run(cmd, env=env, capture_output=True, text=True)
+
+result = run(inputs, a.scratch/'fixture-cache.cpp', hlsl)
+if result.returncode:
+    raise RuntimeError(result.stdout+result.stderr)
+for name, (swizzle, predicate) in cases.items():
+    text = (hlsl/(name+'.hlsl')).read_text()
+    assert 'float4 r7 = float4((iPos.xy - 0.5)' in text, name
+    x, y = 'xyzw'[swizzle & 3], 'xyzw'[(swizzle >> 2) & 3]
+    statement = f'r2.xyzw = float4(ddx_coarse(r7.{x}), ddy_coarse(r7.{x}), ddx_coarse(r7.{y}), ddy_coarse(r7.{y})).xyzw;'
+    assert statement in text, name
+    if predicate is not None:
+        assert re.search(r'if \(' + ('' if predicate else '!') + r'p0\)\s*\{\s*' + re.escape(statement), text), name
+reversed_text = (hlsl/'reversed.hlsl').read_text()
+assert ')).wzyx;' in reversed_text
+constant_text = (hlsl/'constants.hlsl').read_text()
+assert 'r2.x = 0.0;' in constant_text and 'r2.y = 1.0;' in constant_text
+assert 'ddx_coarse(' not in constant_text
+for name, opts in [('relative_source', {'relative':1 << 11}),
+                   ('relative_destination', {'relative':1 << 18}), ('vertex', {'vertex':True})]:
+    directory = a.scratch/name
+    directory.mkdir(exist_ok=True)
+    (directory/'fixture.bin').write_bytes(fixture(**opts))
+    rejected = run(directory, a.scratch/(name+'.cpp'))
+    assert rejected.returncode and 'vertex/relative addressing unsupported' in rejected.stderr, name
+(a.scratch/'verification.json').write_text(json.dumps({'passed':True, 'hlsl_dxil_fixtures':50,
+    'source_swizzles':16, 'predication':[None,False,True], 'refusal_cases':3,
+    'shared_bytes':624, 'installed':False, 'game_visual_validation':False}, indent=2)+'\n')
+print('PASS: 50 gradient microcode/HLSL/DXIL fixtures and 3 unsupported-mode refusals')

From c62dcb801ff02708861f5fbbc1ecc522cfeef446 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 15:46:55 +0300
Subject: [PATCH 41/44] Record monitor-3 capture and repair bounded scene
 rearming and alias snapshots

---
 rexlego/src/gpu_native/textures.cpp           | 33 +++++++--
 tools/gpu-aot/native-capture-20261004.md      | 72 +++++++++++++++++++
 .../native-validation-pending-20261003.md     |  8 ++-
 tools/gpu-aot/read_guest_memory.cpp           | 27 +++++--
 tools/gpu-aot/run_native_probe.ps1            | 22 ++++++
 tools/gpu-aot/test_read_guest_memory.py       | 63 ++++++++++++++++
 tools/gpu-aot/test_texture_upload_capture.py  | 36 ++++++++--
 7 files changed, 241 insertions(+), 20 deletions(-)
 create mode 100644 tools/gpu-aot/native-capture-20261004.md
 create mode 100644 tools/gpu-aot/test_read_guest_memory.py

diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp
index a1a972f..c4f6c70 100644
--- a/rexlego/src/gpu_native/textures.cpp
+++ b/rexlego/src/gpu_native/textures.cpp
@@ -471,7 +471,9 @@ u32 LevelOffset(const TextureResource& resource, u32 level) {
 // Opt-in diagnostic: inspect the actual base-level bytes uploaded to D3D12,
 // independently of draw geometry, shaders, swizzle and sampler state.
 struct TextureCaptureState {
-  std::filesystem::path root, trigger;
+  std::filesystem::path root, trigger, active_root;
+  std::string stage;
+  std::unordered_set<std::string> completed_stage_names;
   bool all_levels = false, armed = false;
   u32 checked_frame = ~u32{0};
   u64 written_bytes = 0;
@@ -500,14 +502,37 @@ TextureCaptureState& TextureCapture() {
 // Disabled captures do no filesystem work. The trigger is checked once/frame.
 bool TextureUploadCapturePending(const TextureResource& resource) {
   auto& state = TextureCapture();
-  if (state.root.empty() || state.dumped.size() >= (state.all_levels ? 32u : 8u) ||
-      state.written_bytes >= 128ull * 1024 * 1024 || state.dumped.contains(resource.guest_address)) return false;
+  if (state.root.empty() || state.written_bytes >= 128ull * 1024 * 1024) return false;
   if (!state.trigger.empty() && state.checked_frame != g_probe_frame.load()) {
     state.checked_frame = g_probe_frame.load();
     std::error_code error;
     state.armed = std::filesystem::is_regular_file(state.trigger, error) && !error;
+    if (state.armed && state.all_levels) {
+      // A fresh stage name rearms the per-scene quota. Never reset the global
+      // byte budget or overwrite an earlier scene. Read bounded trigger text.
+      char name[34]{};
+      std::ifstream input(state.trigger, std::ios::binary);
+      input.read(name, sizeof(name));
+      std::string stage(name, size_t(input.gcount()));
+      while (!stage.empty() && (stage.back() == '\n' || stage.back() == '\r')) stage.pop_back();
+      if (stage.empty()) stage = "capture";
+      const bool valid = stage.size() <= 32 && std::all_of(stage.begin(), stage.end(), [](char c) {
+        return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_';
+      });
+      if (!valid || (stage != state.stage &&
+          (state.completed_stage_names.size() >= 4 || state.completed_stage_names.contains(stage)))) {
+        state.armed = false;
+      } else if (stage != state.stage) {
+        state.completed_stage_names.insert(stage);
+        state.stage = std::move(stage);
+        state.active_root = state.root / state.stage;
+        state.dumped.clear();
+      }
+    }
   }
   if (!state.trigger.empty() && !state.armed) return false;
+  if (state.dumped.size() >= (state.all_levels ? 32u : 8u) ||
+      state.dumped.contains(resource.guest_address)) return false;
   if (state.all_levels && (u32(resource.guest_fetch.dimension) != 1 || resource.levels > 16)) return false;
   if (resource.format != plume::RenderFormat::BC1_UNORM &&
       resource.format != plume::RenderFormat::BC3_UNORM &&
@@ -526,7 +551,7 @@ void DumpTextureUpload(const TextureResource& resource, const u8* data,
                        std::span<const u8> base = {}, std::span<const u8> mips = {}) {
   if (!TextureUploadCapturePending(resource)) return;
   auto& state = TextureCapture();
-  const auto& root = state.root;
+  const auto& root = state.active_root.empty() ? state.root : state.active_root;
   const bool bc1 = resource.format == plume::RenderFormat::BC1_UNORM;
   const bool bc3 = resource.format == plume::RenderFormat::BC3_UNORM;
   const bool compressed = bc1 || bc3;
diff --git a/tools/gpu-aot/native-capture-20261004.md b/tools/gpu-aot/native-capture-20261004.md
new file mode 100644
index 0000000..738ad4b
--- /dev/null
+++ b/tools/gpu-aot/native-capture-20261004.md
@@ -0,0 +1,72 @@
+# Monitor3 format/HUD/LOTR diagnostic, October4
+
+Owner reached Vorton and LOTR in diagnostic PID1100, started15:21:49 Moscow.
+Build27794e2f1412daae, exe89d48797bd6944c1724bd6c167952cd8dc93792ecbfadf2cf89dd21692f8ca09.
+The prior manual PID24428 was captured and closed before launching diagnostics.
+No SDK/plugin/config changes; native D3D12, actual720p/output1080p. The saved
+configuration has depth_of_field=false and both runs log the native DoF-disable
+path. Remaining softness is not established to be DoF.
+
+The owner-assisted screenshots still show blank portrait circles, texture
+stripes and tall LOTR artifacts. These are failed visual checks, not repaired
+results. Diagnostic timings are not clean gameplay performance benchmarks.
+PID1100 was closed after capture. No further launch is authorized by this run.
+
+Local evidence under session-20261004/format-forensics1:
+
+- hub.png / lotr.png; saved log and exact process record.
+- hub-events.tsv / lotr-events.tsv; live mask owner/material/backend/header
+  snapshots. One portrait_setup with all fields readable; no matching
+  portrait_bound_draw in the captured trace. That establishes no matched mask
+  binding, not absence of every possible portrait draw or the cause of failure.
+- Automatic full-mip upload dump:32 textures at hub frame2009, with exact
+  guest spans. Its per-process32-texture limit exhausted in the hub, so it did
+  not collect subsequent LOTR textures. Future multi-scene captures need a
+  separate budget/arming design; do not present these as LOTR mip evidence.
+- LOTR:15 recent texture headers, six large BC1/BC3 base memory spans. All six
+  headers remained stable before/after reading. The external reader originally
+  omitted the Windows E/F guest alias+4096 host offset. These raw spans contain
+  a preceding page and lack the last4096 source bytes. Initial decoded-bases
+  PNGs are diagnostic-reader artifacts, not evidence of renderer corruption.
+  The manifest explicitly marks these limitations. Object headers at A/B
+  addresses are unaffected. Do not infer intact whole mips from partial spans.
+- Missing-shader capture contains only the already known A58 memexport
+  container. This run does not support guessing additional shader substitutes.
+
+The SDK's actual mapping code confirmed the reader offset issue. The reader now
+matches Windows E/F translation, supports explicitly bounded spans up to8MiB,
+at most32 addresses/128MiB total and retains executable identity verification.
+test_read_guest_memory.py executes real ReadProcessMemory against an isolated
+synthetic child: ordinary/alias pages,1024/4096 byte reads, invalid limits,
+address overflow and wrong executable all pass. No game is launched by this test.
+
+## Gradient query correlation
+
+Two of the four hub_lotr_05 gradient shaders are actually referenced in this
+run's texture binding trace:
+
+- FBD34779A28C36BE:14 recorded references.
+- D70DC053F574C4C6:13 recorded references.
+
+The other two have no recorded reference. Rows are preserved in
+active-gradient-events.tsv. The isolated GetTextureGradients correction is
+therefore relevant to active LOTR materials; it is not a proven fix for the
+entire screenshot. It preserves the624-byte ABI and does not include rejected
+explicit-LOD changes. Full archive preparation proceeds offline; image
+validation remains pending.
+
+## Offline capture rearming repair
+
+The diagnostic capture now accepts up to four fresh scene names in the trigger
+file, with32 textures per scene and the same128MiB cumulative payload ceiling.
+Scene output directories preserve earlier captures. A reused scene, malformed
+name/path or fifth scene is refused. Trigger checks precede the per-scene quota
+check, so exhausting hub quota cannot prevent later rearming. Legacy logo
+captures retain their existing base-only behavior. Capture remains opt-in.
+
+For a future authorized probe, use `run_native_probe.ps1 -Label LABEL
+-CaptureStage hub`, then a fresh stage such as lotr only after owner readiness.
+This command validates the exact running process and settings, writes bounded
+ASCII stage text through atomic file replacement, and does not launch a game.
+The actual capture-body tests pass scene rearming after quota exhaustion,
+preserved DDS/raw bytes, scene/path/count refusal and retained global budget.
diff --git a/tools/gpu-aot/native-validation-pending-20261003.md b/tools/gpu-aot/native-validation-pending-20261003.md
index f715e88..4a03b1c 100644
--- a/tools/gpu-aot/native-validation-pending-20261003.md
+++ b/tools/gpu-aot/native-validation-pending-20261003.md
@@ -1,8 +1,10 @@
 # Required runtime validation after October 3 offline fixes
 
-Latest October4 instruction: **keep working offline; no launch yet**. Earlier
-launch authorization has been revoked. Monitor3 applies to future authorized
-probes. The owner rejected the combined explicit-LOD candidate; production
+Latest October4 state: the owner assisted the single monitor3 diagnostic
+oct04-format-forensics1 in Vorton and LOTR; that instance has been closed.
+Continue offline; another launch requires owner authorization. See
+native-capture-20261004.md for the failed image checks and evidence limitations.
+The owner rejected the combined explicit-LOD candidate; production
 has returned to the coupled624-byte host/shader ABI. Texture format fixes and
 late-trigger mip forensics have passed offline checks, but are visually untested.
 Use questions for user-assisted hub readiness; no activation or injected input.
diff --git a/tools/gpu-aot/read_guest_memory.cpp b/tools/gpu-aot/read_guest_memory.cpp
index 19de9cb..80529a3 100644
--- a/tools/gpu-aot/read_guest_memory.cpp
+++ b/tools/gpu-aot/read_guest_memory.cpp
@@ -8,16 +8,27 @@
 #include <iostream>
 #include <array>
 #include <string>
+#include <vector>
 
 int wmain(int argc, wchar_t** argv) {
   if (argc < 6) {
-    std::cerr << "usage: reader PID EXPECTED_EXE HOST_GUEST_BASE OUTPUT_DIR GUEST_ADDRESS...\n";
+    std::cerr << "usage: reader PID EXPECTED_EXE HOST_GUEST_BASE OUTPUT_DIR [--bytes LENGTH] GUEST_ADDRESS...\n";
     return 2;
   }
   try {
     const auto pid = std::stoull(argv[1], nullptr, 0);
     const auto base = std::stoull(argv[3], nullptr, 0);
-    if (!pid || pid > MAXDWORD || argc > 37) return 2;
+    int first_address = 5;
+    size_t length = 1024;
+    if (std::wstring(argv[5]) == L"--bytes") {
+      if (argc < 8) return 2;
+      const auto requested = std::stoull(argv[6], nullptr, 0);
+      if (!requested || requested > 8ull * 1024 * 1024) return 2;
+      length = size_t(requested);
+      first_address = 7;
+    }
+    if (!pid || pid > MAXDWORD || argc - first_address > 32 ||
+        uint64_t(length) * (argc - first_address) > 128ull * 1024 * 1024) return 2;
     HANDLE process = OpenProcess(PROCESS_VM_READ | PROCESS_QUERY_LIMITED_INFORMATION,
                                  FALSE, DWORD(pid));
     if (!process) return 3;
@@ -30,15 +41,17 @@ int wmain(int argc, wchar_t** argv) {
     }
     std::filesystem::create_directories(argv[4]);
     bool failed = false;
-    for (int i = 5; i < argc; ++i) {
+    for (int i = first_address; i < argc; ++i) {
       const auto address = std::stoull(argv[i], nullptr, 0);
-      constexpr size_t length = 1024;
+      // Match Memory::TranslateVirtual for Windows physical heap aliases.
+      // MapViewOfFileEx rounds the E/F heap's 4KB backing offset down to64KB.
+      const uint64_t host_offset = address >= 0xE0000000ull ? 0x1000ull : 0;
       if (!address || address > 0x100000000ull - length ||
-          base > UINT64_MAX - address - length) return 2;
-      std::array<char, length> bytes{};
+          base > UINT64_MAX - address - host_offset - length) return 2;
+      std::vector<char> bytes(length);
       SIZE_T read = 0;
       const bool complete = ReadProcessMemory(process,
-          reinterpret_cast<const void*>(base + address), bytes.data(), bytes.size(), &read);
+          reinterpret_cast<const void*>(base + address + host_offset), bytes.data(), bytes.size(), &read);
       if (!complete || read != length) {
         std::cerr << "Read failed guest=" << std::hex << address << " bytes=" << read << '\n';
         failed = true; continue;
diff --git a/tools/gpu-aot/run_native_probe.ps1 b/tools/gpu-aot/run_native_probe.ps1
index 6f9b650..1dc9768 100644
--- a/tools/gpu-aot/run_native_probe.ps1
+++ b/tools/gpu-aot/run_native_probe.ps1
@@ -24,6 +24,7 @@ param(
     [switch]$ConfiguredRenderer,
     [string]$LogoCaptureRenderDocDll,
     [ValidateRange(1,8)][int]$Monitor = 3,
+    [ValidatePattern('^[a-z0-9_-]{1,32}$')][string]$CaptureStage,
     [switch]$Close
 )
 $ErrorActionPreference = 'Stop'
@@ -39,6 +40,27 @@ $projectRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
 $runRoot = Join-Path $projectRoot 'rexlego/out/native-gpu/session-20260930'
 $exePath = Join-Path $projectRoot 'rexlego/out/build/win-amd64-release/legodimensions.exe'
 $recordPath = Join-Path $runRoot "$Label-process.json"
+if ($CaptureStage) {
+    if ($Close) { throw 'Select capture stage or close, not both' }
+    $record = Get-Content -LiteralPath $recordPath | ConvertFrom-Json
+    $game = Get-Process -Id $record.id -ErrorAction Stop
+    if (!$record.textureForensics -or !$record.textureCaptureTrigger -or
+        $game.Path -ne $record.path -or
+        $game.StartTime.ToUniversalTime().Ticks -ne ([datetime]$record.startedAt).ToUniversalTime().Ticks) {
+        throw 'Capture probe identity/settings mismatch'
+    }
+    $stageRoot = Split-Path $record.textureCaptureTrigger
+    if (Test-Path -LiteralPath (Join-Path $stageRoot $CaptureStage)) { throw 'Use a fresh capture stage name' }
+    $temporaryTrigger = Join-Path $stageRoot (([guid]::NewGuid().ToString('N')) + '.trigger.tmp')
+    [IO.File]::WriteAllText($temporaryTrigger, $CaptureStage, [Text.Encoding]::ASCII)
+    if (Test-Path -LiteralPath $record.textureCaptureTrigger) {
+        [IO.File]::Replace($temporaryTrigger, $record.textureCaptureTrigger, $null)
+    } else {
+        [IO.File]::Move($temporaryTrigger, $record.textureCaptureTrigger)
+    }
+    Write-Output "Armed texture capture stage '$CaptureStage' for PID $($record.id)"
+    return
+}
 if ($Close) {
     $record = Get-Content -LiteralPath $recordPath | ConvertFrom-Json
     $game = Get-Process -Id $record.id -ErrorAction SilentlyContinue
diff --git a/tools/gpu-aot/test_read_guest_memory.py b/tools/gpu-aot/test_read_guest_memory.py
new file mode 100644
index 0000000..42e877c
--- /dev/null
+++ b/tools/gpu-aot/test_read_guest_memory.py
@@ -0,0 +1,63 @@
+"""Read actual synthetic process pages; verify Windows E/F alias offset and limits."""
+import argparse
+from pathlib import Path
+import subprocess
+import json
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('output', type=Path)
+a = p.parse_args()
+a.output.mkdir(parents=True, exist_ok=True)
+root = Path(__file__).resolve().parents[2]
+source = a.output/'child.cpp'
+source.write_text(r'''
+#include <windows.h>
+#include <cstdio>
+#include <cstring>
+int main() {
+  for (uintptr_t guest : {uintptr_t(0xA0000000), uintptr_t(0xE0000000)}) {
+    auto* memory = VirtualAlloc(reinterpret_cast<void*>(0x100000000ull + guest),
+        8192, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+    if (!memory) return 1;
+    std::memset(memory, 0x19, 4096);
+    std::memset(static_cast<char*>(memory) + 4096, 0x73, 4096);
+  }
+  std::puts("ready"); std::fflush(stdout); std::getchar();
+}
+'''.replace('#include <cstring>', '#include <cstring>\n#include <initializer_list>'))
+child_exe = a.output/'child.exe'
+reader = a.output/'reader.exe'
+subprocess.run(['clang++', '-std=c++20', '-O2', str(source), '-o', str(child_exe)], check=True)
+subprocess.run(['clang++', '-std=c++20', '-O2', '-municode',
+    str(root/'tools/gpu-aot/read_guest_memory.cpp'), '-o', str(reader)], check=True)
+child = subprocess.Popen([str(child_exe.resolve())], stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE, text=True, creationflags=subprocess.CREATE_NO_WINDOW)
+try:
+    assert child.stdout.readline().strip() == 'ready'
+    def read(folder, addresses, length=None, expected=None):
+        cmd = [str(reader.resolve()), str(child.pid), str((expected or child_exe).resolve()),
+               '0x100000000', str(folder)]
+        if length is not None:
+            cmd.extend(['--bytes', str(length)])
+        return subprocess.run(cmd + addresses, capture_output=True, text=True)
+    folder = a.output/'valid'
+    result = read(folder, ['0xA0000000', '0xE0000000'])
+    assert result.returncode == 0, result.stderr
+    assert (folder/'guest-2684354560.bin').read_bytes() == bytes([0x19])*1024
+    assert (folder/'guest-3758096384.bin').read_bytes() == bytes([0x73])*1024
+    result = read(a.output/'long', ['0xE0000000'], 4096)
+    assert result.returncode == 0, result.stderr
+    assert (a.output/'long/guest-3758096384.bin').read_bytes() == bytes([0x73])*4096
+    for addresses, length in [(['0xE0000000'], 0), (['0xE0000000'], 8*1024*1024+1),
+                              (['0xA0000000']*33, None), (['0xA0000000']*17, 8*1024*1024),
+                              (['0xFFFFFFF0'], None)]:
+        assert read(a.output/'invalid', addresses, length).returncode == 2
+    assert read(a.output/'wrong-exe', ['0xA0000000'], expected=reader).returncode == 4
+finally:
+    child.communicate('\n', timeout=10)
+assert child.returncode == 0
+(a.output/'verification.json').write_text(json.dumps({'passed':True,
+    'checks':['default1024 bytes','variable4096 bytes','E/F +4KB offset','ordinary address unchanged',
+              'length/address/count/total bounds','process executable identity'],
+    'game_launched':False}, indent=2)+'\n')
+print('PASS: actual bounded process reads, alias offset and executable identity')
diff --git a/tools/gpu-aot/test_texture_upload_capture.py b/tools/gpu-aot/test_texture_upload_capture.py
index c3e9875..0ff6a31 100644
--- a/tools/gpu-aot/test_texture_upload_capture.py
+++ b/tools/gpu-aot/test_texture_upload_capture.py
@@ -17,6 +17,7 @@
 cpp=a.output/'test.cpp'
 cpp.write_text(r'''
 #include <cstdint>
+#include <algorithm>
 #include <atomic>
 #include <array>
 #include <cstdlib>
@@ -65,6 +66,24 @@
   r.guest_address=43;r.guest_fetch.dimension=2;
   if(TextureUploadCapturePending(r))return 5;
   r.guest_fetch.dimension=1;
+  r.guest_address=42;
+  for(u32 i=0;i<32;++i)TextureCapture().dumped.insert(i+100);
+  std::ofstream(argv[2])<<"lotr";++g_probe_frame;
+  if(!TextureUploadCapturePending(r))return 7;
+  const auto first_budget=TextureCapture().written_bytes;
+  DumpTextureUpload(r,data.data(),base,mips);
+  if(TextureCapture().written_bytes<=first_budget)return 8;
+  std::ofstream(argv[2])<<"../invalid";++g_probe_frame;
+  if(TextureUploadCapturePending(r))return 9;
+  std::ofstream(argv[2])<<"armed";++g_probe_frame;
+  if(TextureUploadCapturePending(r))return 10; // Refuse overwriting prior scene.
+  for(const char* stage:{"third","fourth"}){
+    std::ofstream(argv[2])<<stage;++g_probe_frame;
+    if(!TextureUploadCapturePending(r))return 11;
+    DumpTextureUpload(r,data.data(),base,mips);
+  }
+  std::ofstream(argv[2])<<"fifth";++g_probe_frame;
+  if(TextureUploadCapturePending(r))return 12;
   TextureCapture().written_bytes=128ull*1024*1024;
   if(TextureUploadCapturePending(r))return 6;
   return 0;
@@ -97,7 +116,7 @@
 trigger=a.output/'arm-capture'
 trigger.unlink(missing_ok=True)
 subprocess.run([str(exe.resolve()),str(all_data.resolve()),str(trigger.resolve())],check=True)
-b=(all_data/'42.dds').read_bytes()
+b=(all_data/'armed/42.dds').read_bytes()
 assert struct.unpack_from('<I',b,28)[0]==7
 expected=bytearray()
 for level in range(7):
@@ -105,13 +124,18 @@
     for row in range((h+3)//4):
         expected.extend((level*23+row*7+column)&255 for column in range(((w+3)//4)*8))
 assert b[128:]==expected
-assert (all_data/'42.guest-base.bin').read_bytes()==bytes([0xAB])*4096
-assert (all_data/'42.guest-mips.bin').read_bytes()==bytes([0xEF])*2048
-assert 'levels=7' in (all_data/'42.txt').read_text()
-assert not (all_data/'43.dds').exists()
+assert (all_data/'armed/42.guest-base.bin').read_bytes()==bytes([0xAB])*4096
+assert (all_data/'armed/42.guest-mips.bin').read_bytes()==bytes([0xEF])*2048
+assert 'levels=7' in (all_data/'armed/42.txt').read_text()
+assert not (all_data/'armed/43.dds').exists()
+for stage in ('lotr','third','fourth'):
+    assert (all_data/stage/'42.dds').read_bytes()==b
+assert not (all_data/'fifth').exists()
 (a.output/'verification.json').write_text(json.dumps({'passed':True,'formats':['BC3/DXT5','BC1/DXT1','RGBA8'],
     'checks':['odd dimensions','256-byte row padding stripped','exact payload preserved','both TT shader variants','unrelated shader filtered',
               'late trigger','cached-upload capture request','all seven packed-tail mip sizes','512-byte mip gaps stripped',
-              'exact raw guest backing','volume refused','128MiB budget'],
+              'exact raw guest backing','volume refused','128MiB budget',
+              'rearm after full scene quota','scene files preserved','no reused scene overwrite',
+              'four scene limit','invalid scene path refused','global byte budget retained'],
     'game_launched':False},indent=2)+'\n')
 print('PASS: DXT5/DXT1/RGBA capture preserves exact rows and filters unrelated shaders')

From 6834b2362f739b2bb24b97280bc8d5a7e40fa31f Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Sun, 4 Oct 2026 16:04:00 +0300
Subject: [PATCH 42/44] Record gradient archive integration and five captured
 placement shaders

---
 tools/gpu-aot/README.md                       | 20 +++++++++-----
 tools/gpu-aot/native-capture-20261004.md      | 11 +++++---
 .../gpu-aot/native-texture-format-20261004.md | 26 +++++++++++++++++++
 .../native-validation-pending-20261003.md     |  4 +--
 tools/gpu-aot/native-work-20261004.md         | 26 +++++++++++++++++++
 5 files changed, 76 insertions(+), 11 deletions(-)

diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index 5cc9280..49fbbd1 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -9,11 +9,19 @@ shader pipeline. Generated archives, HLSL, DXIL, and build trees stay under
 The owner rejected the combined explicit-LOD build: LOTR looked worse, with
 vertical artifacts, while texture stripes and blank portraits remained. It has
 been rolled back as a coupled host/constants/shader-bank change. Production
-again uses the October1 alpha/viewport compiler in
-`session-20261001/viewport-candidate/build/alpha_compiler.exe`, its matching
-`source/shader_common.h`, and **624 shared bytes**. Main9290/runtime250 shader
-coverage, the Enter transition correction and Alt+Enter handling are retained.
-The rollback is an offline build verification, not a new visual success claim.
+uses the gradient compiler in
+`session-20261004/gradients-candidate/build/gradient_compiler.exe`, prepared from
+the exact October1 alpha/viewport source with `prepare_gradients_candidate.py`.
+The matching `source/shader_common.h` retains **624 shared bytes**. Main9290
+shaders/18444 linked variants and runtime255/458 are installed; placement index
+18993 preserves every previous record. The runtime additions are five complete
+containers captured in the October4 diagnostic, not reference-game content.
+Only89 main DXIL payloads change, exactly the89 gradient-query shaders; all
+other9201 payloads and old250 runtime HLSL files are byte-identical.
+The Enter transition correction and Alt+Enter handling are retained.
+Build proof: `session-20261004/gradients-candidate/installed-build-proof.json`.
+This is offline verification; portraits, prompts, texture stripes and gameplay
+performance remain unverified. DoF remains OFF; no additional game launch.
 
 The rejected candidate compiler is the combined alpha/viewport/explicit-LOD
 source in `session-20261004/lod-combined/source`, prepared reproducibly with:
@@ -29,7 +37,7 @@ host upload. Do not reinstall this candidate without isolating its regression.
 The original isolated September30 LOD candidate lacks the later alpha/viewport
 fixes and must not replace this combined compiler.
 
-Main coverage remains9290 (known A58 memexport exclusion), runtime250; all prior
+Main coverage remains9290 (known A58 memexport exclusion), runtime255; all prior
 hashes and specialization masks are preserved. The candidate implements ignored
 explicit mip-level instructions but failed the owner's image check. Synthetic
 fixture success did not establish the game's complete sampling correctness.
diff --git a/tools/gpu-aot/native-capture-20261004.md b/tools/gpu-aot/native-capture-20261004.md
index 738ad4b..fcd6054 100644
--- a/tools/gpu-aot/native-capture-20261004.md
+++ b/tools/gpu-aot/native-capture-20261004.md
@@ -30,8 +30,13 @@ Local evidence under session-20261004/format-forensics1:
   PNGs are diagnostic-reader artifacts, not evidence of renderer corruption.
   The manifest explicitly marks these limitations. Object headers at A/B
   addresses are unaffected. Do not infer intact whole mips from partial spans.
-- Missing-shader capture contains only the already known A58 memexport
-  container. This run does not support guessing additional shader substitutes.
+- Recursive missing-shader review found five new complete containers under
+  placement-containers, in addition to the known A58 memexport container. The
+  original top-level-only count missed these nested files. The captured PS
+  identities are35F76DC3372D5773/DBFA1692650B44C6; VS identities
+  1AFAC761791379C0/43DF3B3B6D90016E/45C2959A30C6DCAE. All five compile,
+  with exact captured identities added to the runtime archive/index. Incomplete
+  placement-raw spans are excluded; no shader substitutes are guessed.
 
 The SDK's actual mapping code confirmed the reader offset issue. The reader now
 matches Windows E/F translation, supports explicitly bounded spans up to8MiB,
@@ -52,7 +57,7 @@ The other two have no recorded reference. Rows are preserved in
 active-gradient-events.tsv. The isolated GetTextureGradients correction is
 therefore relevant to active LOTR materials; it is not a proven fix for the
 entire screenshot. It preserves the624-byte ABI and does not include rejected
-explicit-LOD changes. Full archive preparation proceeds offline; image
+explicit-LOD changes. Matching main archives are now installed in the offline build; image
 validation remains pending.
 
 ## Offline capture rearming repair
diff --git a/tools/gpu-aot/native-texture-format-20261004.md b/tools/gpu-aot/native-texture-format-20261004.md
index b79926e..e1868dd 100644
--- a/tools/gpu-aot/native-texture-format-20261004.md
+++ b/tools/gpu-aot/native-texture-format-20261004.md
@@ -87,3 +87,29 @@ session-20261004/gradients-candidate and are **not installed** into production.
 All250 captured runtime shaders also compile with byte-identical emitted HLSL
 against the prior production compiler. None uses GetTextureGradients, so this
 is a useful check that the isolated change leaves those existing draws alone.
+
+## Installed gradient-query and captured-shader build
+
+Final offline build fingerprint60c62465f26a1152, exe SHA256
+19b93e7c54978cee515f8b0ad8417088a4c7d925e2089fb6d641cf4198ef61c3.
+Matching main9290/18444 and runtime255/458 archives are installed; every
+specialization subset is present. Exact main DXIL comparison changes89/9290
+payloads, matching all89 gradient-query containers; other9201 unchanged.
+All old250 runtime identities/masks,448 linked keys and HLSL bytes preserved.
+Five nested placement containers from the closed diagnostic add ten index
+records:18983->18993 with every old record retained. Corrects the earlier
+nonrecursive missing-container count. No reference-game payloads embedded.
+
+Native build passes with624 shared bytes. Runtime DLL pair and configuration
+match the before-installed SHA256 records, DoF stays OFF, SDK/Vulkan/Plume are
+untouched. Tests include50 gradient translation/DXIL fixtures plus3 refusals,
+100 actual gradient instructions, captured-shader compilation and complete
+prelinked variant coverage. Proofs are under
+session-20261004/gradients-candidate:payload-differential-proof.json,
+runtime255-coverage-proof.json,index-proof.json,installed-build-proof.json.
+Coupled rollback files remain in before-installed.
+
+Game remains closed, no new launch. The captured hub/LOTR images still fail
+portrait/texture checks; this build has not been visually validated. Actual
+1080p rendering, stable60FPS, AMD and three story-level/cutscene checks remain
+pending. Do not count diagnostic traces as clean performance benchmarks.
diff --git a/tools/gpu-aot/native-validation-pending-20261003.md b/tools/gpu-aot/native-validation-pending-20261003.md
index 4a03b1c..f8ee305 100644
--- a/tools/gpu-aot/native-validation-pending-20261003.md
+++ b/tools/gpu-aot/native-validation-pending-20261003.md
@@ -26,8 +26,8 @@ Once launches are authorized:
    Preserve the override material/texture identity if the portrait remains blank.
    October4: use -LongProbe -PortraitTrace -TextureForensics -CaptureMissing to correlate the actual mask's
    NuTexture/backend/active D3D header with draw shader/texture/constant captures. See
-   native-work-20261004.md. Arm the fresh process record's textureCaptureTrigger
-   only after the owner reaches the affected scene. Keep this separate from clean FPS measurements.
+   native-work-20261004.md. Use -CaptureStage hub only after owner readiness, then -CaptureStage lotr
+   for a fresh scene; never reuse a stage name. Each stage retains earlier data. Keep this separate from clean FPS measurements.
 3. Audit buffer watches in hub and three story levels, loading/cutscenes and
    mesh-pool relocations. Require zero stale-clean errors. Until then buffer
    watches stay OFF by default, independent of the existing texture watch.
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index d3847c0..26fb191 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -204,3 +204,29 @@ SDK/Plume change or hotfix change is authorized/implied by this rollback.
 ## Offline texture format and scene capture follow-up
 
 See native-texture-format-20261004.md for tested format replication/DXT3A fixes, actual SDK GPU oracle results, instruction audit and late-trigger mip capture. Owner explicitly reaffirmed no launch. Visual failures and native1080p/60FPS remain unresolved.
+
+## Installed gradient-query and captured-shader build
+
+Final offline build fingerprint60c62465f26a1152, exe SHA256
+19b93e7c54978cee515f8b0ad8417088a4c7d925e2089fb6d641cf4198ef61c3.
+Matching main9290/18444 and runtime255/458 archives are installed; every
+specialization subset is present. Exact main DXIL comparison changes89/9290
+payloads, matching all89 gradient-query containers; other9201 unchanged.
+All old250 runtime identities/masks,448 linked keys and HLSL bytes preserved.
+Five nested placement containers from the closed diagnostic add ten index
+records:18983->18993 with every old record retained. Corrects the earlier
+nonrecursive missing-container count. No reference-game payloads embedded.
+
+Native build passes with624 shared bytes. Runtime DLL pair and configuration
+match the before-installed SHA256 records, DoF stays OFF, SDK/Vulkan/Plume are
+untouched. Tests include50 gradient translation/DXIL fixtures plus3 refusals,
+100 actual gradient instructions, captured-shader compilation and complete
+prelinked variant coverage. Proofs are under
+session-20261004/gradients-candidate:payload-differential-proof.json,
+runtime255-coverage-proof.json,index-proof.json,installed-build-proof.json.
+Coupled rollback files remain in before-installed.
+
+Game remains closed, no new launch. The captured hub/LOTR images still fail
+portrait/texture checks; this build has not been visually validated. Actual
+1080p rendering, stable60FPS, AMD and three story-level/cutscene checks remain
+pending. Do not count diagnostic traces as clean performance benchmarks.

From f1ddad695119db32f9a511f68ed204238d1969e9 Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 6 Oct 2026 18:14:14 +0300
Subject: [PATCH 43/44] Prepare native renderer draft for collaboration and
 portable helper checks

---
 .github/workflows/native-gpu-checks.yml  |  18 ++++
 rexlego/src/gpu_native/draw.cpp          |  33 +++++++
 rexlego/src/gpu_native/polygon_offset.h  |  36 +++++++
 tools/gpu-aot/CMakeLists.txt             |  22 +++++
 tools/gpu-aot/COLLABORATION.md           | 115 +++++++++++++++++++++++
 tools/gpu-aot/README.md                  |   3 +
 tools/gpu-aot/native-work-20261004.md    |  36 +++++++
 tools/gpu-aot/prepare_alpha_candidate.py |   4 +-
 tools/gpu-aot/prepare_native_compiler.py |  39 ++++++++
 tools/gpu-aot/test_polygon_offset.cpp    |  20 ++++
 10 files changed, 325 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/native-gpu-checks.yml
 create mode 100644 rexlego/src/gpu_native/polygon_offset.h
 create mode 100644 tools/gpu-aot/CMakeLists.txt
 create mode 100644 tools/gpu-aot/COLLABORATION.md
 create mode 100644 tools/gpu-aot/prepare_native_compiler.py
 create mode 100644 tools/gpu-aot/test_polygon_offset.cpp

diff --git a/.github/workflows/native-gpu-checks.yml b/.github/workflows/native-gpu-checks.yml
new file mode 100644
index 0000000..fb15c50
--- /dev/null
+++ b/.github/workflows/native-gpu-checks.yml
@@ -0,0 +1,18 @@
+name: Native GPU portable checks
+on:
+  pull_request:
+  push:
+    branches: [codex/native-renderer]
+permissions:
+  contents: read
+jobs:
+  portable:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure asset-free checks
+        run: cmake -S tools/gpu-aot -B native-checks -DCMAKE_BUILD_TYPE=Release
+      - name: Build
+        run: cmake --build native-checks --parallel 2
+      - name: Test
+        run: ctest --test-dir native-checks --output-on-failure
diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp
index 140a351..446370b 100644
--- a/rexlego/src/gpu_native/draw.cpp
+++ b/rexlego/src/gpu_native/draw.cpp
@@ -1,4 +1,5 @@
 #include <algorithm>
+#include <bit>
 #include <array>
 #include <atomic>
 #include <chrono>
@@ -30,6 +31,7 @@
 #include "gpu_native/d3d.h"
 #include "gpu_native/draw.h"
 #include "gpu_native/depth_state.h"
+#include "gpu_native/polygon_offset.h"
 #include "gpu_native/stencil_state.h"
 #include "gpu_native/device.h"
 #include "gpu_native/shaders.h"
@@ -100,6 +102,10 @@ struct PipelineKey {
   std::array<u32, kNativeRenderTargets> blend_controls{0x10001, 0x10001, 0x10001, 0x10001};
   u32 topology = 0;
   u32 spec_constants = 0;
+  // Polygon offset in host units: D3D12 integer DepthBias and the bits of the
+  // float SlopeScaledDepthBias.
+  i32 depth_bias = 0;
+  u32 slope_scaled_depth_bias = 0;
   std::array<u32, kNativeVertexStreams> strides{};
   bool operator==(const PipelineKey&) const = default;
 };
@@ -115,6 +121,7 @@ struct PipelineKeyHash {
     mix(key.stencil_masks); mix(key.stencil_face);
     for (const auto blend : key.blend_controls) mix(blend);
     mix(key.topology); mix(key.spec_constants);
+    mix(u32(key.depth_bias)); mix(key.slope_scaled_depth_bias);
     for (const auto stride : key.strides) mix(stride);
     return hash;
   }
@@ -389,6 +396,10 @@ plume::RenderPipeline* GetPipeline(
   desc.depthEnabled = depth_state.enabled;
   desc.depthWriteEnabled = depth_state.write;
   desc.depthClipEnabled = !key.clip_disable;
+  if (depth_format != plume::RenderFormat::UNKNOWN) {
+    desc.depthBias = key.depth_bias;
+    desc.slopeScaledDepthBias = std::bit_cast<float>(key.slope_scaled_depth_bias);
+  }
   const auto stencil = DecodeStencilState(key.depth_control, key.stencil_masks,
       depth_format == plume::RenderFormat::D32_FLOAT_S8_UINT,
       topology == plume::RenderPrimitiveTopology::TRIANGLE_LIST ||
@@ -725,6 +736,28 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed,
       *reinterpret_cast<const be_u32*>(state_bytes + 10556),
       *reinterpret_cast<const be_f32*>(state_bytes + 10620));
   if (alpha.enabled) key.spec_constants |= 2u;
+  // Polygon offset. TU23 D3DRS_SLOPESCALEDEPTHBIAS (83FB88A8) stores the
+  // scale, already in 1/16 subpixel units, at +10832 (front) / +10840 (back);
+  // D3DRS_DEPTHBIAS (83FB8970) stores the offset at +10836 / +10844; both set
+  // PA_SU_SC_MODE_CNTL (+10568) poly_offset_front/back_enable (bits 11/12).
+  // Convert the stored offset for the host rasterizer. Face selection follows
+  // SDK's GetPreferredFacePolygonOffset/GetD3D10IntegerPolygonOffset: native
+  // never culls, so front wins, then back; non-triangles use para_enable.
+  // Unorm24 and float24 guests both come to ~offset * 2^24 host D32 units.
+  // LEGO_NATIVE_NO_DEPTH_BIAS=1 restores the old behaviour for A/B checks.
+  static const bool depth_bias_disabled = std::getenv("LEGO_NATIVE_NO_DEPTH_BIAS") != nullptr;
+  if (!depth_bias_disabled && depth.format != plume::RenderFormat::UNKNOWN) {
+    const u32 mode = *reinterpret_cast<const be_u32*>(state_bytes + 10568);
+    const auto poly = [&](u32 offset) {
+      return float(*reinterpret_cast<const be_f32*>(state_bytes + offset));
+    };
+    const bool polygonal = topology == plume::RenderPrimitiveTopology::TRIANGLE_LIST ||
+                           topology == plume::RenderPrimitiveTopology::TRIANGLE_STRIP;
+    const auto bias = DecodePolygonOffset(mode, polygonal,
+        poly(10832), poly(10836), poly(10840), poly(10844));
+    key.depth_bias = bias.depth_bias;
+    key.slope_scaled_depth_bias = bias.slope_bits;
+  }
   for (u32 i = 0; i < kNativeVertexStreams; ++i) {
     key.strides[i] = bindings.vertex_streams[i].stride;
   }
diff --git a/rexlego/src/gpu_native/polygon_offset.h b/rexlego/src/gpu_native/polygon_offset.h
new file mode 100644
index 0000000..21fcd36
--- /dev/null
+++ b/rexlego/src/gpu_native/polygon_offset.h
@@ -0,0 +1,36 @@
+#pragma once
+#include <algorithm>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+
+namespace legodimensions::gpu_native {
+struct PolygonOffset {
+  int32_t depth_bias = 0;
+  uint32_t slope_bits = 0;
+};
+
+// Native rasterization currently disables culling. Prefer an enabled front
+// offset, falling back to the back offset when both front values are zero.
+inline PolygonOffset DecodePolygonOffset(uint32_t mode, bool polygonal,
+    float front_scale, float front_offset, float back_scale, float back_offset) {
+  float scale = 0.0f, offset = 0.0f;
+  if (polygonal) {
+    if (mode & (1u << 11)) { scale = front_scale; offset = front_offset; }
+    if ((mode & (1u << 12)) && scale == 0.0f && offset == 0.0f) {
+      scale = back_scale; offset = back_offset;
+    }
+  } else if (mode & (1u << 13)) {
+    scale = front_scale; offset = front_offset;
+  }
+  PolygonOffset result;
+  if (std::isfinite(offset) && offset != 0.0f) {
+    const int32_t magnitude = int32_t(std::min(
+        std::ceil(std::abs(offset) * 16777215.0f), 1.0e9f));
+    result.depth_bias = offset < 0.0f ? -magnitude : magnitude;
+  }
+  if (std::isfinite(scale) && scale != 0.0f)
+    result.slope_bits = std::bit_cast<uint32_t>(scale * (1.0f / 16.0f));
+  return result;
+}
+}  // namespace legodimensions::gpu_native
diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt
new file mode 100644
index 0000000..210f5b2
--- /dev/null
+++ b/tools/gpu-aot/CMakeLists.txt
@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 3.25)
+project(NativeGpuPortableChecks LANGUAGES CXX)
+include(CTest)
+
+# No SDK, generated game code, game assets, DXC, or graphics device required.
+set(CHECKS blend_state buffer_header color_resolve_swizzle descriptor_retirement
+    pixel_shader_filter polygon_offset pool_copy portrait_identity present_rect
+    resolve_region shader_texture_usage texture_alpha4_upload texture_number
+    texture_volume_upload tile_extent tonemap_dof vertex_byte_order vertex_upload
+    window_mode)
+foreach(check IN LISTS CHECKS)
+    add_executable(test_${check} test_${check}.cpp)
+    target_compile_features(test_${check} PRIVATE cxx_std_20)
+    target_include_directories(test_${check} PRIVATE ../../rexlego/src)
+    # The fixtures use assert; keep them active in Release builds too.
+    if(MSVC)
+        target_compile_options(test_${check} PRIVATE /UNDEBUG)
+    else()
+        target_compile_options(test_${check} PRIVATE -UNDEBUG)
+    endif()
+    add_test(NAME ${check} COMMAND test_${check})
+endforeach()
diff --git a/tools/gpu-aot/COLLABORATION.md b/tools/gpu-aot/COLLABORATION.md
new file mode 100644
index 0000000..32b00a6
--- /dev/null
+++ b/tools/gpu-aot/COLLABORATION.md
@@ -0,0 +1,115 @@
+# Native renderer contributor handoff
+
+This branch contains an experimental C++ native D3D12 renderer and its AOT
+translation tools. `LEGODIMENSIONS_NATIVE_GPU` defaults to `OFF`. This is a
+development handoff, not a completed renderer or a release.
+
+## Linux limitations
+
+- `rexlego/CMakeLists.txt` explicitly rejects `LEGODIMENSIONS_NATIVE_GPU=ON`
+  outside Windows. The native implementation includes Win32/D3D12 APIs. No
+  Linux native renderer, Wine, or cross-compilation run was validated here.
+- The current compiler-preparation CMake uses Windows DXC `.lib`/`.dll` files,
+  MSVC runtime options, and Windows dependency output names. The runtime probe
+  scripts are PowerShell/Windows tools. They are not Linux build instructions.
+- A clean clone lacks `rexlego/generated/rexglue.cmake`, translated game C++,
+  game data, and AOT shader archives. These are deliberately ignored. The game
+  CMake includes generated/rexglue.cmake before its native-platform check.
+  Follow `README-dev.md` using your own game inputs for game-code generation.
+- The asset-free CMake checks below do not configure the game or require its
+  generated C++, SDK submodules, DXC, a GPU, or game data. Ubuntu CI runs this
+  target. It tests helpers, not image correctness or game performance.
+
+## Clone and portable checks
+
+```sh
+git clone --branch codex/native-renderer --recurse-submodules https://github.com/NeverCookFirst/DimensionsRecomp.git
+cd DimensionsRecomp
+cmake -S tools/gpu-aot -B native-checks -DCMAKE_BUILD_TYPE=Release
+cmake --build native-checks --parallel 2
+ctest --test-dir native-checks --output-on-failure
+```
+
+Requirements for this target: Git, CMake >=3.25 and a C++20 compiler.
+It registers 19 tests. Release assertions stay enabled. The workflow is
+`.github/workflows/native-gpu-checks.yml`; consult its actual run result before
+reporting Linux validation.
+
+The SDK gitlink is `17e3675767067339581018c7e722e1d8d65de201`, published on
+the fork's `codex/native-renderer` branch. Its dependency draft is
+https://github.com/NeverCookFirst/rexglue-sdk/pull/1. Use the gitlink with
+`git submodule update --init --recursive`; `git submodule update --remote`
+selects branch tips instead of the recorded dependency revision.
+
+## Reconstruct current shader compiler inputs
+
+The compiler source is reconstructed from an external pinned checkout plus
+tracked adaptations. It is not contained in the ignored local session trees.
+In a fresh checkout, initialize the external compiler:
+
+```sh
+git clone https://github.com/zolaware/reblue-XenosRecomp.git research/reblue/thirdparty/XenosRecomp
+git -C research/reblue/thirdparty/XenosRecomp checkout 339af41df2c23dbe3256c1c377716b81a0e0fe6b
+git -C research/reblue/thirdparty/XenosRecomp submodule update --init --recursive
+git -C research/reblue/thirdparty/XenosRecomp apply ../../../../tools/gpu-aot/patches/xenosrecomp-lego.patch
+python tools/gpu-aot/prepare_native_compiler.py research/reblue/thirdparty/XenosRecomp rexlego/out/native-gpu/current-compiler
+```
+
+The wrapper checks the revision and applied patch, then invokes alpha,
+viewport and gradient preparations in that order. It requires a fresh output
+directory, does not modify the compiler checkout, and does not install banks.
+The resulting ten C++/header files were compared with the October4 installed
+compiler source and matched after newline normalization. Shared constants
+remain 624 bytes. This source comparison is not a Linux compiler-build test.
+
+For Windows compiler builds, `prepare_alpha_candidate.py` currently expects
+the pinned compiler's dependencies already built in `out/build/lego-release`:
+`xxhash.lib`, `zstd_static.lib`, `fmt.lib` and `fmt.dll`, plus the pinned DXC
+distribution. Its generated CMake lists the exact paths. Build those upstream
+dependencies in an MSVC/Windows SDK environment before configuring
+`current-compiler/gradients/source`; the executable target is
+`gradient_compiler`. The wrapper accepts a checkout at any filesystem path.
+
+Use this compiler for both main and captured runtime containers. Rebuild both
+raw archives, prelinked variants and the placement index together. Never pair
+624-byte host uploads with the rejected 752-byte explicit-LOD archives. The
+extractor/index/prelink commands and input layouts are described in README.md.
+
+## Local inputs and collaboration
+
+Git contains project-owned source, tools, patches and investigation notes.
+It does not contain game executables/data, generated game C++, shader
+containers/DXIL/HLSL banks, saves, captures, or the Avengers reference install.
+No Avengers game payload is embedded. Contributors generate game-derived
+inputs from their own copies; private owner-to-owner transfers are separate
+from this PR. A clone cannot reproduce the exact 255-container runtime corpus
+without those local captured inputs.
+
+For a private workspace handoff, inventory `rexlego/generated`, selected
+`rexlego/out/native-gpu` inputs/banks/rollback proofs, save/config files and
+captures. Keep compiler/runtime hashes with them. Do not copy CMake build trees
+as a substitute for configuring the new machine: their cache stores absolute
+paths. The ignored root PROJECT-MEMORY.md and original task prompt are local
+context; this tracked document and dated notes provide the public handoff.
+
+Use personal branches and PRs against `codex/native-renderer`. Record the
+game and SDK commits, shader compiler input revision, constants ABI, archive
+hashes, actual render-target dimensions and capture/test identifier with each
+result. Do not report diagnostic tracing as a clean FPS benchmark.
+
+## Verified state and remaining checks
+
+On October6, the 19 asset-free checks passed on Windows with Clang20.1.8.
+The Windows native game target also built successfully; no game was launched
+for this publication. The depth-bias addition is included in pipeline identity
+and rasterizer configuration and has helper checks for face selection,
+positive/negative offsets and non-finite inputs. Its effect on gameplay images
+has not been tested. `LEGO_NATIVE_NO_DEPTH_BIAS` disables it when present.
+
+The October4 installed archives had main9290/18444 and runtime255/458
+shader/linked-variant coverage, with18993 placement records. These are local
+artifact counts, not files shipped in this PR. The last captured hub/LOTR
+images still showed blank portraits and rendering artifacts. Visual checks
+of the current compiler/bias combination, actual internal1920x1080, stable60FPS,
+three story levels/cutscenes and AMD hardware remain outstanding. The latest
+saved owner configuration has depth of field disabled.
diff --git a/tools/gpu-aot/README.md b/tools/gpu-aot/README.md
index 49fbbd1..30d5938 100644
--- a/tools/gpu-aot/README.md
+++ b/tools/gpu-aot/README.md
@@ -4,6 +4,9 @@ This directory keeps the reproducible, project-owned part of the native D3D12
 shader pipeline. Generated archives, HLSL, DXIL, and build trees stay under
 `rexlego/out/native-gpu` and are intentionally not committed.
 
+Start with [the contributor handoff](COLLABORATION.md) for clone instructions,
+the Linux restrictions, asset-free checks and current compiler reconstruction.
+
 ## Current compiler and constants ABI (October 4, after owner visual test)
 
 The owner rejected the combined explicit-LOD build: LOTR looked worse, with
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index 26fb191..bf168f6 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -230,3 +230,39 @@ Game remains closed, no new launch. The captured hub/LOTR images still fail
 portrait/texture checks; this build has not been visually validated. Actual
 1080p rendering, stable60FPS, AMD and three story-level/cutscene checks remain
 pending. Do not count diagnostic traces as clean performance benchmarks.
+
+## October 5: polygon offset (depth bias) was never applied (Claude)
+
+Native pipelines had no depth bias at all. TU23 render-state table 0x847F9B18:
+state 51 setter 83FB88A8 = D3DRS_SLOPESCALEDEPTHBIAS (value * subpixel factor
+-> device +10832 front scale, +10840 back scale); state 52 setter 83FB8970 =
+D3DRS_DEPTHBIAS (-> +10836 front offset, +10844 back offset). Both set
+PA_SU_SC_MODE_CNTL (+10568) bits 11/12 (poly_offset_front/back_enable).
+draw.cpp now reads these into PipelineKey (depth_bias, slope_scaled_depth_bias)
+using the SDK's GetPreferredFacePolygonOffset / GetD3D10IntegerPolygonOffset
+rules (offset * 2^24 ceil, slope * 1/16). Hypothesis: the contour/ripple
+"texture stripes" on terrain, snow and characters are shadow acne, not mip
+data. Also verified offline: MergeTextureFetch matches 83FB58A8 bit for bit,
+and DecodeTextureSampler matches the SDK fetch layout. A/B switch:
+LEGO_NATIVE_NO_DEPTH_BIAS=1. Built, NOT visually tested (owner tests).
+Tall coloured LOTR lines are absent in lotr-before.png and present after the
+gradient bank + five captured placement shaders were installed: bisect those
+two changes first. Use RenderDoc mesh viewer on the line draw.
+
+
+## October6 collaborative draft handoff
+
+Owner explicitly authorized publishing draft PRs, superseding the earlier
+local-only Git rule for this handoff. SDK committed dependency17e3675 is
+published without modifying or rebuilding the SDK worktree. Its draft is
+NeverCookFirst/rexglue-sdk#1. The game branch retains the exact SDK gitlink.
+
+Added asset-free CMake/CTest checks and Ubuntu CI; all19 checks pass locally
+on Windows/Clang20.1.8. Polygon offset is factored into a tested helper;
+visual effects remain untested. Native Windows target builds; no game launch.
+Compiler reconstruction from pinned339af41 plus the tracked patch and the
+alpha->viewport->gradient scripts matches all10 installed C++/header inputs
+(after newline normalization). No game-derived archives or captures published.
+COLLABORATION.md states the Windows-only native CMake restriction, Windows
+compiler dependency requirements, missing generated/game/captured inputs and
+public/private handoff split. No Linux game or native renderer validation.
diff --git a/tools/gpu-aot/prepare_alpha_candidate.py b/tools/gpu-aot/prepare_alpha_candidate.py
index 5974d6c..7fa4a39 100644
--- a/tools/gpu-aot/prepare_alpha_candidate.py
+++ b/tools/gpu-aot/prepare_alpha_candidate.py
@@ -10,9 +10,11 @@
 
 p = argparse.ArgumentParser(description=__doc__)
 p.add_argument('scratch', type=Path)
+p.add_argument('--checkout', type=Path,
+               default=Path(__file__).resolve().parents[2] / 'research/reblue/thirdparty/XenosRecomp')
 a = p.parse_args()
 root = Path(__file__).resolve().parents[2]
-checkout = root / 'research/reblue/thirdparty/XenosRecomp'
+checkout = a.checkout.resolve()
 module = checkout / 'XenosRecomp'
 source = a.scratch.resolve() / 'source'
 source.mkdir(parents=True, exist_ok=True)
diff --git a/tools/gpu-aot/prepare_native_compiler.py b/tools/gpu-aot/prepare_native_compiler.py
new file mode 100644
index 0000000..c23938d
--- /dev/null
+++ b/tools/gpu-aot/prepare_native_compiler.py
@@ -0,0 +1,39 @@
+"""Reconstruct the current ABI624 compiler from a pinned, patched checkout.
+
+Only writes the requested scratch directory. Does not patch the checkout,
+build anything, install archives, or launch the game.
+"""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+PIN = '339af41df2c23dbe3256c1c377716b81a0e0fe6b'
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('checkout', type=Path)
+p.add_argument('scratch', type=Path)
+a = p.parse_args()
+checkout, scratch = a.checkout.resolve(), a.scratch.resolve()
+root = Path(__file__).resolve().parents[2]
+revision = subprocess.check_output(
+    ['git', '-C', str(checkout), 'rev-parse', 'HEAD'], text=True).strip()
+if revision != PIN:
+    p.error(f'Expected XenosRecomp {PIN}, got {revision}')
+patch = root / 'tools/gpu-aot/patches/xenosrecomp-lego.patch'
+subprocess.run(['git', '-C', str(checkout), 'apply', '--reverse', '--check', str(patch)], check=True)
+if scratch.exists():
+    p.error('Use a fresh scratch directory to avoid mixing compiler generations')
+scripts = root / 'tools/gpu-aot'
+subprocess.run([sys.executable, str(scripts/'prepare_alpha_candidate.py'),
+                str(scratch/'alpha'), '--checkout', str(checkout)], check=True)
+subprocess.run([sys.executable, str(scripts/'prepare_viewport_candidate.py'),
+                str(scratch/'alpha/source'), str(scratch/'viewport')], check=True)
+subprocess.run([sys.executable, str(scripts/'prepare_gradients_candidate.py'),
+                str(scratch/'viewport/source'), str(scratch/'gradients')], check=True)
+(scratch/'compiler-inputs.json').write_text(json.dumps({
+    'xenosrecomp_revision': revision, 'shared_constants_bytes': 624,
+    'source': str(scratch/'gradients/source'), 'built': False,
+    'archives_installed': False, 'game_launched': False,
+}, indent=2), encoding='utf-8')
+print(scratch/'gradients/source')
diff --git a/tools/gpu-aot/test_polygon_offset.cpp b/tools/gpu-aot/test_polygon_offset.cpp
new file mode 100644
index 0000000..6d9a0af
--- /dev/null
+++ b/tools/gpu-aot/test_polygon_offset.cpp
@@ -0,0 +1,20 @@
+#include "gpu_native/polygon_offset.h"
+#include <cassert>
+#include <limits>
+using namespace legodimensions::gpu_native;
+int main() {
+  constexpr auto front = 1u << 11, back = 1u << 12, para = 1u << 13;
+  assert(DecodePolygonOffset(0, true, 16, 1, 32, 2).depth_bias == 0);
+  auto p = DecodePolygonOffset(front | back, true, 16, 1, 32, -1);
+  assert(p.depth_bias == 16777215 && std::bit_cast<float>(p.slope_bits) == 1);
+  p = DecodePolygonOffset(front | back, true, 0, 0, -32, -1);
+  assert(p.depth_bias == -16777215 && std::bit_cast<float>(p.slope_bits) == -2);
+  assert(DecodePolygonOffset(front, false, 16, 1, 0, 0).depth_bias == 0);
+  assert(DecodePolygonOffset(para, false, 16, 1, 0, 0).depth_bias == 16777215);
+  assert(DecodePolygonOffset(front, true, 0, 1.0f / 16777216.0f, 0, 0).depth_bias == 1);
+  assert(DecodePolygonOffset(front, true, 0, -1.0f / 16777216.0f, 0, 0).depth_bias == -1);
+  p = DecodePolygonOffset(front, true, std::numeric_limits<float>::infinity(),
+      std::numeric_limits<float>::quiet_NaN(), 0, 0);
+  assert(p.depth_bias == 0 && p.slope_bits == 0);
+  assert(DecodePolygonOffset(front, true, 0, std::numeric_limits<float>::max(), 0, 0).depth_bias == 1000000000);
+}

From d94c2fd289a557d24bdec6ec0329fe9d9710cfab Mon Sep 17 00:00:00 2001
From: NeverCookFirst <46416213+NeverCookFirst@users.noreply.github.com>
Date: Tue, 6 Oct 2026 18:15:50 +0300
Subject: [PATCH 44/44] Record verified Ubuntu portable checks and published
 draft handoff

---
 tools/gpu-aot/COLLABORATION.md        | 4 +++-
 tools/gpu-aot/native-work-20261004.md | 4 ++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/gpu-aot/COLLABORATION.md b/tools/gpu-aot/COLLABORATION.md
index 32b00a6..19948c6 100644
--- a/tools/gpu-aot/COLLABORATION.md
+++ b/tools/gpu-aot/COLLABORATION.md
@@ -99,7 +99,9 @@ result. Do not report diagnostic tracing as a clean FPS benchmark.
 
 ## Verified state and remaining checks
 
-On October6, the 19 asset-free checks passed on Windows with Clang20.1.8.
+On October6, the 19 asset-free checks passed on Windows with Clang20.1.8
+and on Ubuntu CI with GCC13.3.0. The successful Linux run is
+https://github.com/NeverCookFirst/DimensionsRecomp/actions/runs/37485760314.
 The Windows native game target also built successfully; no game was launched
 for this publication. The depth-bias addition is included in pipeline identity
 and rasterizer configuration and has helper checks for face selection,
diff --git a/tools/gpu-aot/native-work-20261004.md b/tools/gpu-aot/native-work-20261004.md
index bf168f6..cb742a2 100644
--- a/tools/gpu-aot/native-work-20261004.md
+++ b/tools/gpu-aot/native-work-20261004.md
@@ -266,3 +266,7 @@ alpha->viewport->gradient scripts matches all10 installed C++/header inputs
 COLLABORATION.md states the Windows-only native CMake restriction, Windows
 compiler dependency requirements, missing generated/game/captured inputs and
 public/private handoff split. No Linux game or native renderer validation.
+
+Main draft published:NeverCookFirst/DimensionsRecomp#25. Ubuntu CI
+run37485760314 passes19/19 asset-free checks with GCC13.3.0. This
+validates the portable helper target, not the Linux game renderer.