From 14029c1b3b86699e9e43912e0bbbd263fe8613ab Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 26 Jun 2026 03:02:51 +0000 Subject: [PATCH 1/4] Warn when snapshot vert count exceeds DrawIndexed upper bound For an indexed triangle list, the maximum unique verts a draw can touch is prim_count * 3. When compute_prim_vert_count derives num_vertices from the full bound VB (vb_size / vert_size) it can vastly exceed that bound, which is a strong signal the VB is shared across many draws or the geometry is CPU/software-animated and not usefully snapshottable. It also tends to coincide with a missing-index-buffer error. Log a warning in snapshot::take so users get a clear in-log hint instead of a confusing error chain. No behavior change; diagnostic only. Authored by Claude (claude-opus-4-7) --- Native/hook_snapshot/src/hook_snapshot.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Native/hook_snapshot/src/hook_snapshot.rs b/Native/hook_snapshot/src/hook_snapshot.rs index 7e1bb9b..0684ffb 100644 --- a/Native/hook_snapshot/src/hook_snapshot.rs +++ b/Native/hook_snapshot/src/hook_snapshot.rs @@ -147,6 +147,23 @@ pub unsafe fn take(devptr:&mut DevicePointer, sd:&mut types::interop::SnapshotDa unsafe { write_log_file(&format!("==> New snap started: prims: {}, verts: {}, basevert: {}, startindex: {}", sd.prim_count, sd.num_vertices, sd.base_vertex_index, sd.start_index)); + // For an indexed triangle list, the most unique verts a draw can touch is prim_count * 3 + // (no shared indices). If num_vertices exceeds that, it most likely came from + // vb_size / vert_size (the whole bound VB) rather than this draw's range -- see + // compute_prim_vert_count in hook_render_d3d11.rs. That's a strong hint that either the + // VB is shared across many draws, or the geometry is CPU/software-animated and not + // usefully snapshottable, and may also explain a subsequent missing-index-buffer error. + let implied_max_verts = sd.prim_count.saturating_mul(3); + if sd.num_vertices > implied_max_verts { + write_log_file(&format!( + "WARNING: snapshot vert count {} exceeds the upper bound implied by DrawIndexed \ + parameters ({} = prim_count * 3). num_vertices likely reflects the full bound \ + vertex buffer rather than this draw's range. If this geometry is CPU/software \ + animated it will not be modable; the snapshot may also fail with a missing \ + index buffer or produce a mismatched mesh.", + sd.num_vertices, implied_max_verts)); + } + pre_rc = devptr.get_ref_count(); (*gs).device = Some(*devptr); From a6a14adec41ba630394377b57a4755133d914372 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 26 Jun 2026 17:03:55 +0000 Subject: [PATCH 2/4] Capture and slice dynamically-updated DX11 buffers for snapshots Character-mesh parts packed into a large dynamically-updated vertex/index buffer (a "megabuffer") failed to snapshot, logging "failed to get index buffer data, was not previously saved" and a vertex count equal to the whole bound buffer rather than the draw's range. Two native fixes (DX11 only; managed/interop unchanged, no version bump): 1. Capture buffers filled after creation. hook_CreateBuffer only copied data supplied as pInitialData. Now, when precopy is enabled, also hook ID3D11DeviceContext::Map/Unmap and UpdateSubresource to copy index and vertex buffer bytes whenever the game fills them. CreateBuffer records per-buffer metadata (type + size) so the new hooks can identify tracked VB/IBs cheaply; a createtime entry is pushed only on new keys so the expiry GC stays correct and continuously-updated buffers self-heal. 2. Snapshot only the draw's sub-region. set_buffers_d3d11 assumed per-mesh buffers. It now slices the prim_count*3 indices starting at start_index (honoring bind offsets), finds the referenced vertex range, carves out those vertices, and re-bases the indices to 0 so managed reads a self-contained mesh from offset 0 (its tested path). num_vertices, base_vertex_index, min_vertex_index and start_index are set accordingly. Authored by Claude Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DsyY4xhCQs2tQ1f3BkmDTL --- Native/hook_core/src/hook_device_d3d11.rs | 77 +++++++-- Native/hook_core/src/hook_render_d3d11.rs | 200 +++++++++++++++++++++- Native/hook_snapshot/src/hook_snapshot.rs | 119 ++++++++++--- Native/shared_dx/src/defs_dx11.rs | 25 ++- Native/shared_dx/src/dx11rs.rs | 13 ++ Native/shared_dx/src/types_dx11.rs | 3 + 6 files changed, 389 insertions(+), 48 deletions(-) diff --git a/Native/hook_core/src/hook_device_d3d11.rs b/Native/hook_core/src/hook_device_d3d11.rs index 9203cc3..857b075 100644 --- a/Native/hook_core/src/hook_device_d3d11.rs +++ b/Native/hook_core/src/hook_device_d3d11.rs @@ -384,6 +384,23 @@ pub unsafe fn apply_context_hooks(context:*mut ID3D11DeviceContext, first_hook:b (*vtbl).PSSetShaderResources = hook_PSSetShaderResources; func_hooked += 1; } + // Map/Unmap/UpdateSubresource are hooked so that buffers the game fills *after* creation + // (e.g. dynamic "megabuffers" updated via Map/WRITE_NO_OVERWRITE or UpdateSubresource) can be + // copied for snapshotting. Installed unconditionally like the draw hooks; the hook bodies are + // no-ops unless precopy_data is enabled, so the runtime precopy toggle works without rehooking + // the (already-copied) context vtable. + if (*vtbl).Map as usize != hook_Map as usize { + (*vtbl).Map = hook_Map; + func_hooked += 1; + } + if (*vtbl).Unmap as usize != hook_Unmap as usize { + (*vtbl).Unmap = hook_Unmap; + func_hooked += 1; + } + if (*vtbl).UpdateSubresource as usize != hook_UpdateSubresource as usize { + (*vtbl).UpdateSubresource = hook_UpdateSubresource; + func_hooked += 1; + } if TRACK_REHOOK_TIME { let now = SystemTime::now(); @@ -636,6 +653,9 @@ unsafe fn hook_d3d11(device:*mut ID3D11Device,_swapchain:*mut IDXGISwapChain, co let real_ia_set_input_layout = (*vtbl).IASetInputLayout; let real_ia_set_primitive_topology = (*vtbl).IASetPrimitiveTopology; let real_ps_set_shader_resources = (*vtbl).PSSetShaderResources; + let real_map = (*vtbl).Map; + let real_unmap = (*vtbl).Unmap; + let real_update_subresource = (*vtbl).UpdateSubresource; // since we always make a copy of the vtable in the context at the moment, we don't search // for the real functions as we do in the device case, since a new context should always have @@ -672,6 +692,9 @@ unsafe fn hook_d3d11(device:*mut ID3D11Device,_swapchain:*mut IDXGISwapChain, co real_ia_set_input_layout, real_ia_set_primitive_topology, real_ps_set_shader_resources, + real_map, + real_unmap, + real_update_subresource, }; Ok(HookDirect3D11 { context: hook_context }) @@ -1017,31 +1040,47 @@ unsafe extern "system" fn hook_CreateBuffer( ppBuffer ); - if res == 0 && ppBuffer != null_mut() && (*ppBuffer) != null_mut() { - // if its an index buffer with data, we need to copy it out + if res == 0 && ppBuffer != null_mut() && (*ppBuffer) != null_mut() && !pDesc.is_null() { + // For index/vertex buffers, record metadata (type + size) so the Map/Unmap/ + // UpdateSubresource hooks can identify this buffer later -- even if it is created empty + // and filled afterwards (the "megabuffer" case). If initial data was supplied we also + // copy it out now, since DX11 offers no way to read the buffer back from the CPU later. let is_ib = (*pDesc).BindFlags & D3D11_BIND_INDEX_BUFFER != 0; let is_vb = (*pDesc).BindFlags & D3D11_BIND_VERTEX_BUFFER != 0; - if !pDesc.is_null() && (is_ib || is_vb) - && !pInitialData.is_null() && !(*pInitialData).pSysMem.is_null() { - if (*pInitialData).SysMemPitch != 0 || (*pInitialData).SysMemSlicePitch != 0 { - write_log_file(&format!("WARNING: hook_CreateBuffer: index or vertex buffer created with pitch or slice pitch, copy unimplemented")); - } else { - let vlen = (*pDesc).ByteWidth as usize; - let mut dest_v:Vec = Vec::with_capacity(vlen); - std::ptr::copy_nonoverlapping::((*pInitialData).pSysMem as *const u8, dest_v.as_mut_ptr(), vlen); - dest_v.set_len(vlen); - dev_state_d3d11_write() - .map(|(_lock,ds)| { + if is_ib || is_vb { + let buf_ptr = *ppBuffer as usize; + let byte_width = (*pDesc).ByteWidth; + + let has_pitch = !pInitialData.is_null() + && ((*pInitialData).SysMemPitch != 0 || (*pInitialData).SysMemSlicePitch != 0); + if has_pitch { + write_log_file("WARNING: hook_CreateBuffer: index or vertex buffer created with pitch or slice pitch, copy unimplemented"); + } + let initial_copy: Option> = + if !pInitialData.is_null() && !(*pInitialData).pSysMem.is_null() && !has_pitch { + let vlen = byte_width as usize; + let mut dest_v:Vec = Vec::with_capacity(vlen); + std::ptr::copy_nonoverlapping::((*pInitialData).pSysMem as *const u8, dest_v.as_mut_ptr(), vlen); + dest_v.set_len(vlen); + Some(dest_v) + } else { + None + }; + + dev_state_d3d11_write() + .map(|(_lock,ds)| { + ds.rs.device_buffer_meta.insert(buf_ptr, (is_ib, byte_width)); + if let Some(dest_v) = initial_copy { if is_ib { - ds.rs.device_index_buffer_data.insert(*ppBuffer as usize, dest_v); - ds.rs.device_index_buffer_createtime.push((*ppBuffer as usize, SystemTime::now())); + ds.rs.device_index_buffer_data.insert(buf_ptr, dest_v); + ds.rs.device_index_buffer_createtime.push((buf_ptr, SystemTime::now())); } else if is_vb { - ds.rs.device_vertex_buffer_data.insert(*ppBuffer as usize, dest_v); - ds.rs.device_vertex_buffer_createtime.push((*ppBuffer as usize, SystemTime::now())); + ds.rs.device_vertex_buffer_data.insert(buf_ptr, dest_v); + ds.rs.device_vertex_buffer_createtime.push((buf_ptr, SystemTime::now())); } - }); - } + } + }); } } diff --git a/Native/hook_core/src/hook_render_d3d11.rs b/Native/hook_core/src/hook_render_d3d11.rs index f18d7d1..7524e61 100644 --- a/Native/hook_core/src/hook_render_d3d11.rs +++ b/Native/hook_core/src/hook_render_d3d11.rs @@ -17,17 +17,19 @@ use types::native_mod::{ModD3DData, ModD3DState, NativeModData}; use winapi::ctypes::c_void; use winapi::shared::dxgiformat::{DXGI_FORMAT, DXGI_FORMAT_UNKNOWN, DXGI_FORMAT_R8G8B8A8_UNORM}; use winapi::shared::dxgitype::DXGI_SAMPLE_DESC; -use winapi::shared::winerror::{E_NOINTERFACE}; +use winapi::shared::winerror::{E_NOINTERFACE, E_FAIL}; use winapi::um::d3d11::{ID3D11Buffer, ID3D11InputLayout, D3D11_PRIMITIVE_TOPOLOGY, ID3D11ShaderResourceView, D3D11_SHADER_RESOURCE_VIEW_DESC, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, - ID3D11Texture2D, ID3D11Resource}; + ID3D11Texture2D, ID3D11Resource, + D3D11_MAP, D3D11_MAPPED_SUBRESOURCE, D3D11_BOX, + D3D11_MAP_WRITE, D3D11_MAP_WRITE_DISCARD, D3D11_MAP_WRITE_NO_OVERWRITE, D3D11_MAP_READ_WRITE}; use winapi::shared::ntdef::ULONG; use winapi::um::d3dcommon::{D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST, D3D11_SRV_DIMENSION_TEXTURE2D}; use winapi::um::processthreadsapi::GetCurrentProcessId; use winapi::um::unknwnbase::IUnknown; use winapi::um::winuser::{EnumWindows, GetWindowThreadProcessId, GetParent, GetDesktopWindow, GetForegroundWindow}; -use winapi::um::{d3d11::ID3D11DeviceContext, winnt::INT}; +use winapi::um::{d3d11::ID3D11DeviceContext, winnt::{INT, HRESULT}}; use winapi::shared::minwindef::UINT; use device_state::{dev_state_d3d11_read, dev_state_d3d11_write}; use shared_dx::error::{Result, HookError}; @@ -492,6 +494,198 @@ const MM_DISABLE:bool = true; /// their logic. const MM_DISABLE:bool = false; +/// True for map types that (may) write the buffer, i.e. those whose contents we want to capture. +#[inline] +fn is_write_map(map_type: D3D11_MAP) -> bool { + map_type == D3D11_MAP_WRITE + || map_type == D3D11_MAP_WRITE_DISCARD + || map_type == D3D11_MAP_WRITE_NO_OVERWRITE + || map_type == D3D11_MAP_READ_WRITE +} + +/// Insert/overwrite captured buffer bytes for a tracked VB/IB. +/// +/// A `createtime` entry is pushed only when the key is new. Pushing a duplicate `(ptr, time)` +/// tuple on every update would let the expiry GC (see `expire_data`) remove still-live data when +/// the oldest tuple's cutoff is reached. With this rule, a continuously-updated buffer self-heals: +/// after the GC eventually expires it, the next update finds the key absent and re-inserts the +/// data plus a fresh `createtime`, so any subsequent draw/snapshot still sees current bytes. +fn capture_buffer_data(rs: &mut DX11RenderState, is_ib: bool, buf_ptr: usize, data: Vec) { + if is_ib { + let is_new = !rs.device_index_buffer_data.contains_key(&buf_ptr); + rs.device_index_buffer_data.insert(buf_ptr, data); + if is_new { + rs.device_index_buffer_createtime.push((buf_ptr, SystemTime::now())); + } + } else { + let is_new = !rs.device_vertex_buffer_data.contains_key(&buf_ptr); + rs.device_vertex_buffer_data.insert(buf_ptr, data); + if is_new { + rs.device_vertex_buffer_createtime.push((buf_ptr, SystemTime::now())); + } + } +} + +/// Patch a sub-range of a tracked VB/IB's captured bytes (used for boxed UpdateSubresource). +/// Ensures a full-size (`byte_width`) zero-filled copy exists first, then overwrites +/// `[offset, offset+src.len())`. +fn patch_captured_buffer(rs: &mut DX11RenderState, is_ib: bool, buf_ptr: usize, + byte_width: usize, offset: usize, src: &[u8]) { + let (map, ctlist) = if is_ib { + (&mut rs.device_index_buffer_data, &mut rs.device_index_buffer_createtime) + } else { + (&mut rs.device_vertex_buffer_data, &mut rs.device_vertex_buffer_createtime) + }; + let is_new = !map.contains_key(&buf_ptr); + let entry = map.entry(buf_ptr).or_insert_with(|| vec![0u8; byte_width]); + if entry.len() < byte_width { + entry.resize(byte_width, 0u8); + } + let end = offset + src.len(); + if end <= entry.len() { + entry[offset..end].copy_from_slice(src); + } + if is_new { + ctlist.push((buf_ptr, SystemTime::now())); + } +} + +/// Hooked `ID3D11DeviceContext::Map`. When precopy is enabled, remembers the CPU pointer of a +/// write-mapped tracked VB/IB so `hook_Unmap` can copy its contents. Buffers the game fills via +/// Map (e.g. dynamic ring "megabuffers") are otherwise invisible to `hook_CreateBuffer`. +pub unsafe extern "system" fn hook_Map( + THIS: *mut ID3D11DeviceContext, + pResource: *mut ID3D11Resource, + Subresource: UINT, + MapType: D3D11_MAP, + MapFlags: UINT, + pMappedResource: *mut D3D11_MAPPED_SUBRESOURCE, +) -> HRESULT { + let hook_context = match get_hook_context() { + Ok(ctx) => ctx, + Err(_) => return E_FAIL, + }; + let hr = (hook_context.real_map)(THIS, pResource, Subresource, MapType, MapFlags, pMappedResource); + + if GLOBAL_STATE.run_conf.precopy_data + && hr == 0 + && Subresource == 0 + && !pMappedResource.is_null() + && is_write_map(MapType) { + let cpu_ptr = (*pMappedResource).pData as usize; + if cpu_ptr != 0 { + let res_key = pResource as usize; + // read-lock to check if this is a tracked VB/IB (skips the write lock for the very + // common constant-buffer Map case), then write-lock only to record the pending map. + let meta = dev_state_d3d11_read() + .and_then(|(_lck, state)| state.rs.device_buffer_meta.get(&res_key).copied()); + if let Some((is_ib, byte_width)) = meta { + dev_state_d3d11_write().map(|(_lock, ds)| { + ds.rs.mapped_buffers.insert(res_key, (cpu_ptr, is_ib, byte_width)); + }); + } + } + } + hr +} + +/// Hooked `ID3D11DeviceContext::Unmap`. Copies a pending write-mapped VB/IB's bytes into the +/// snapshot buffer store *before* calling the real Unmap (which invalidates the mapped pointer). +pub unsafe extern "system" fn hook_Unmap( + THIS: *mut ID3D11DeviceContext, + pResource: *mut ID3D11Resource, + Subresource: UINT, +) { + let hook_context = match get_hook_context() { + Ok(ctx) => ctx, + Err(_) => return, + }; + + if GLOBAL_STATE.run_conf.precopy_data && Subresource == 0 { + let res_key = pResource as usize; + // cheap read-lock membership check; only take the write lock for actually-tracked unmaps. + let is_pending = dev_state_d3d11_read() + .map(|(_lck, state)| state.rs.mapped_buffers.contains_key(&res_key)) + .unwrap_or(false); + if is_pending { + let pending = dev_state_d3d11_write() + .and_then(|(_lock, ds)| ds.rs.mapped_buffers.remove(&res_key)); + if let Some((cpu_ptr, is_ib, byte_width)) = pending { + if cpu_ptr != 0 && byte_width > 0 { + // copy outside the lock to minimize lock hold time; cpu_ptr stays valid until + // the real Unmap below. + let vlen = byte_width as usize; + let mut dest_v: Vec = Vec::with_capacity(vlen); + std::ptr::copy_nonoverlapping::(cpu_ptr as *const u8, dest_v.as_mut_ptr(), vlen); + dest_v.set_len(vlen); + dev_state_d3d11_write().map(|(_lock, ds)| { + capture_buffer_data(&mut ds.rs, is_ib, res_key, dest_v); + }); + } + } + } + } + + (hook_context.real_unmap)(THIS, pResource, Subresource); +} + +/// Hooked `ID3D11DeviceContext::UpdateSubresource`. Captures bytes written to a tracked VB/IB +/// for buffers the game updates this way instead of via Map. +pub unsafe extern "system" fn hook_UpdateSubresource( + THIS: *mut ID3D11DeviceContext, + pDstResource: *mut ID3D11Resource, + DstSubresource: UINT, + pDstBox: *const D3D11_BOX, + pSrcData: *const c_void, + SrcRowPitch: UINT, + SrcDepthPitch: UINT, +) { + let hook_context = match get_hook_context() { + Ok(ctx) => ctx, + Err(_) => return, + }; + + if GLOBAL_STATE.run_conf.precopy_data && DstSubresource == 0 && !pSrcData.is_null() { + let res_key = pDstResource as usize; + let meta = dev_state_d3d11_read() + .and_then(|(_lck, state)| state.rs.device_buffer_meta.get(&res_key).copied()); + if let Some((is_ib, byte_width)) = meta { + if byte_width > 0 { + if pDstBox.is_null() { + // full-resource update. + let vlen = byte_width as usize; + let mut dest_v: Vec = Vec::with_capacity(vlen); + std::ptr::copy_nonoverlapping::(pSrcData as *const u8, dest_v.as_mut_ptr(), vlen); + dest_v.set_len(vlen); + dev_state_d3d11_write().map(|(_lock, ds)| { + capture_buffer_data(&mut ds.rs, is_ib, res_key, dest_v); + }); + } else { + // boxed (partial) update: for a buffer, left/right are byte offsets. + let left = (*pDstBox).left as usize; + let right = (*pDstBox).right as usize; + let bw = byte_width as usize; + if right > left && right <= bw { + let span = right - left; + let mut src_copy: Vec = Vec::with_capacity(span); + std::ptr::copy_nonoverlapping::(pSrcData as *const u8, src_copy.as_mut_ptr(), span); + src_copy.set_len(span); + dev_state_d3d11_write().map(|(_lock, ds)| { + patch_captured_buffer(&mut ds.rs, is_ib, res_key, bw, left, &src_copy); + }); + } else { + write_log_file(&format!( + "hook_UpdateSubresource: ignoring out-of-range box update (left {}, right {}, byte_width {})", + left, right, bw)); + } + } + } + } + } + + (hook_context.real_update_subresource)(THIS, pDstResource, DstSubresource, pDstBox, pSrcData, SrcRowPitch, SrcDepthPitch); +} + pub unsafe extern "system" fn hook_draw_indexed( THIS: *mut ID3D11DeviceContext, IndexCount: UINT, diff --git a/Native/hook_snapshot/src/hook_snapshot.rs b/Native/hook_snapshot/src/hook_snapshot.rs index 0684ffb..1cf8020 100644 --- a/Native/hook_snapshot/src/hook_snapshot.rs +++ b/Native/hook_snapshot/src/hook_snapshot.rs @@ -762,13 +762,59 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn _ => return Err(HookError::SnapshotFailed(format!("unknown index buffer format: {:x}", curr_ibuffer_format))), }; - // should match expected size - let ex_size = (sd.prim_count * 3 * index_size) as usize; - if ib_copy.len() != ex_size { - return Err(HookError::SnapshotFailed(format!("index buffer data size mismatch, expected: {}, got: {}", ex_size, ib_copy.len()))); + let isz = index_size as usize; + + // The bound buffer may be a large shared/dynamic "megabuffer" holding many meshes, so + // extract just the index range this draw uses: prim_count*3 indices starting at + // start_index (plus any byte offset baked into the bound IB). Then find the min/max + // referenced vertex so we can carve out a self-contained, re-based mesh below. This + // generalizes the old per-mesh assumption (start_index==0, slice==whole buffer). + let ib_count = (sd.prim_count as usize) * 3; + let ib_byte_start = curr_ibuffer_offset as usize + (sd.start_index as usize) * isz; + let ib_byte_end = ib_byte_start + ib_count * isz; + if ib_count == 0 { + return Err(HookError::SnapshotFailed("no indices to snap".to_string())); + } + if ib_byte_end > ib_copy.len() { + return Err(HookError::SnapshotFailed(format!( + "index range out of bounds: need bytes [{}, {}) but index buffer copy is only {} bytes (start_index {}, prim_count {}); buffer data may be stale", + ib_byte_start, ib_byte_end, ib_copy.len(), sd.start_index, sd.prim_count))); + } + let read_index = |i: usize| -> u32 { + let off = ib_byte_start + i * isz; + if isz == 2 { + u16::from_le_bytes([ib_copy[off], ib_copy[off + 1]]) as u32 + } else { + u32::from_le_bytes([ib_copy[off], ib_copy[off + 1], ib_copy[off + 2], ib_copy[off + 3]]) + } + }; + let mut min_idx = u32::MAX; + let mut max_idx = 0u32; + for i in 0..ib_count { + let v = read_index(i); + if v < min_idx { min_idx = v; } + if v > max_idx { max_idx = v; } + } + if min_idx > max_idx { + return Err(HookError::SnapshotFailed("degenerate index range".to_string())); + } + let unique_verts = (max_idx - min_idx + 1) as usize; + + // Re-base indices to 0 (subtract min_idx) so they index into the vertex slice carved out + // below; managed uses index values directly, so they must be 0-based. + let mut ib_slice: Vec = Vec::with_capacity(ib_count * isz); + for i in 0..ib_count { + let rebased = read_index(i) - min_idx; + if isz == 2 { + ib_slice.extend_from_slice(&(rebased as u16).to_le_bytes()); + } else { + ib_slice.extend_from_slice(&rebased.to_le_bytes()); + } } - write_log_file(&format!("index buffer size: {}, format: {}", ib_copy.len(), curr_ibuffer_format)); + write_log_file(&format!( + "index buffer: full copy {} bytes, format {}; draw slice {} indices (start_index {}), vert range [{}, {}] -> {} unique verts", + ib_copy.len(), curr_ibuffer_format, ib_count, sd.start_index, min_idx, max_idx, unique_verts)); // now same for vertex buffers const MAX_VBUFFERS: usize = 16; @@ -782,28 +828,51 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn let _vb_rods = curr_vbuffers.iter().filter(|vb| !vb.is_null()) .map(|vb| ReleaseOnDrop::new(*vb)).collect::>(); - // filter active - let curr_vbuffers = curr_vbuffers.iter().filter(|vb| !vb.is_null()).collect::>(); - if curr_vbuffers.is_empty() { + // require exactly one active vertex buffer; remember its slot so we can honor its bind offset. + let active_slots = (0..MAX_VBUFFERS).filter(|&i| !curr_vbuffers[i].is_null()).collect::>(); + if active_slots.is_empty() { return Err(HookError::SnapshotFailed("no vertex buffers".to_string())); } - if curr_vbuffers.len() > 1 { - return Err(HookError::SnapshotFailed(format!("more than 1 vertex buffer not supported (got {})", curr_vbuffers.len()))); + if active_slots.len() > 1 { + return Err(HookError::SnapshotFailed(format!("more than 1 vertex buffer not supported (got {})", active_slots.len()))); } + let vb_slot = active_slots[0]; + let vb_ptr = curr_vbuffers[vb_slot]; // copy the data - let vb_copy = { - let vb_usize = *curr_vbuffers[0] as usize; - state.rs.device_vertex_buffer_data.get(&vb_usize).map(|v| v.clone()) - } + let vb_copy = state.rs.device_vertex_buffer_data.get(&(vb_ptr as usize)).map(|v| v.clone()) .ok_or_else(|| { HookError::SnapshotFailed("failed to get vertex buffer data, was not previously saved".to_string()) })?; - // number of vertices should be = size / vert size - let num_verts = vb_copy.len() / vert_size; - if sd.num_vertices != num_verts as u32 { - return Err(HookError::SnapshotFailed(format!("vertex buffer data size mismatch, expected: {}, got: {}", sd.num_vertices, num_verts))); + + // Carve out just the vertices this draw references: + // [base_vertex_index + min_idx .. base_vertex_index + max_idx + 1], honoring any byte + // offset baked into the bound VB. + let vb_total_verts = vb_copy.len() / vert_size; + let vstart_vert = sd.base_vertex_index as i64 + min_idx as i64; + if vstart_vert < 0 { + return Err(HookError::SnapshotFailed(format!( + "computed negative vertex start ({}); base_vertex_index {}, min_idx {}", + vstart_vert, sd.base_vertex_index, min_idx))); } - write_log_file(&format!("vertex buffer size: {}, num verts: {}, vertsize: {}", vb_copy.len(), num_verts, vert_size)); + let vb_byte_start = curr_vbuffer_offsets[vb_slot] as usize + (vstart_vert as usize) * vert_size; + let vb_byte_end = vb_byte_start + unique_verts * vert_size; + if vb_byte_end > vb_copy.len() { + return Err(HookError::SnapshotFailed(format!( + "vertex range out of bounds: need bytes [{}, {}) but vertex buffer copy is only {} bytes ({} verts); buffer data may be stale", + vb_byte_start, vb_byte_end, vb_copy.len(), vb_total_verts))); + } + let vb_slice: Vec = vb_copy[vb_byte_start..vb_byte_end].to_vec(); + + write_log_file(&format!("vertex buffer: full copy {} bytes ({} verts), vertsize {}; draw slice {} verts", + vb_copy.len(), vb_total_verts, vert_size, unique_verts)); + + // Rewrite the draw params so managed reads the self-contained slice from offset 0 (its + // well-tested path): num_vertices is the unique-vert count, and the offsets are now baked + // into the slices. + sd.num_vertices = unique_verts as u32; + sd.base_vertex_index = 0; + sd.min_vertex_index = 0; + sd.start_index = 0; // now save all the srvs that might contain textures, note any that are 2D and save the // indexes of those so that managed code has them @@ -828,10 +897,10 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn sd.rend_data.d3d11 = D3D11SnapshotRendData { layout_elems: decl_data, layout_size_bytes: layout_data_size as u64, - ib_data: ib_copy.as_ptr(), - vb_data: vb_copy.as_ptr(), - ib_size_bytes: ib_copy.len() as u64, - vb_size_bytes: vb_copy.len() as u64, + ib_data: ib_slice.as_ptr(), + vb_data: vb_slice.as_ptr(), + ib_size_bytes: ib_slice.len() as u64, + vb_size_bytes: vb_slice.len() as u64, ib_index_size_bytes: index_size as u32, vb_vert_size_bytes: vert_size as u32, act_tex_indices: tex_indices.as_ptr(), @@ -841,8 +910,8 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn return Ok(Box::new(D3D11SnapDeviceBuffers{ _context_rod: context_rod, _ld: ld, - _ib_data: ib_copy, - _vb_data: vb_copy, + _ib_data: ib_slice, + _vb_data: vb_slice, srvs: orig_srvs, srv_2d_tex: tex_indices, _srv_rods, diff --git a/Native/shared_dx/src/defs_dx11.rs b/Native/shared_dx/src/defs_dx11.rs index 2b172cd..d038f44 100644 --- a/Native/shared_dx/src/defs_dx11.rs +++ b/Native/shared_dx/src/defs_dx11.rs @@ -5,7 +5,8 @@ use winapi::shared::minwindef::{UINT, INT, ULONG}; use winapi::um::d3d11::{ID3D11Buffer, ID3D11InputLayout, D3D11_INPUT_ELEMENT_DESC, ID3D11Device, D3D11_PRIMITIVE_TOPOLOGY, ID3D11ShaderResourceView, D3D11_BUFFER_DESC, - D3D11_SUBRESOURCE_DATA, ID3D11Resource, D3D11_TEXTURE2D_DESC, ID3D11Texture2D}; + D3D11_SUBRESOURCE_DATA, ID3D11Resource, D3D11_TEXTURE2D_DESC, ID3D11Texture2D, + D3D11_MAP, D3D11_MAPPED_SUBRESOURCE, D3D11_BOX}; use winapi::um::d3d11::ID3D11DeviceContext; use winapi::um::unknwnbase::IUnknown; use winapi::um::winnt::HRESULT; @@ -77,6 +78,28 @@ pub type DrawIndexedFn = unsafe extern "system" fn ( StartIndexLocation: UINT, BaseVertexLocation: INT, ) -> (); +pub type MapFn = unsafe extern "system" fn ( + THIS: *mut ID3D11DeviceContext, + pResource: *mut ID3D11Resource, + Subresource: UINT, + MapType: D3D11_MAP, + MapFlags: UINT, + pMappedResource: *mut D3D11_MAPPED_SUBRESOURCE, +) -> HRESULT; +pub type UnmapFn = unsafe extern "system" fn ( + THIS: *mut ID3D11DeviceContext, + pResource: *mut ID3D11Resource, + Subresource: UINT, +) -> (); +pub type UpdateSubresourceFn = unsafe extern "system" fn ( + THIS: *mut ID3D11DeviceContext, + pDstResource: *mut ID3D11Resource, + DstSubresource: UINT, + pDstBox: *const D3D11_BOX, + pSrcData: *const c_void, + SrcRowPitch: UINT, + SrcDepthPitch: UINT, +) -> (); pub type DrawFn = unsafe extern "system" fn ( THIS: *mut ID3D11DeviceContext, VertexCount: UINT, diff --git a/Native/shared_dx/src/dx11rs.rs b/Native/shared_dx/src/dx11rs.rs index 0c57a64..c8db1f0 100644 --- a/Native/shared_dx/src/dx11rs.rs +++ b/Native/shared_dx/src/dx11rs.rs @@ -154,6 +154,17 @@ pub struct DX11RenderState { /// Controls when vertex data is removed pub device_vertex_buffer_createtime: Vec<(usize,SystemTime)>, pub device_vertex_buffer_totalsize_nextlog: (usize,usize), + /// Metadata for every index/vertex buffer created while precopy is enabled, keyed by buffer + /// pointer. The tuple is `(is_index_buffer, byte_width)`. This lets the Map/Unmap/ + /// UpdateSubresource hooks identify whether a resource being updated is a tracked VB/IB (and + /// its size) without a `GetDesc` call on the hot path. Needed because the game may create a + /// buffer empty and fill it later via Map or UpdateSubresource, in which case + /// `device_*_buffer_data` won't have an entry yet. + pub device_buffer_meta: FnvHashMap, + /// Pending Map->Unmap records keyed by resource pointer. The CPU pointer returned by Map is + /// only known at Map time, but the data must be copied at Unmap (before the real Unmap + /// invalidates the pointer). Tuple is `(cpu_ptr, is_index_buffer, byte_width)`. + pub mapped_buffers: FnvHashMap, } impl DX11RenderState { @@ -172,6 +183,8 @@ impl DX11RenderState { device_vertex_buffer_data: FnvHashMap::with_capacity_and_hasher(1600, Default::default()), device_vertex_buffer_createtime: Vec::new(), device_vertex_buffer_totalsize_nextlog: (0,0), + device_buffer_meta: FnvHashMap::with_capacity_and_hasher(1600, Default::default()), + mapped_buffers: FnvHashMap::with_capacity_and_hasher(16, Default::default()), } } diff --git a/Native/shared_dx/src/types_dx11.rs b/Native/shared_dx/src/types_dx11.rs index 0e1f031..36c5154 100644 --- a/Native/shared_dx/src/types_dx11.rs +++ b/Native/shared_dx/src/types_dx11.rs @@ -23,6 +23,9 @@ pub struct HookDirect3D11Context { pub real_ia_set_input_layout: IASetInputLayoutFn, pub real_ia_set_primitive_topology: IASetPrimitiveTopologyFn, pub real_ps_set_shader_resources: PSSetShaderResourcesFn, + pub real_map: MapFn, + pub real_unmap: UnmapFn, + pub real_update_subresource: UpdateSubresourceFn, } #[derive(Clone, Copy)] pub struct HookDirect3D11 { From 892b122c73a2a0aecec6ff44a4553cadf72bb954 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Aug 2026 19:55:51 +0000 Subject: [PATCH 3/4] make DX11 dynamic buffer snapshots usable, behind a feature flag Authored by Claude (claude-opus-5) Builds on the preceding commit, which could capture and slice buffers the game fills after creation but was not usable in practice: it could not be turned on without restarting the game, it made the game unplayably slow once on, and when it did produce a snapshot there was no way to tell a good one from a garbage one. Introduces "dynamic buffer" as the shorthand for an index or vertex buffer whose contents we get by watching the game write it, rather than from the pInitialData it was created with. Gated behind a new `snapshot-dynamic-buffers` Cargo feature, off by default, since snapshotting is still slower on this path than the plain whole-buffer one and the path it replaces has stricter size checks. Declared on hook_core, propagated to hook_snapshot and shared_dx. With it off the DLL behaves as it did before this work: the write hooks move to a gated hook_dynamic_buffers module and are not installed, their fn pointers leave HookDirect3D11Context (which is Copy and returned by value on every draw), hook_CreateBuffer records no metadata, and set_buffers_d3d11 hands over whole bound buffers with the original checks. Enabling precopy in game now works. The write hooks decided whether a resource was a tracked VB/IB by consulting metadata only hook_CreateBuffer writes, and that is only hooked when precopy was already on when the device was hooked -- so every buffer the game had already created was invisible, which is precisely the long-lived megabuffers this exists for. They now resolve unknown resources themselves via GetType/GetDesc and cache the answer, negative results included so that constant buffer writes stay cheap. Capture runs only while a snapshot is in progress rather than for the rest of the session. The cost is one copy of an entire buffer per write, and a game may write one many times per frame, which is the difference between slow and seconds per frame. SnapPreCopyAlways=1 restores always-on capture. The copy also reuses its destination allocation: at these sizes the allocator returns large blocks to the OS, so a fresh Vec per write meant faulting in new zero pages every time, costing more than the copy. Fixes a snapshot window that could fail to terminate. An earlier attempt here discounted capture time from the window's elapsed wall clock so a slow captured frame would not expire it; that is a feedback loop, since capture only runs while the window is open, and a frame that is nearly all capture advances the clock by nearly nothing. The window is wall clock again; snap_ms is the knob for making it longer. Adds a capture-provenance diagnostic, since a megabuffer part sometimes snapshots as poly soup and nothing in the log distinguished a stale copy from a current one. Each captured buffer records the write path that last produced its bytes (including the D3D11_MAP type), a global capture sequence number, a capture count and the stored length; the index and vertex buffer's are logged at snapshot time. Comparing the two sequence numbers says whether the pair came from the same batch of writes. Holding shift with the clear-texture-lists key drops everything captured from a dynamic buffer. These maps are keyed on raw buffer pointers, nothing hooks buffer Release, and the capture paths' staleness guards only run when a capture happens -- so after a scene change a snapshot can read bytes that no longer describe what the draw will read. Buffers filled at creation are kept, since DX11 will not read a buffer back and they are not affected. The two are told apart by the provenance above. The DX11Metrics counters this adds are named dyn_precopy_* and report under "dyn precopy:", to mark them as measuring only the atypical dynamic path; they stay zero in a default build. --- DEVNOTES.md | 42 +- Native/global_state/src/global_state.rs | 14 +- Native/hook_core/Cargo.toml | 7 +- Native/hook_core/src/hook_device_d3d11.rs | 122 ++++-- Native/hook_core/src/hook_dynamic_buffers.rs | 425 +++++++++++++++++++ Native/hook_core/src/hook_render.rs | 12 + Native/hook_core/src/hook_render_d3d11.rs | 202 +-------- Native/hook_core/src/input_commands.rs | 32 +- Native/hook_core/src/lib.rs | 2 + Native/hook_snapshot/Cargo.toml | 7 +- Native/hook_snapshot/src/hook_snapshot.rs | 241 +++++++---- Native/input/src/input.rs | 13 + Native/shared_dx/Cargo.toml | 9 +- Native/shared_dx/src/dx11rs.rs | 127 +++++- Native/shared_dx/src/types.rs | 22 + Native/shared_dx/src/types_dx11.rs | 7 + 16 files changed, 946 insertions(+), 338 deletions(-) create mode 100644 Native/hook_core/src/hook_dynamic_buffers.rs diff --git a/DEVNOTES.md b/DEVNOTES.md index 342b28f..bb6e8af 100644 --- a/DEVNOTES.md +++ b/DEVNOTES.md @@ -92,4 +92,44 @@ managed code so this diminishes the utility of moving this code to rust, since t be hot reloaded. The ability to reload the managed code is very useful during the process of adding support for a new game, since often tiny tweaks need to be made to formats and such and its very tedious to restart the whole game just for those. - +### DX11 dynamic buffer snapshots (the `snapshot-dynamic-buffers` feature) + +Some DX11 games pack many meshes into a few large buffers (a "megabuffer") that are created empty +and filled later via `Map`/`Unmap` or `UpdateSubresource`, which `hook_CreateBuffer` never sees. +The `snapshot-dynamic-buffers` Cargo feature captures those buffers and slices the drawn +sub-region out of them at snapshot time. It lives in `hook_core` and transitively enables the +same-named feature in `hook_snapshot` and `shared_dx`. Build with +`--features=snapshot-dynamic-buffers` from `Native/hook_core`; `rb.sh` and `r2026g1.sh` take a +feature name as their first argument. + +The code calls these **dynamic buffers**, as shorthand for "an index or vertex buffer whose +contents ModelMod obtains by watching the game write it, rather than from the `pInitialData` it +was created with". Anything described that way, and anything named `dyn_*`, only does something +when this feature is enabled; a default build sees only the static creation-time path. + +It is off by default because it is still slower at snapshotting than the plain path, and the +whole-buffer path it replaces has strict size checks that are a useful sanity check on games that +do use one buffer per mesh. + +Two things about it are worth knowing at this level; the code comments cover the rest. + +Capture only runs while a snapshot is in progress, not for the whole session, because the cost is +one copy of an entire buffer per *update* of that buffer and a game may refill one many times per +frame. `SnapPreCopyAlways=1` in the registry restores always-on capture, needed only if a game +refills its mesh buffers less often than one snap window. + +Holding **shift** with the clear-texture-lists key additionally drops everything captured from a +dynamic buffer, so the next snapshot uses freshly captured data or fails cleanly rather than +possibly serving bytes from a buffer the game has since destroyed. The key on its own leaves +captured data alone. Worth doing on entering a new scene; the cost is that a dropped buffer has +to be captured again before it can be snapshotted, which a game that writes its mesh buffers +rarely may not do within a snap window. Buffers filled at creation time are never dropped, since +DX11 cannot read a buffer back and they are not affected by the problem. + +Precopy itself can be turned on either by the `SnapPreCopyData` registry value at startup or in +game by the clear-texture-lists key. The in-game route works for dynamically updated buffers, but +a buffer filled once at creation time before precopy was on is unrecoverable without recreating +it, since DX11 will not read a buffer back. + +`process_metrics` reports capture count, MB copied, time spent and largest buffer; that is the +first thing to look at when snapshotting is slower than expected. diff --git a/Native/global_state/src/global_state.rs b/Native/global_state/src/global_state.rs index c579b53..38e84ed 100644 --- a/Native/global_state/src/global_state.rs +++ b/Native/global_state/src/global_state.rs @@ -114,6 +114,18 @@ pub struct ClrState { pub struct RunConf { pub precopy_data: bool, + /// When true (the default), DX11 dynamic buffer capture only copies buffers while a snapshot + /// is actually in progress (`is_snapping`), rather than on every write for the rest of the + /// session. Copying a large dynamic buffer on every one of its (often very many) writes per + /// frame is what makes precopy unplayably slow; the snapshot only needs the bytes written + /// during the frames it is capturing. + /// + /// Set the `SnapPreCopyAlways` registry dword to 1 to get the old always-on behavior, which + /// is the fallback if a game writes its mesh buffers less often than once per snap window. + /// + /// Only has an effect with the `snapshot-dynamic-buffers` feature: without it the registry + /// value is not even read, and nothing reads this field, so a default build ignores it. + pub precopy_only_when_snapping: bool, pub force_tex_cpu_read: bool, /// Game profile data loaded from the profile found for this registry key /// (example: `Software\ModelMod\Profiles\Profile0000`), or empty if none was found. @@ -219,6 +231,7 @@ lazy_static! { pub static mut GLOBAL_STATE: HookState = HookState { run_conf: RunConf { precopy_data: false, + precopy_only_when_snapping: true, force_tex_cpu_read: false, profile: EMPTY_GAME_PROFILE, }, @@ -273,7 +286,6 @@ pub static mut ANIM_SNAP_STATE:UnsafeCell> = UnsafeCell::n /// finishes loading. Callers should keep the lock for as short a span as /// possible — particularly on the DIP path. pub static LOADED_MODS: Mutex> = Mutex::new(None); - const TRACK_GS_PTR:bool = true; /// Container structure providing access to the global state pointer. diff --git a/Native/hook_core/Cargo.toml b/Native/hook_core/Cargo.toml index e34cf17..b7ecc1a 100644 --- a/Native/hook_core/Cargo.toml +++ b/Native/hook_core/Cargo.toml @@ -50,4 +50,9 @@ ProductVersion = "1.2.0.0" default = [] profile = [] mmdisable = [] -frequent-updates = [] \ No newline at end of file +frequent-updates = [] +# Experimental: capture dynamically-updated (Map/UpdateSubresource) DX11 index/vertex buffers and +# slice the drawn sub-region so meshes stored in large shared "megabuffers" can be snapshotted. +# Off by default: capture only runs during a snapshot, but it still makes snapshotting slower than +# the plain whole-buffer path, which also has stricter size checks. +snapshot-dynamic-buffers = ["hook_snapshot/snapshot-dynamic-buffers", "shared_dx/snapshot-dynamic-buffers"] \ No newline at end of file diff --git a/Native/hook_core/src/hook_device_d3d11.rs b/Native/hook_core/src/hook_device_d3d11.rs index 857b075..56cc239 100644 --- a/Native/hook_core/src/hook_device_d3d11.rs +++ b/Native/hook_core/src/hook_device_d3d11.rs @@ -384,22 +384,26 @@ pub unsafe fn apply_context_hooks(context:*mut ID3D11DeviceContext, first_hook:b (*vtbl).PSSetShaderResources = hook_PSSetShaderResources; func_hooked += 1; } - // Map/Unmap/UpdateSubresource are hooked so that buffers the game fills *after* creation - // (e.g. dynamic "megabuffers" updated via Map/WRITE_NO_OVERWRITE or UpdateSubresource) can be - // copied for snapshotting. Installed unconditionally like the draw hooks; the hook bodies are - // no-ops unless precopy_data is enabled, so the runtime precopy toggle works without rehooking - // the (already-copied) context vtable. - if (*vtbl).Map as usize != hook_Map as usize { - (*vtbl).Map = hook_Map; - func_hooked += 1; - } - if (*vtbl).Unmap as usize != hook_Unmap as usize { - (*vtbl).Unmap = hook_Unmap; - func_hooked += 1; - } - if (*vtbl).UpdateSubresource as usize != hook_UpdateSubresource as usize { - (*vtbl).UpdateSubresource = hook_UpdateSubresource; - func_hooked += 1; + // The dynamic buffer hooks (only with the `snapshot-dynamic-buffers` feature), so that buffers + // the game fills *after* creation -- e.g. a "megabuffer" written via Map/WRITE_NO_OVERWRITE or + // UpdateSubresource -- can be copied for snapshotting. Installed unconditionally within that + // feature, like the draw hooks; the hook bodies do nothing unless a snapshot is running, so + // the runtime precopy toggle works without rehooking the (already-copied) context vtable. + #[cfg(feature = "snapshot-dynamic-buffers")] + { + use crate::hook_dynamic_buffers::{hook_Map, hook_Unmap, hook_UpdateSubresource}; + if (*vtbl).Map as usize != hook_Map as *const () as usize { + (*vtbl).Map = hook_Map; + func_hooked += 1; + } + if (*vtbl).Unmap as usize != hook_Unmap as *const () as usize { + (*vtbl).Unmap = hook_Unmap; + func_hooked += 1; + } + if (*vtbl).UpdateSubresource as usize != hook_UpdateSubresource as *const () as usize { + (*vtbl).UpdateSubresource = hook_UpdateSubresource; + func_hooked += 1; + } } if TRACK_REHOOK_TIME { @@ -653,8 +657,11 @@ unsafe fn hook_d3d11(device:*mut ID3D11Device,_swapchain:*mut IDXGISwapChain, co let real_ia_set_input_layout = (*vtbl).IASetInputLayout; let real_ia_set_primitive_topology = (*vtbl).IASetPrimitiveTopology; let real_ps_set_shader_resources = (*vtbl).PSSetShaderResources; + #[cfg(feature = "snapshot-dynamic-buffers")] let real_map = (*vtbl).Map; + #[cfg(feature = "snapshot-dynamic-buffers")] let real_unmap = (*vtbl).Unmap; + #[cfg(feature = "snapshot-dynamic-buffers")] let real_update_subresource = (*vtbl).UpdateSubresource; // since we always make a copy of the vtable in the context at the moment, we don't search @@ -692,8 +699,11 @@ unsafe fn hook_d3d11(device:*mut ID3D11Device,_swapchain:*mut IDXGISwapChain, co real_ia_set_input_layout, real_ia_set_primitive_topology, real_ps_set_shader_resources, + #[cfg(feature = "snapshot-dynamic-buffers")] real_map, + #[cfg(feature = "snapshot-dynamic-buffers")] real_unmap, + #[cfg(feature = "snapshot-dynamic-buffers")] real_update_subresource, }; @@ -736,6 +746,16 @@ pub unsafe fn query_and_set_runconf_in_globalstate(check_precopy:bool) -> bool { } } + // Opt back in to copying mesh buffers on every update rather than only while a snapshot is + // running. Only useful for a game that updates its buffers less often than once per snap + // window; it is otherwise the difference between a playable framerate and a slideshow. + // Only the dynamic buffer capture reads this, so don't bother querying it without that. + #[cfg(feature = "snapshot-dynamic-buffers")] + { + GLOBAL_STATE.run_conf.precopy_only_when_snapping = + !matches!(util::reg_query_root_dword("SnapPreCopyAlways"), Ok(v) if v > 0); + } + let force_tex_cpu_read = util::reg_query_root_dword("SnapForceTexCpuRead"); let old_force_tex_cpu_read = GLOBAL_STATE.run_conf.force_tex_cpu_read; if let Ok(force_tex_cpu_read) = force_tex_cpu_read { @@ -746,8 +766,15 @@ pub unsafe fn query_and_set_runconf_in_globalstate(check_precopy:bool) -> bool { if old_force_tex_cpu_read != GLOBAL_STATE.run_conf.force_tex_cpu_read { changed = true; } - write_log_file(&format!("runconf: precopy data: {}, force tex cpu read: {} (setting changed: {})", - GLOBAL_STATE.run_conf.precopy_data, GLOBAL_STATE.run_conf.force_tex_cpu_read, + #[cfg(feature = "snapshot-dynamic-buffers")] + let dynbuf = format!(" (dynamic buffer capture only while snapping: {})", + GLOBAL_STATE.run_conf.precopy_only_when_snapping); + #[cfg(not(feature = "snapshot-dynamic-buffers"))] + let dynbuf = ""; + write_log_file(&format!("runconf: precopy data: {}{}, force tex cpu read: {} (setting changed: {})", + GLOBAL_STATE.run_conf.precopy_data, + dynbuf, + GLOBAL_STATE.run_conf.force_tex_cpu_read, changed, )); write_log_file(&format!("runconf: game profile: {:?}", GLOBAL_STATE.run_conf.profile)); @@ -1041,13 +1068,32 @@ unsafe extern "system" fn hook_CreateBuffer( ); if res == 0 && ppBuffer != null_mut() && (*ppBuffer) != null_mut() && !pDesc.is_null() { - // For index/vertex buffers, record metadata (type + size) so the Map/Unmap/ - // UpdateSubresource hooks can identify this buffer later -- even if it is created empty - // and filled afterwards (the "megabuffer" case). If initial data was supplied we also - // copy it out now, since DX11 offers no way to read the buffer back from the CPU later. + // For index/vertex buffers, record metadata (type + size) so the dynamic buffer hooks can + // identify this buffer later -- even if it is created empty and filled afterwards (the + // "megabuffer" case). If initial data was supplied we also copy it out now, since DX11 + // offers no way to read the buffer back from the CPU later; that copy is the ordinary + // static path and happens with or without the feature. let is_ib = (*pDesc).BindFlags & D3D11_BIND_INDEX_BUFFER != 0; let is_vb = (*pDesc).BindFlags & D3D11_BIND_VERTEX_BUFFER != 0; - if is_ib || is_vb { + if !(is_ib || is_vb) { + // Not a mesh buffer. The meta map is keyed on the raw pointer and D3D reuses freed + // addresses, so a constant buffer landing on a former VB/IB address would otherwise + // inherit that buffer's stale entry. Only correct that when an entry actually + // exists: constant buffer creation can be frequent and shouldn't pay for a write + // lock just to record an absence the dynamic buffer hooks would fill in themselves. + #[cfg(feature = "snapshot-dynamic-buffers")] + { + let stale = dev_state_d3d11_read() + .map(|(_lck, state)| state.rs.device_buffer_meta.contains_key(&(*ppBuffer as usize))) + .unwrap_or(false); + if stale { + dev_state_d3d11_write().map(|(_lock,ds)| { + ds.rs.device_buffer_meta.insert(*ppBuffer as usize, + shared_dx::dx11rs::BufferMeta::Other); + }); + } + } + } else { let buf_ptr = *ppBuffer as usize; let byte_width = (*pDesc).ByteWidth; @@ -1067,10 +1113,28 @@ unsafe extern "system" fn hook_CreateBuffer( None }; - dev_state_d3d11_write() - .map(|(_lock,ds)| { - ds.rs.device_buffer_meta.insert(buf_ptr, (is_ib, byte_width)); - if let Some(dest_v) = initial_copy { + #[cfg(feature = "snapshot-dynamic-buffers")] + dev_state_d3d11_write().map(|(_lock,ds)| { + ds.rs.device_buffer_meta.insert(buf_ptr, + shared_dx::dx11rs::BufferMeta::Mesh { is_index: is_ib, byte_width }); + }); + + // take the write lock only when there's data to store (matches original behavior). + if let Some(dest_v) = initial_copy { + dev_state_d3d11_write() + .map(|(_lock,ds)| { + #[cfg(feature = "snapshot-dynamic-buffers")] + { + use shared_dx::dx11rs::{BufferCaptureInfo, BufferWriteKind, + BUFFER_CAPTURE_SEQ}; + let seq = BUFFER_CAPTURE_SEQ.fetch_add(1, Ordering::Relaxed); + let count = ds.rs.buffer_capture_info.get(&buf_ptr) + .map(|i| i.count).unwrap_or(0) + 1; + ds.rs.buffer_capture_info.insert(buf_ptr, BufferCaptureInfo { + kind: BufferWriteKind::InitialData, + seq, count, bytes: dest_v.len(), + }); + } if is_ib { ds.rs.device_index_buffer_data.insert(buf_ptr, dest_v); ds.rs.device_index_buffer_createtime.push((buf_ptr, SystemTime::now())); @@ -1079,8 +1143,8 @@ unsafe extern "system" fn hook_CreateBuffer( ds.rs.device_vertex_buffer_data.insert(buf_ptr, dest_v); ds.rs.device_vertex_buffer_createtime.push((buf_ptr, SystemTime::now())); } - } - }); + }); + } } } diff --git a/Native/hook_core/src/hook_dynamic_buffers.rs b/Native/hook_core/src/hook_dynamic_buffers.rs new file mode 100644 index 0000000..2420390 --- /dev/null +++ b/Native/hook_core/src/hook_dynamic_buffers.rs @@ -0,0 +1,425 @@ +//! Experimental capture of DX11 "dynamic buffers" for snapshotting. +//! +//! Compiled only with the `snapshot-dynamic-buffers` feature, and the origin of the "dynamic +//! buffer" shorthand used throughout: an index or vertex buffer whose contents we get by watching +//! the game write it, rather than from the `pInitialData` it was created with. Some games pack +//! many meshes into one large such buffer (a "megabuffer") that is created empty and filled later +//! via `Map`/`Unmap` (often `WRITE_NO_OVERWRITE`) or `UpdateSubresource` -- neither of which +//! `hook_CreateBuffer` sees. These hooks copy the buffer bytes whenever the game fills them so the +//! snapshot code can read them back. +//! +//! Capture only runs while a snapshot is actually in progress; see `precopy_capture_active`. + +use std::sync::atomic::Ordering; +use std::time::SystemTime; + +use winapi::ctypes::c_void; +use winapi::shared::minwindef::UINT; +use winapi::shared::winerror::E_FAIL; +use winapi::um::winnt::HRESULT; +use winapi::um::d3d11::{ID3D11Buffer, ID3D11DeviceContext, ID3D11Resource, + D3D11_BUFFER_DESC, D3D11_MAP, D3D11_MAPPED_SUBRESOURCE, D3D11_BOX, + D3D11_BIND_INDEX_BUFFER, D3D11_BIND_VERTEX_BUFFER, + D3D11_RESOURCE_DIMENSION, D3D11_RESOURCE_DIMENSION_BUFFER, + D3D11_MAP_WRITE, D3D11_MAP_WRITE_DISCARD, D3D11_MAP_WRITE_NO_OVERWRITE, D3D11_MAP_READ_WRITE}; + +use global_state::GLOBAL_STATE; +use device_state::{dev_state_d3d11_read, dev_state_d3d11_write}; +use shared_dx::dx11rs::{BufferCaptureInfo, BufferMeta, BufferWriteKind, DX11RenderState, + BUFFER_CAPTURE_SEQ}; +use shared_dx::types::DX11Metrics; +use shared_dx::util::write_log_file; + +use crate::hook_render_d3d11::get_hook_context; + +/// True for map types that (may) write the buffer, i.e. those whose contents we want to capture. +#[inline] +fn is_write_map(map_type: D3D11_MAP) -> bool { + map_type == D3D11_MAP_WRITE + || map_type == D3D11_MAP_WRITE_DISCARD + || map_type == D3D11_MAP_WRITE_NO_OVERWRITE + || map_type == D3D11_MAP_READ_WRITE +} + +/// True when the Map/Unmap/UpdateSubresource hooks should actually copy buffer contents. +/// +/// Precopy being enabled is necessary but, by default, not sufficient. A game that packs meshes +/// into a large dynamic buffer typically refills it many times per frame, and copying +/// the whole thing on each of those writes costs orders of magnitude more than the snapshot +/// itself: at tens of MB a pop, a few hundred updates per frame is tens of GB of memcpy per frame. +/// The snapshot only needs the bytes that were written during the frames it is capturing, so by +/// default capture is restricted to the snapshot window (`is_snapping`, which lasts `snap_ms` +/// after the snap key). Set the `SnapPreCopyAlways` registry dword to 1 for the old always-on +/// behavior, needed only if a game updates its mesh buffers less often than that window. +#[inline] +fn precopy_capture_active() -> bool { + unsafe { + GLOBAL_STATE.run_conf.precopy_data + && (!GLOBAL_STATE.run_conf.precopy_only_when_snapping || GLOBAL_STATE.is_snapping) + } +} + +/// Record the cost of a capture so `process_metrics` can report it. Called with the device state +/// already write-locked for the copy, so it doesn't cost a second acquisition. +fn note_capture_metrics(metrics: &mut DX11Metrics, bytes: usize, nanos: u64) { + metrics.dyn_precopy_captures += 1; + metrics.dyn_precopy_bytes += bytes as u64; + metrics.dyn_precopy_nanos += nanos; + if bytes as u32 > metrics.dyn_precopy_largest { + metrics.dyn_precopy_largest = bytes as u32; + } +} + +/// Record how a buffer's stored bytes were just produced. Diagnostic only; see +/// `BufferCaptureInfo`. Called with the state already write-locked for the copy. +fn note_capture_provenance(rs: &mut DX11RenderState, buf_ptr: usize, kind: BufferWriteKind, + bytes: usize) { + let seq = BUFFER_CAPTURE_SEQ.fetch_add(1, Ordering::Relaxed); + let count = rs.buffer_capture_info.get(&buf_ptr).map(|i| i.count).unwrap_or(0) + 1; + rs.buffer_capture_info.insert(buf_ptr, BufferCaptureInfo { kind, seq, count, bytes }); +} + +/// Copy `len` bytes from `src` into the stored copy of a tracked VB/IB. +/// +/// The destination allocation is reused across updates rather than replaced with a fresh `Vec`. +/// These buffers are large and the game may refill one many times per frame, so allocating and +/// freeing a multi-megabyte block each time costs far more than the copy itself: the allocator +/// hands large blocks straight back to the OS, so every update would fault in a fresh set of +/// zero pages. Reusing the allocation means holding the write lock across the copy, which is the +/// cheaper of the two. +/// +/// A `createtime` entry is pushed only when the key is new. Pushing a duplicate `(ptr, time)` +/// tuple on every update would let the expiry GC (see `expire_data`) remove still-live data when +/// the oldest tuple's cutoff is reached. With this rule, a continuously-updated buffer self-heals: +/// after the GC eventually expires it, the next update finds the key absent and re-inserts the +/// data plus a fresh `createtime`, so any subsequent draw/snapshot still sees current bytes. +unsafe fn capture_buffer_data(rs: &mut DX11RenderState, is_ib: bool, buf_ptr: usize, + src: *const u8, len: usize, kind: BufferWriteKind) { + note_capture_provenance(rs, buf_ptr, kind, len); + let (map, ctlist) = if is_ib { + (&mut rs.device_index_buffer_data, &mut rs.device_index_buffer_createtime) + } else { + (&mut rs.device_vertex_buffer_data, &mut rs.device_vertex_buffer_createtime) + }; + let is_new = !map.contains_key(&buf_ptr); + let dest = map.entry(buf_ptr).or_insert_with(|| Vec::with_capacity(len)); + dest.clear(); + dest.reserve(len); + std::ptr::copy_nonoverlapping::(src, dest.as_mut_ptr(), len); + dest.set_len(len); + if is_new { + ctlist.push((buf_ptr, SystemTime::now())); + } +} + +/// Patch a sub-range of a tracked VB/IB's captured bytes (used for boxed UpdateSubresource). +/// Ensures a full-size (`byte_width`) zero-filled copy exists first, then overwrites +/// `[offset, offset+src.len())`. +unsafe fn patch_captured_buffer(rs: &mut DX11RenderState, is_ib: bool, buf_ptr: usize, + byte_width: usize, offset: usize, src: *const u8, src_len: usize) { + note_capture_provenance(rs, buf_ptr, BufferWriteKind::UpdateSubresourceBox( + offset as u32, (offset + src_len) as u32), byte_width); + let (map, ctlist) = if is_ib { + (&mut rs.device_index_buffer_data, &mut rs.device_index_buffer_createtime) + } else { + (&mut rs.device_vertex_buffer_data, &mut rs.device_vertex_buffer_createtime) + }; + let is_new = !map.contains_key(&buf_ptr); + let entry = map.entry(buf_ptr).or_insert_with(|| vec![0u8; byte_width]); + if entry.len() < byte_width { + entry.resize(byte_width, 0u8); + } + let end = offset + src_len; + if end <= entry.len() { + std::ptr::copy_nonoverlapping::(src, entry.as_mut_ptr().add(offset), src_len); + } + if is_new { + ctlist.push((buf_ptr, SystemTime::now())); + } +} + +/// How many buffers the lazy discovery path (`resolve_buffer_meta`) has found so far. Only used +/// to throttle logging, which is why Relaxed ordering is fine. +static LAZY_MESH_BUFFERS_FOUND: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); +/// Log at most this many lazily discovered mesh buffers, so that enabling precopy in game gives +/// visible confirmation in the log without spamming it for the rest of the session. +const LAZY_MESH_BUFFER_LOG_LIMIT: usize = 20; + +/// Ask a resource what it is. Returns `BufferMeta::Other` for anything that isn't a non-empty +/// vertex or index buffer. +/// +/// `GetType` is checked before casting to `ID3D11Buffer`: `ID3D11Buffer` derives from +/// `ID3D11Resource` so the cast is only valid once we know the resource really is a buffer. +unsafe fn query_buffer_meta(res: *mut ID3D11Resource) -> BufferMeta { + if res.is_null() { + return BufferMeta::Other; + } + let mut dim: D3D11_RESOURCE_DIMENSION = 0; + (*res).GetType(&mut dim); + if dim != D3D11_RESOURCE_DIMENSION_BUFFER { + return BufferMeta::Other; + } + let buf = res as *mut ID3D11Buffer; + let mut desc: D3D11_BUFFER_DESC = std::mem::zeroed(); + (*buf).GetDesc(&mut desc); + let is_ib = desc.BindFlags & D3D11_BIND_INDEX_BUFFER != 0; + let is_vb = desc.BindFlags & D3D11_BIND_VERTEX_BUFFER != 0; + if (is_ib || is_vb) && desc.ByteWidth > 0 { + BufferMeta::Mesh { is_index: is_ib, byte_width: desc.ByteWidth } + } else { + BufferMeta::Other + } +} + +/// Cheap "is this a VB/IB we care about" filter for the dynamic buffer write hot path. +/// +/// `hook_CreateBuffer` fills in `device_buffer_meta` at creation time, but it is only hooked when +/// precopy was already enabled when the device was hooked. When precopy is instead enabled at +/// runtime (`cmd_clear_texture_lists`), every buffer the game already created is missing from the +/// map -- and those are exactly the long-lived "megabuffers" we want, which are typically created +/// once at startup and only ever refilled, so waiting for a `CreateBuffer` that never comes means +/// the runtime toggle can never capture anything. +/// +/// So on a miss, ask the resource itself and cache the answer, negative results included: the +/// overwhelming majority of maps are constant buffers, and those must not pay for a `GetType`/ +/// `GetDesc` pair on every frame. The cache is bounded by the number of distinct resources the +/// game creates, so this is effectively a one-time cost per resource. +unsafe fn resolve_buffer_meta(res: *mut ID3D11Resource) -> BufferMeta { + let res_key = res as usize; + let cached = dev_state_d3d11_read() + .and_then(|(_lck, state)| state.rs.device_buffer_meta.get(&res_key).copied()); + if let Some(meta) = cached { + return meta; + } + + let meta = query_buffer_meta(res); + dev_state_d3d11_write().map(|(_lock, ds)| { + ds.rs.device_buffer_meta.insert(res_key, meta); + }); + if let BufferMeta::Mesh { is_index, byte_width } = meta { + let found = LAZY_MESH_BUFFERS_FOUND.fetch_add(1, Ordering::Relaxed); + if found < LAZY_MESH_BUFFER_LOG_LIMIT { + write_log_file(&format!( + "dyn precopy: now tracking {} buffer {:x} ({} bytes) discovered via update hook", + if is_index { "index" } else { "vertex" }, res_key, byte_width)); + if found == LAZY_MESH_BUFFER_LOG_LIMIT - 1 { + write_log_file("dyn precopy: (further buffer discoveries will not be logged)"); + } + } + } + meta +} + +/// Drop every captured buffer whose stored bytes came from a dynamic write. +/// +/// Called from the clear-texture-lists key, which is the point at which the user signals a fresh +/// start (typically after entering a new scene). Two things go stale across that boundary and +/// neither is self-correcting: +/// +/// The maps here are keyed on raw buffer pointers. Nothing hooks buffer `Release`, so when the +/// game destroys its megabuffers on a scene change and D3D hands the same addresses back for the +/// replacements, the old entries are still sitting there. A snapshot then reads a dead buffer's +/// bytes for a live buffer and produces a plausible-looking mesh made of the wrong triangles. +/// +/// Even without address reuse, a buffer the game filled long ago and has not written through any +/// path we observe since will hand back whatever it last held. +/// +/// After this, a dynamic buffer must be captured again before it can be snapshotted, so a +/// snapshot that would previously have produced garbage fails with "was not previously saved" +/// instead. That is the intended trade. +/// +/// Buffers whose bytes came from `pInitialData` are kept: DX11 will not read a buffer back, so +/// dropping those would permanently break snapshotting the ordinary static meshes, which have +/// nothing to do with this problem and are not stale. +pub fn reset_captured_dynamic_buffers() { + dev_state_d3d11_write().map(|(_lock, ds)| { + let rs = &mut ds.rs; + let mut dropped = 0usize; + let mut freed = 0usize; + { + // disjoint field borrows: the provenance map is read while the data maps are pruned. + let info = &rs.buffer_capture_info; + // anything we can't positively identify as static initial data is treated as dynamic; + // every capture path records provenance, so this should not happen in practice. + let keep = |ptr: &usize| matches!(info.get(ptr), + Some(i) if i.kind == BufferWriteKind::InitialData); + + rs.device_index_buffer_data.retain(|ptr, data| { + let k = keep(ptr); + if !k { dropped += 1; freed += data.len(); } + k + }); + rs.device_index_buffer_createtime.retain(|(ptr, _)| keep(ptr)); + rs.device_vertex_buffer_data.retain(|ptr, data| { + let k = keep(ptr); + if !k { dropped += 1; freed += data.len(); } + k + }); + rs.device_vertex_buffer_createtime.retain(|(ptr, _)| keep(ptr)); + } + rs.buffer_capture_info.retain(|_, i| i.kind == BufferWriteKind::InitialData); + // pure caches that re-populate on demand, so clearing them costs nothing and drops + // whatever was left behind by buffers that have since been released. + rs.device_buffer_meta.clear(); + rs.mapped_buffers.clear(); + + write_log_file(&format!( + "dyn precopy: dropped {} dynamic buffer(s) ({} bytes); each must be captured \ + again before it can be snapshotted", + dropped, freed)); + }); +} + +/// Hooked `ID3D11DeviceContext::Map`. When precopy is enabled, remembers the CPU pointer of a +/// write-mapped tracked VB/IB so `hook_Unmap` can copy its contents. Buffers the game fills via +/// Map (e.g. dynamic ring "megabuffers") are otherwise invisible to `hook_CreateBuffer`. +pub unsafe extern "system" fn hook_Map( + THIS: *mut ID3D11DeviceContext, + pResource: *mut ID3D11Resource, + Subresource: UINT, + MapType: D3D11_MAP, + MapFlags: UINT, + pMappedResource: *mut D3D11_MAPPED_SUBRESOURCE, +) -> HRESULT { + let hook_context = match get_hook_context() { + Ok(ctx) => ctx, + Err(_) => return E_FAIL, + }; + let hr = (hook_context.real_map)(THIS, pResource, Subresource, MapType, MapFlags, pMappedResource); + + if precopy_capture_active() + && hr == 0 + && Subresource == 0 + && !pMappedResource.is_null() + && is_write_map(MapType) { + let cpu_ptr = (*pMappedResource).pData as usize; + if cpu_ptr != 0 { + let res_key = pResource as usize; + // check whether this is a VB/IB we track (a cached lookup for all but the first map + // of a given resource, which keeps the very common constant-buffer case cheap), then + // write-lock only to record the pending map. + if let BufferMeta::Mesh { is_index, byte_width } = resolve_buffer_meta(pResource) { + dev_state_d3d11_write().map(|(_lock, ds)| { + ds.rs.mapped_buffers.insert(res_key, (cpu_ptr, is_index, byte_width, MapType)); + }); + } + } + } + hr +} + +/// Hooked `ID3D11DeviceContext::Unmap`. Copies a pending write-mapped VB/IB's bytes into the +/// snapshot buffer store *before* calling the real Unmap (which invalidates the mapped pointer). +pub unsafe extern "system" fn hook_Unmap( + THIS: *mut ID3D11DeviceContext, + pResource: *mut ID3D11Resource, + Subresource: UINT, +) { + let hook_context = match get_hook_context() { + Ok(ctx) => ctx, + Err(_) => return, + }; + + if GLOBAL_STATE.run_conf.precopy_data && Subresource == 0 { + let res_key = pResource as usize; + // cheap read-lock membership check; only take the write lock for actually-tracked unmaps. + let is_pending = dev_state_d3d11_read() + .map(|(_lck, state)| state.rs.mapped_buffers.contains_key(&res_key)) + .unwrap_or(false); + if is_pending { + let pending = dev_state_d3d11_write() + .and_then(|(_lock, ds)| ds.rs.mapped_buffers.remove(&res_key)); + if let Some((cpu_ptr, is_ib, byte_width, map_type)) = pending { + // Re-query rather than trusting the cached metadata before reading byte_width + // bytes out of the mapped pointer. The cache is keyed on a raw pointer and D3D + // reuses freed addresses, so a stale entry could otherwise send us reading past + // the end of a smaller buffer. This costs a GetDesc only on unmaps of buffers we + // actually intend to copy, which is negligible next to the copy itself. + let current = query_buffer_meta(pResource); + let confirmed = current == (BufferMeta::Mesh { is_index: is_ib, byte_width }); + if !confirmed { + dev_state_d3d11_write().map(|(_lock, ds)| { + ds.rs.device_buffer_meta.insert(res_key, current); + }); + write_log_file(&format!( + "hook_Unmap: skipping stale capture for resource {:x} (expected {:?}, found {:?})", + res_key, BufferMeta::Mesh { is_index: is_ib, byte_width }, current)); + } else if cpu_ptr != 0 && byte_width > 0 { + // cpu_ptr stays valid until the real Unmap below. + let vlen = byte_width as usize; + dev_state_d3d11_write().map(|(_lock, ds)| { + let start = SystemTime::now(); + capture_buffer_data(&mut ds.rs, is_ib, res_key, cpu_ptr as *const u8, vlen, + BufferWriteKind::Map(map_type)); + let nanos = start.elapsed().map(|d| d.as_nanos() as u64).unwrap_or(0); + note_capture_metrics(&mut ds.metrics, vlen, nanos); + }); + } + } + } + } + + (hook_context.real_unmap)(THIS, pResource, Subresource); +} + +/// Hooked `ID3D11DeviceContext::UpdateSubresource`. Captures bytes written to a tracked VB/IB +/// for buffers the game updates this way instead of via Map. +pub unsafe extern "system" fn hook_UpdateSubresource( + THIS: *mut ID3D11DeviceContext, + pDstResource: *mut ID3D11Resource, + DstSubresource: UINT, + pDstBox: *const D3D11_BOX, + pSrcData: *const c_void, + SrcRowPitch: UINT, + SrcDepthPitch: UINT, +) { + let hook_context = match get_hook_context() { + Ok(ctx) => ctx, + Err(_) => return, + }; + + if precopy_capture_active() && DstSubresource == 0 && !pSrcData.is_null() { + let res_key = pDstResource as usize; + // as in hook_Map, this resolves (and caches) the resource type on first sight so that + // buffers created before a runtime precopy enable are still picked up. + if let BufferMeta::Mesh { .. } = resolve_buffer_meta(pDstResource) { + // and as in hook_Unmap, take the size from a fresh query rather than the pointer-keyed + // cache before reading that many bytes out of pSrcData. + if let BufferMeta::Mesh { is_index: is_ib, byte_width } = query_buffer_meta(pDstResource) { + if pDstBox.is_null() { + // full-resource update. + let vlen = byte_width as usize; + dev_state_d3d11_write().map(|(_lock, ds)| { + let start = SystemTime::now(); + capture_buffer_data(&mut ds.rs, is_ib, res_key, pSrcData as *const u8, vlen, + BufferWriteKind::UpdateSubresource); + let nanos = start.elapsed().map(|d| d.as_nanos() as u64).unwrap_or(0); + note_capture_metrics(&mut ds.metrics, vlen, nanos); + }); + } else { + // boxed (partial) update: for a buffer, left/right are byte offsets. + let left = (*pDstBox).left as usize; + let right = (*pDstBox).right as usize; + let bw = byte_width as usize; + if right > left && right <= bw { + let span = right - left; + dev_state_d3d11_write().map(|(_lock, ds)| { + let start = SystemTime::now(); + patch_captured_buffer(&mut ds.rs, is_ib, res_key, bw, left, + pSrcData as *const u8, span); + let nanos = start.elapsed().map(|d| d.as_nanos() as u64).unwrap_or(0); + note_capture_metrics(&mut ds.metrics, span, nanos); + }); + } else { + write_log_file(&format!( + "hook_UpdateSubresource: ignoring out-of-range box update (left {}, right {}, byte_width {})", + left, right, bw)); + } + } + } + } + } + + (hook_context.real_update_subresource)(THIS, pDstResource, DstSubresource, pDstBox, pSrcData, SrcRowPitch, SrcDepthPitch); +} diff --git a/Native/hook_core/src/hook_render.rs b/Native/hook_core/src/hook_render.rs index 8009087..9350e03 100644 --- a/Native/hook_core/src/hook_render.rs +++ b/Native/hook_core/src/hook_render.rs @@ -146,6 +146,18 @@ pub fn process_metrics(preserve_prims:bool, interval:u32) { let rehook_ms = metrics.rehook_time_nanos / 1000 / 1000; write_log_file(&format!(" rehook calls: {}, total ms: {}", metrics.rehook_calls, rehook_ms)); } + if metrics.dyn_precopy_captures > 0 { + // If dynamic buffer copying is what's killing the framerate, it shows + // up here: compare the copy time against the interval it was measured + // over. + let mb = metrics.dyn_precopy_bytes as f64 / (1024.0 * 1024.0); + let copy_ms = metrics.dyn_precopy_nanos / 1000 / 1000; + write_log_file(&format!( + " dyn precopy: {} captures, {:.1} MB copied in {} ms (largest buffer {:.1} MB) over {} ms", + metrics.dyn_precopy_captures, mb, copy_ms, + metrics.dyn_precopy_largest as f64 / (1024.0 * 1024.0), + ms_since_reset)); + } if metrics.drawn_recently.len() > 0 { write_log_file(" drawn recently:"); for (pv, ds) in &metrics.drawn_recently { diff --git a/Native/hook_core/src/hook_render_d3d11.rs b/Native/hook_core/src/hook_render_d3d11.rs index 7524e61..4df1675 100644 --- a/Native/hook_core/src/hook_render_d3d11.rs +++ b/Native/hook_core/src/hook_render_d3d11.rs @@ -17,19 +17,17 @@ use types::native_mod::{ModD3DData, ModD3DState, NativeModData}; use winapi::ctypes::c_void; use winapi::shared::dxgiformat::{DXGI_FORMAT, DXGI_FORMAT_UNKNOWN, DXGI_FORMAT_R8G8B8A8_UNORM}; use winapi::shared::dxgitype::DXGI_SAMPLE_DESC; -use winapi::shared::winerror::{E_NOINTERFACE, E_FAIL}; +use winapi::shared::winerror::E_NOINTERFACE; use winapi::um::d3d11::{ID3D11Buffer, ID3D11InputLayout, D3D11_PRIMITIVE_TOPOLOGY, ID3D11ShaderResourceView, D3D11_SHADER_RESOURCE_VIEW_DESC, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, - ID3D11Texture2D, ID3D11Resource, - D3D11_MAP, D3D11_MAPPED_SUBRESOURCE, D3D11_BOX, - D3D11_MAP_WRITE, D3D11_MAP_WRITE_DISCARD, D3D11_MAP_WRITE_NO_OVERWRITE, D3D11_MAP_READ_WRITE}; + ID3D11Texture2D, ID3D11Resource}; use winapi::shared::ntdef::ULONG; use winapi::um::d3dcommon::{D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST, D3D11_SRV_DIMENSION_TEXTURE2D}; use winapi::um::processthreadsapi::GetCurrentProcessId; use winapi::um::unknwnbase::IUnknown; use winapi::um::winuser::{EnumWindows, GetWindowThreadProcessId, GetParent, GetDesktopWindow, GetForegroundWindow}; -use winapi::um::{d3d11::ID3D11DeviceContext, winnt::{INT, HRESULT}}; +use winapi::um::{d3d11::ID3D11DeviceContext, winnt::INT}; use winapi::shared::minwindef::UINT; use device_state::{dev_state_d3d11_read, dev_state_d3d11_write}; use shared_dx::error::{Result, HookError}; @@ -44,7 +42,7 @@ use fnv::FnvHashMap; /// so that callers do not need to hold the device-state lock while invoking /// the real fns (which can re-enter our hooks on the same thread and would /// otherwise deadlock). -fn get_hook_context() -> Result { +pub(crate) fn get_hook_context() -> Result { match dev_state_d3d11_read() { Some((_lck, state)) => Ok(state.hooks.context), None => { @@ -494,198 +492,6 @@ const MM_DISABLE:bool = true; /// their logic. const MM_DISABLE:bool = false; -/// True for map types that (may) write the buffer, i.e. those whose contents we want to capture. -#[inline] -fn is_write_map(map_type: D3D11_MAP) -> bool { - map_type == D3D11_MAP_WRITE - || map_type == D3D11_MAP_WRITE_DISCARD - || map_type == D3D11_MAP_WRITE_NO_OVERWRITE - || map_type == D3D11_MAP_READ_WRITE -} - -/// Insert/overwrite captured buffer bytes for a tracked VB/IB. -/// -/// A `createtime` entry is pushed only when the key is new. Pushing a duplicate `(ptr, time)` -/// tuple on every update would let the expiry GC (see `expire_data`) remove still-live data when -/// the oldest tuple's cutoff is reached. With this rule, a continuously-updated buffer self-heals: -/// after the GC eventually expires it, the next update finds the key absent and re-inserts the -/// data plus a fresh `createtime`, so any subsequent draw/snapshot still sees current bytes. -fn capture_buffer_data(rs: &mut DX11RenderState, is_ib: bool, buf_ptr: usize, data: Vec) { - if is_ib { - let is_new = !rs.device_index_buffer_data.contains_key(&buf_ptr); - rs.device_index_buffer_data.insert(buf_ptr, data); - if is_new { - rs.device_index_buffer_createtime.push((buf_ptr, SystemTime::now())); - } - } else { - let is_new = !rs.device_vertex_buffer_data.contains_key(&buf_ptr); - rs.device_vertex_buffer_data.insert(buf_ptr, data); - if is_new { - rs.device_vertex_buffer_createtime.push((buf_ptr, SystemTime::now())); - } - } -} - -/// Patch a sub-range of a tracked VB/IB's captured bytes (used for boxed UpdateSubresource). -/// Ensures a full-size (`byte_width`) zero-filled copy exists first, then overwrites -/// `[offset, offset+src.len())`. -fn patch_captured_buffer(rs: &mut DX11RenderState, is_ib: bool, buf_ptr: usize, - byte_width: usize, offset: usize, src: &[u8]) { - let (map, ctlist) = if is_ib { - (&mut rs.device_index_buffer_data, &mut rs.device_index_buffer_createtime) - } else { - (&mut rs.device_vertex_buffer_data, &mut rs.device_vertex_buffer_createtime) - }; - let is_new = !map.contains_key(&buf_ptr); - let entry = map.entry(buf_ptr).or_insert_with(|| vec![0u8; byte_width]); - if entry.len() < byte_width { - entry.resize(byte_width, 0u8); - } - let end = offset + src.len(); - if end <= entry.len() { - entry[offset..end].copy_from_slice(src); - } - if is_new { - ctlist.push((buf_ptr, SystemTime::now())); - } -} - -/// Hooked `ID3D11DeviceContext::Map`. When precopy is enabled, remembers the CPU pointer of a -/// write-mapped tracked VB/IB so `hook_Unmap` can copy its contents. Buffers the game fills via -/// Map (e.g. dynamic ring "megabuffers") are otherwise invisible to `hook_CreateBuffer`. -pub unsafe extern "system" fn hook_Map( - THIS: *mut ID3D11DeviceContext, - pResource: *mut ID3D11Resource, - Subresource: UINT, - MapType: D3D11_MAP, - MapFlags: UINT, - pMappedResource: *mut D3D11_MAPPED_SUBRESOURCE, -) -> HRESULT { - let hook_context = match get_hook_context() { - Ok(ctx) => ctx, - Err(_) => return E_FAIL, - }; - let hr = (hook_context.real_map)(THIS, pResource, Subresource, MapType, MapFlags, pMappedResource); - - if GLOBAL_STATE.run_conf.precopy_data - && hr == 0 - && Subresource == 0 - && !pMappedResource.is_null() - && is_write_map(MapType) { - let cpu_ptr = (*pMappedResource).pData as usize; - if cpu_ptr != 0 { - let res_key = pResource as usize; - // read-lock to check if this is a tracked VB/IB (skips the write lock for the very - // common constant-buffer Map case), then write-lock only to record the pending map. - let meta = dev_state_d3d11_read() - .and_then(|(_lck, state)| state.rs.device_buffer_meta.get(&res_key).copied()); - if let Some((is_ib, byte_width)) = meta { - dev_state_d3d11_write().map(|(_lock, ds)| { - ds.rs.mapped_buffers.insert(res_key, (cpu_ptr, is_ib, byte_width)); - }); - } - } - } - hr -} - -/// Hooked `ID3D11DeviceContext::Unmap`. Copies a pending write-mapped VB/IB's bytes into the -/// snapshot buffer store *before* calling the real Unmap (which invalidates the mapped pointer). -pub unsafe extern "system" fn hook_Unmap( - THIS: *mut ID3D11DeviceContext, - pResource: *mut ID3D11Resource, - Subresource: UINT, -) { - let hook_context = match get_hook_context() { - Ok(ctx) => ctx, - Err(_) => return, - }; - - if GLOBAL_STATE.run_conf.precopy_data && Subresource == 0 { - let res_key = pResource as usize; - // cheap read-lock membership check; only take the write lock for actually-tracked unmaps. - let is_pending = dev_state_d3d11_read() - .map(|(_lck, state)| state.rs.mapped_buffers.contains_key(&res_key)) - .unwrap_or(false); - if is_pending { - let pending = dev_state_d3d11_write() - .and_then(|(_lock, ds)| ds.rs.mapped_buffers.remove(&res_key)); - if let Some((cpu_ptr, is_ib, byte_width)) = pending { - if cpu_ptr != 0 && byte_width > 0 { - // copy outside the lock to minimize lock hold time; cpu_ptr stays valid until - // the real Unmap below. - let vlen = byte_width as usize; - let mut dest_v: Vec = Vec::with_capacity(vlen); - std::ptr::copy_nonoverlapping::(cpu_ptr as *const u8, dest_v.as_mut_ptr(), vlen); - dest_v.set_len(vlen); - dev_state_d3d11_write().map(|(_lock, ds)| { - capture_buffer_data(&mut ds.rs, is_ib, res_key, dest_v); - }); - } - } - } - } - - (hook_context.real_unmap)(THIS, pResource, Subresource); -} - -/// Hooked `ID3D11DeviceContext::UpdateSubresource`. Captures bytes written to a tracked VB/IB -/// for buffers the game updates this way instead of via Map. -pub unsafe extern "system" fn hook_UpdateSubresource( - THIS: *mut ID3D11DeviceContext, - pDstResource: *mut ID3D11Resource, - DstSubresource: UINT, - pDstBox: *const D3D11_BOX, - pSrcData: *const c_void, - SrcRowPitch: UINT, - SrcDepthPitch: UINT, -) { - let hook_context = match get_hook_context() { - Ok(ctx) => ctx, - Err(_) => return, - }; - - if GLOBAL_STATE.run_conf.precopy_data && DstSubresource == 0 && !pSrcData.is_null() { - let res_key = pDstResource as usize; - let meta = dev_state_d3d11_read() - .and_then(|(_lck, state)| state.rs.device_buffer_meta.get(&res_key).copied()); - if let Some((is_ib, byte_width)) = meta { - if byte_width > 0 { - if pDstBox.is_null() { - // full-resource update. - let vlen = byte_width as usize; - let mut dest_v: Vec = Vec::with_capacity(vlen); - std::ptr::copy_nonoverlapping::(pSrcData as *const u8, dest_v.as_mut_ptr(), vlen); - dest_v.set_len(vlen); - dev_state_d3d11_write().map(|(_lock, ds)| { - capture_buffer_data(&mut ds.rs, is_ib, res_key, dest_v); - }); - } else { - // boxed (partial) update: for a buffer, left/right are byte offsets. - let left = (*pDstBox).left as usize; - let right = (*pDstBox).right as usize; - let bw = byte_width as usize; - if right > left && right <= bw { - let span = right - left; - let mut src_copy: Vec = Vec::with_capacity(span); - std::ptr::copy_nonoverlapping::(pSrcData as *const u8, src_copy.as_mut_ptr(), span); - src_copy.set_len(span); - dev_state_d3d11_write().map(|(_lock, ds)| { - patch_captured_buffer(&mut ds.rs, is_ib, res_key, bw, left, &src_copy); - }); - } else { - write_log_file(&format!( - "hook_UpdateSubresource: ignoring out-of-range box update (left {}, right {}, byte_width {})", - left, right, bw)); - } - } - } - } - } - - (hook_context.real_update_subresource)(THIS, pDstResource, DstSubresource, pDstBox, pSrcData, SrcRowPitch, SrcDepthPitch); -} - pub unsafe extern "system" fn hook_draw_indexed( THIS: *mut ID3D11DeviceContext, IndexCount: UINT, diff --git a/Native/hook_core/src/input_commands.rs b/Native/hook_core/src/input_commands.rs index b335353..9fdde1a 100644 --- a/Native/hook_core/src/input_commands.rs +++ b/Native/hook_core/src/input_commands.rs @@ -190,6 +190,22 @@ fn cmd_clear_texture_lists(device: DevicePointer) { hook_snapshot::reset(); + // Holding shift additionally drops everything captured from a dynamic buffer. + // + // That is worth doing after a scene change, since those captures may belong to buffers the + // game has destroyed since (and whose addresses may now hold something else), but it is not + // always wanted: a dropped buffer has to be captured again before it can be snapshotted, and + // a game that writes its mesh buffers rarely may not oblige within a snap window. So the + // plain key keeps whatever has been captured, and shift asks for a clean slate. + #[cfg(feature = "snapshot-dynamic-buffers")] + { + if input::press_shift_down() { + crate::hook_dynamic_buffers::reset_captured_dynamic_buffers(); + } else { + write_log_file("hold shift with the clear key to also drop captured dynamic buffers"); + } + } + unsafe { if !GLOBAL_STATE.run_conf.precopy_data || !GLOBAL_STATE.run_conf.force_tex_cpu_read { // Since they pressed the clear texture key that signals they intend to snapshot, so @@ -204,7 +220,19 @@ fn cmd_clear_texture_lists(device: DevicePointer) { write_log_file(&format!("failed to reapply device hook: {:?}", e)) }).unwrap_or(false) }) { - write_log_file(&format!("==> precopy data now enabled; it was disabled, so you will need to reload game data for snapshots")); + write_log_file("==> precopy data now enabled; it was disabled at startup"); + #[cfg(feature = "snapshot-dynamic-buffers")] + if let DevicePointer::D3D11(_) = device { + // DX11: buffers the game refills via Map/UpdateSubresource are picked up from + // the next update onward without a reload, because the update hooks resolve + // buffer metadata on demand. Buffers that were filled once at creation time + // are a different story: DX11 can't read them back, and the CreateBuffer hook + // was not installed when they were made, so only a reload gets those. + write_log_file("==> DX11: dynamically updated meshes will be captured on their next update; \ + statically created ones still need a reload of the game data that owns them"); + } + #[cfg(not(feature = "snapshot-dynamic-buffers"))] + write_log_file("==> you will need to reload game data for snapshots"); } // For DX9: log whether force_tex_cpu_read is enabled so the user knows @@ -378,6 +406,7 @@ fn setup_fkey_input(device: DevicePointer, inp: &mut input::Input) { input::DIK_F4, Box::new(move || cmd_select_prev_texture(device)), ); + // hold shift to also drop captured dynamic buffers; see cmd_clear_texture_lists inp.add_press_fn(input::DIK_F6, Box::new(move || cmd_clear_texture_lists(device))); inp.add_press_fn(input::DIK_F7, Box::new(cmd_take_snapshot)); inp.add_press_fn(input::DIK_NUMPAD8, Box::new(select_next_variant)); @@ -394,6 +423,7 @@ fn setup_punct_input(device: DevicePointer, inp: &mut input::Input) { // If you change these, be sure to change LocStrings/ProfileText in MMLaunch! inp.add_press_fn(input::DIK_BACKSLASH, Box::new(move || cmd_reload_mods(device))); inp.add_press_fn(input::DIK_RBRACKET, Box::new(cmd_toggle_show_mods)); + // hold shift to also drop captured dynamic buffers; see cmd_clear_texture_lists inp.add_press_fn(input::DIK_SEMICOLON, Box::new(move || cmd_clear_texture_lists(device))); inp.add_press_fn( input::DIK_COMMA, diff --git a/Native/hook_core/src/lib.rs b/Native/hook_core/src/lib.rs index 9aff43e..9a64f04 100644 --- a/Native/hook_core/src/lib.rs +++ b/Native/hook_core/src/lib.rs @@ -43,6 +43,8 @@ mod hook_device; //mod hook_constants; mod mod_render; mod hook_device_d3d11; +#[cfg(feature = "snapshot-dynamic-buffers")] +mod hook_dynamic_buffers; pub use interop::{LogError, LogInfo, LogWarn}; pub use interop::{OnInitialized, SaveTexture}; diff --git a/Native/hook_snapshot/Cargo.toml b/Native/hook_snapshot/Cargo.toml index 42e4f46..abd07ff 100644 --- a/Native/hook_snapshot/Cargo.toml +++ b/Native/hook_snapshot/Cargo.toml @@ -25,4 +25,9 @@ lazy_static = "1.1.0" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["libloaderapi", "d3d9", "objidlbase", "processthreadsapi", "memoryapi", "winerror", "winuser", "winreg", - "dinput"] } \ No newline at end of file + "dinput"] } + +[features] +# See hook_core's snapshot-dynamic-buffers feature; this gates the snapshot-time sub-region +# slicing/re-basing in set_buffers_d3d11. Enabled transitively by hook_core's feature of the same name. +snapshot-dynamic-buffers = ["shared_dx/snapshot-dynamic-buffers"] diff --git a/Native/hook_snapshot/src/hook_snapshot.rs b/Native/hook_snapshot/src/hook_snapshot.rs index 1cf8020..fd36fbc 100644 --- a/Native/hook_snapshot/src/hook_snapshot.rs +++ b/Native/hook_snapshot/src/hook_snapshot.rs @@ -750,7 +750,9 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn .get(&(curr_ibuffer as usize)) .map(|v| v.clone()) .ok_or_else(|| { - HookError::SnapshotFailed("failed to get index buffer data, was not previously saved".to_string()) + HookError::SnapshotFailed(format!("failed to get index buffer data, was not previously saved. \ + if this is a dynamically updated buffer, the game may not have refilled it during \ + this snapshot window; try snapping again")) })?; // determine if 16 or 32 bit indices @@ -762,61 +764,7 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn _ => return Err(HookError::SnapshotFailed(format!("unknown index buffer format: {:x}", curr_ibuffer_format))), }; - let isz = index_size as usize; - - // The bound buffer may be a large shared/dynamic "megabuffer" holding many meshes, so - // extract just the index range this draw uses: prim_count*3 indices starting at - // start_index (plus any byte offset baked into the bound IB). Then find the min/max - // referenced vertex so we can carve out a self-contained, re-based mesh below. This - // generalizes the old per-mesh assumption (start_index==0, slice==whole buffer). - let ib_count = (sd.prim_count as usize) * 3; - let ib_byte_start = curr_ibuffer_offset as usize + (sd.start_index as usize) * isz; - let ib_byte_end = ib_byte_start + ib_count * isz; - if ib_count == 0 { - return Err(HookError::SnapshotFailed("no indices to snap".to_string())); - } - if ib_byte_end > ib_copy.len() { - return Err(HookError::SnapshotFailed(format!( - "index range out of bounds: need bytes [{}, {}) but index buffer copy is only {} bytes (start_index {}, prim_count {}); buffer data may be stale", - ib_byte_start, ib_byte_end, ib_copy.len(), sd.start_index, sd.prim_count))); - } - let read_index = |i: usize| -> u32 { - let off = ib_byte_start + i * isz; - if isz == 2 { - u16::from_le_bytes([ib_copy[off], ib_copy[off + 1]]) as u32 - } else { - u32::from_le_bytes([ib_copy[off], ib_copy[off + 1], ib_copy[off + 2], ib_copy[off + 3]]) - } - }; - let mut min_idx = u32::MAX; - let mut max_idx = 0u32; - for i in 0..ib_count { - let v = read_index(i); - if v < min_idx { min_idx = v; } - if v > max_idx { max_idx = v; } - } - if min_idx > max_idx { - return Err(HookError::SnapshotFailed("degenerate index range".to_string())); - } - let unique_verts = (max_idx - min_idx + 1) as usize; - - // Re-base indices to 0 (subtract min_idx) so they index into the vertex slice carved out - // below; managed uses index values directly, so they must be 0-based. - let mut ib_slice: Vec = Vec::with_capacity(ib_count * isz); - for i in 0..ib_count { - let rebased = read_index(i) - min_idx; - if isz == 2 { - ib_slice.extend_from_slice(&(rebased as u16).to_le_bytes()); - } else { - ib_slice.extend_from_slice(&rebased.to_le_bytes()); - } - } - - write_log_file(&format!( - "index buffer: full copy {} bytes, format {}; draw slice {} indices (start_index {}), vert range [{}, {}] -> {} unique verts", - ib_copy.len(), curr_ibuffer_format, ib_count, sd.start_index, min_idx, max_idx, unique_verts)); - - // now same for vertex buffers + // get the single active vertex buffer (common to both feature configs). const MAX_VBUFFERS: usize = 16; let mut curr_vbuffers: [*mut ID3D11Buffer; MAX_VBUFFERS] = [null_mut(); MAX_VBUFFERS]; let mut curr_vbuffer_strides: [UINT; MAX_VBUFFERS] = [0; MAX_VBUFFERS]; @@ -836,43 +784,144 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn if active_slots.len() > 1 { return Err(HookError::SnapshotFailed(format!("more than 1 vertex buffer not supported (got {})", active_slots.len()))); } - let vb_slot = active_slots[0]; - let vb_ptr = curr_vbuffers[vb_slot]; - // copy the data + let vb_ptr = curr_vbuffers[active_slots[0]]; let vb_copy = state.rs.device_vertex_buffer_data.get(&(vb_ptr as usize)).map(|v| v.clone()) .ok_or_else(|| { - HookError::SnapshotFailed("failed to get vertex buffer data, was not previously saved".to_string()) + HookError::SnapshotFailed(format!("failed to get vertex buffer data, was not previously saved. \ + if this is a dynamically updated buffer, the game may not have refilled it during \ + this snapshot window; try snapping again")) })?; - // Carve out just the vertices this draw references: - // [base_vertex_index + min_idx .. base_vertex_index + max_idx + 1], honoring any byte - // offset baked into the bound VB. - let vb_total_verts = vb_copy.len() / vert_size; - let vstart_vert = sd.base_vertex_index as i64 + min_idx as i64; - if vstart_vert < 0 { - return Err(HookError::SnapshotFailed(format!( - "computed negative vertex start ({}); base_vertex_index {}, min_idx {}", - vstart_vert, sd.base_vertex_index, min_idx))); - } - let vb_byte_start = curr_vbuffer_offsets[vb_slot] as usize + (vstart_vert as usize) * vert_size; - let vb_byte_end = vb_byte_start + unique_verts * vert_size; - if vb_byte_end > vb_copy.len() { - return Err(HookError::SnapshotFailed(format!( - "vertex range out of bounds: need bytes [{}, {}) but vertex buffer copy is only {} bytes ({} verts); buffer data may be stale", - vb_byte_start, vb_byte_end, vb_copy.len(), vb_total_verts))); - } - let vb_slice: Vec = vb_copy[vb_byte_start..vb_byte_end].to_vec(); + // Produce the index/vertex bytes to hand to managed. With the snapshot-dynamic-buffers + // feature we slice the draw's sub-region out of (possibly shared/mega) buffers and re-base + // the indices; otherwise we hand over the whole bound buffers (original per-mesh behavior). + #[cfg(feature = "snapshot-dynamic-buffers")] + let (ib_final, vb_final) = { + // Where each copy came from. When a snapshot comes out as garbage, the usual cause is + // that our stored bytes did not match what the draw actually read, so log the write + // path each buffer last took and how recently. "N ago" counts dynamic buffer captures + // of *any* buffer since this one, so a large gap between the two, or a large number on + // either, means the pair is not from the same batch of writes. A copy last written by + // Map(WRITE_DISCARD) is also suspect: discard renames the allocation, so everything + // the game did not rewrite in that map is recycled memory rather than prior contents. + { + use shared_dx::dx11rs::BUFFER_CAPTURE_SEQ; + let now_seq = BUFFER_CAPTURE_SEQ.load(std::sync::atomic::Ordering::Relaxed); + let describe = |ptr: usize| -> String { + match state.rs.buffer_capture_info.get(&ptr) { + Some(i) => format!("{} at capture {} ({} ago), {} captures total, {} bytes", + i.kind, i.seq, now_seq.saturating_sub(i.seq + 1), i.count, i.bytes), + None => "never captured".to_string(), + } + }; + write_log_file(&format!(" ib {:x} provenance: {}", + curr_ibuffer as usize, describe(curr_ibuffer as usize))); + write_log_file(&format!(" vb {:x} provenance: {}", + vb_ptr as usize, describe(vb_ptr as usize))); + } + + let isz = index_size as usize; + + // The bound buffer may be a large shared/dynamic "megabuffer" holding many meshes, so + // extract just the index range this draw uses: prim_count*3 indices starting at + // start_index (plus any byte offset baked into the bound IB). Then find the min/max + // referenced vertex so we can carve out a self-contained, re-based mesh below. This + // generalizes the old per-mesh assumption (start_index==0, slice==whole buffer). + let ib_count = (sd.prim_count as usize) * 3; + let ib_byte_start = curr_ibuffer_offset as usize + (sd.start_index as usize) * isz; + let ib_byte_end = ib_byte_start + ib_count * isz; + if ib_count == 0 { + return Err(HookError::SnapshotFailed("no indices to snap".to_string())); + } + if ib_byte_end > ib_copy.len() { + return Err(HookError::SnapshotFailed(format!( + "index range out of bounds: need bytes [{}, {}) but index buffer copy is only {} bytes (start_index {}, prim_count {}); buffer data may be stale", + ib_byte_start, ib_byte_end, ib_copy.len(), sd.start_index, sd.prim_count))); + } + let read_index = |i: usize| -> u32 { + let off = ib_byte_start + i * isz; + if isz == 2 { + u16::from_le_bytes([ib_copy[off], ib_copy[off + 1]]) as u32 + } else { + u32::from_le_bytes([ib_copy[off], ib_copy[off + 1], ib_copy[off + 2], ib_copy[off + 3]]) + } + }; + let mut min_idx = u32::MAX; + let mut max_idx = 0u32; + for i in 0..ib_count { + let v = read_index(i); + if v < min_idx { min_idx = v; } + if v > max_idx { max_idx = v; } + } + if min_idx > max_idx { + return Err(HookError::SnapshotFailed("degenerate index range".to_string())); + } + let unique_verts = (max_idx - min_idx + 1) as usize; + + // Re-base indices to 0 (subtract min_idx) so they index into the vertex slice carved out + // below; managed uses index values directly, so they must be 0-based. + let mut ib_slice: Vec = Vec::with_capacity(ib_count * isz); + for i in 0..ib_count { + let rebased = read_index(i) - min_idx; + if isz == 2 { + ib_slice.extend_from_slice(&(rebased as u16).to_le_bytes()); + } else { + ib_slice.extend_from_slice(&rebased.to_le_bytes()); + } + } - write_log_file(&format!("vertex buffer: full copy {} bytes ({} verts), vertsize {}; draw slice {} verts", - vb_copy.len(), vb_total_verts, vert_size, unique_verts)); + write_log_file(&format!( + "index buffer: full copy {} bytes, format {}; draw slice {} indices (start_index {}), vert range [{}, {}] -> {} unique verts", + ib_copy.len(), curr_ibuffer_format, ib_count, sd.start_index, min_idx, max_idx, unique_verts)); + + // Carve out just the vertices this draw references: + // [base_vertex_index + min_idx .. base_vertex_index + max_idx + 1], honoring any byte + // offset baked into the bound VB. + let vb_total_verts = vb_copy.len() / vert_size; + let vstart_vert = sd.base_vertex_index as i64 + min_idx as i64; + if vstart_vert < 0 { + return Err(HookError::SnapshotFailed(format!( + "computed negative vertex start ({}); base_vertex_index {}, min_idx {}", + vstart_vert, sd.base_vertex_index, min_idx))); + } + let vb_byte_start = curr_vbuffer_offsets[active_slots[0]] as usize + (vstart_vert as usize) * vert_size; + let vb_byte_end = vb_byte_start + unique_verts * vert_size; + if vb_byte_end > vb_copy.len() { + return Err(HookError::SnapshotFailed(format!( + "vertex range out of bounds: need bytes [{}, {}) but vertex buffer copy is only {} bytes ({} verts); buffer data may be stale", + vb_byte_start, vb_byte_end, vb_copy.len(), vb_total_verts))); + } + let vb_slice: Vec = vb_copy[vb_byte_start..vb_byte_end].to_vec(); + + write_log_file(&format!("vertex buffer: full copy {} bytes ({} verts), vertsize {}; draw slice {} verts", + vb_copy.len(), vb_total_verts, vert_size, unique_verts)); - // Rewrite the draw params so managed reads the self-contained slice from offset 0 (its - // well-tested path): num_vertices is the unique-vert count, and the offsets are now baked - // into the slices. - sd.num_vertices = unique_verts as u32; - sd.base_vertex_index = 0; - sd.min_vertex_index = 0; - sd.start_index = 0; + // Rewrite the draw params so managed reads the self-contained slice from offset 0 (its + // well-tested path): num_vertices is the unique-vert count, and the offsets are now baked + // into the slices. + sd.num_vertices = unique_verts as u32; + sd.base_vertex_index = 0; + sd.min_vertex_index = 0; + sd.start_index = 0; + + (ib_slice, vb_slice) + }; + + #[cfg(not(feature = "snapshot-dynamic-buffers"))] + let (ib_final, vb_final) = { + // original per-mesh behavior: hand over the whole bound buffers, with strict size checks. + let ex_size = (sd.prim_count * 3 * index_size) as usize; + if ib_copy.len() != ex_size { + return Err(HookError::SnapshotFailed(format!("index buffer data size mismatch, expected: {}, got: {}", ex_size, ib_copy.len()))); + } + write_log_file(&format!("index buffer size: {}, format: {}", ib_copy.len(), curr_ibuffer_format)); + let num_verts = vb_copy.len() / vert_size; + if sd.num_vertices != num_verts as u32 { + return Err(HookError::SnapshotFailed(format!("vertex buffer data size mismatch, expected: {}, got: {}", sd.num_vertices, num_verts))); + } + write_log_file(&format!("vertex buffer size: {}, num verts: {}, vertsize: {}", vb_copy.len(), num_verts, vert_size)); + (ib_copy, vb_copy) + }; // now save all the srvs that might contain textures, note any that are 2D and save the // indexes of those so that managed code has them @@ -897,10 +946,10 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn sd.rend_data.d3d11 = D3D11SnapshotRendData { layout_elems: decl_data, layout_size_bytes: layout_data_size as u64, - ib_data: ib_slice.as_ptr(), - vb_data: vb_slice.as_ptr(), - ib_size_bytes: ib_slice.len() as u64, - vb_size_bytes: vb_slice.len() as u64, + ib_data: ib_final.as_ptr(), + vb_data: vb_final.as_ptr(), + ib_size_bytes: ib_final.len() as u64, + vb_size_bytes: vb_final.len() as u64, ib_index_size_bytes: index_size as u32, vb_vert_size_bytes: vert_size as u32, act_tex_indices: tex_indices.as_ptr(), @@ -910,8 +959,8 @@ unsafe fn set_buffers_d3d11(device:*mut ID3D11Device, sd:&mut types::interop::Sn return Ok(Box::new(D3D11SnapDeviceBuffers{ _context_rod: context_rod, _ld: ld, - _ib_data: ib_slice, - _vb_data: vb_slice, + _ib_data: ib_final, + _vb_data: vb_final, srvs: orig_srvs, srv_2d_tex: tex_indices, _srv_rods, @@ -1022,6 +1071,12 @@ pub unsafe fn present_process() { if (*gs).is_snapping { let now = SystemTime::now(); let max_dur = std::time::Duration::from_millis(snap_ms as u64); + // Strictly wall clock, with nothing subtracted from it. Anything that shortens this in + // proportion to work done *during* the window can fail to terminate: buffer capture is + // only running because the window is open, so discounting its cost lets a frame that is + // ~entirely capture advance the clock by ~nothing, which keeps the window open, which + // keeps capture running. Use snap_ms to lengthen the window instead (DX11 generally + // wants more than the default; see the SNAP_CONFIG comment above). let elapsed = now .duration_since((*gs).snap_start) .unwrap_or(max_dur); diff --git a/Native/input/src/input.rs b/Native/input/src/input.rs index 5e29a0e..8b4d366 100644 --- a/Native/input/src/input.rs +++ b/Native/input/src/input.rs @@ -16,6 +16,7 @@ use winapi::shared::guiddef::{GUID, REFGUID, REFIID}; // use winapi::um::wingdi::RGNDATA; use fnv::FnvHashMap; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; //extern HRESULT WINAPI DirectInput8Create(HINSTANCE hinst, DWORD dwVersion, REFIID riidltf, LPVOID *ppvOut, LPUNKNOWN punkOuter); @@ -135,6 +136,17 @@ pub struct KeyEvent { pub pressed: bool, } +/// Whether shift was held when the presses currently being dispatched were read. Set just before +/// the press handlers run, so it is only meaningful from inside one. +static PRESS_SHIFT_DOWN: AtomicBool = AtomicBool::new(false); + +/// True if shift was held for the press being dispatched. Lets a command offer a variant of +/// itself on shift without needing a second key binding. Commands only run while ctrl (or menu) +/// is held, so this is the shift in ctrl-shift-. +pub fn press_shift_down() -> bool { + PRESS_SHIFT_DOWN.load(Ordering::Relaxed) +} + pub struct Input { events: Vec, keyboard_state: Vec, @@ -363,6 +375,7 @@ impl Input { let process_key_events = self.ctrl_pressed || menu_pressed; if process_key_events { + PRESS_SHIFT_DOWN.store(self.shift_pressed, Ordering::Relaxed); for evt in self.events.iter() { //write_log_file(&format!("event: {:x} pressed: {}", ke.key, ke.pressed)); if evt.pressed { diff --git a/Native/shared_dx/Cargo.toml b/Native/shared_dx/Cargo.toml index 519487f..bcfda34 100644 --- a/Native/shared_dx/Cargo.toml +++ b/Native/shared_dx/Cargo.toml @@ -13,4 +13,11 @@ winapi = { version = "0.3", features = ["libloaderapi", "d3d9", "d3d11", "objidl [dependencies] lazy_static = "*" -fnv = "1.0.6" \ No newline at end of file +fnv = "1.0.6" + +[features] +# See hook_core's snapshot-dynamic-buffers feature; this adds the Map/Unmap/UpdateSubresource +# function pointers to HookDirect3D11Context. That struct is Copy and is returned by value on +# every draw call, so the fields are kept out of the default build. Enabled transitively by +# hook_core and hook_snapshot's features of the same name. +snapshot-dynamic-buffers = [] diff --git a/Native/shared_dx/src/dx11rs.rs b/Native/shared_dx/src/dx11rs.rs index c8db1f0..9748068 100644 --- a/Native/shared_dx/src/dx11rs.rs +++ b/Native/shared_dx/src/dx11rs.rs @@ -119,6 +119,89 @@ impl Display for VertexFormat { } } +/// What a resource the game is writing turned out to be, as seen by the dynamic buffer hooks. +/// Cached by resource pointer in `DX11RenderState::device_buffer_meta` so that the very common +/// constant-buffer write costs a single hash lookup rather than a `GetType`/`GetDesc` pair. +/// +/// Note the cache is keyed on a raw pointer and D3D reuses freed addresses, so a cached value is +/// a filter, not a source of truth: capture paths re-query the resource before copying anything +/// out of it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BufferMeta { + /// A vertex or index buffer whose contents we want for snapshots, with its `ByteWidth`. + Mesh { is_index: bool, byte_width: u32 }, + /// Anything else: a constant buffer, a texture, a zero-sized buffer, etc. + Other, +} + +/// How a tracked mesh buffer's stored bytes were last produced. Diagnostic only: a snapshot that +/// comes out as garbage usually means our copy did not match what the draw actually read, and the +/// write path (and its D3D11_MAP type) is the first thing worth knowing. +/// +/// Recorded only while the dynamic buffer feature is on, which is why `InitialData` appears here +/// even though it describes an ordinary static buffer: with the feature on, the creation-time copy +/// is tracked alongside the dynamic ones so the two can be told apart later. +#[cfg(feature = "snapshot-dynamic-buffers")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BufferWriteKind { + /// Copied from `pInitialData` when the buffer was created. + InitialData, + /// Copied at `Unmap`; carries the `D3D11_MAP` type the game passed to `Map`. + Map(u32), + /// Whole-resource `UpdateSubresource`. + UpdateSubresource, + /// Boxed (partial) `UpdateSubresource` covering bytes `[left, right)`. + UpdateSubresourceBox(u32, u32), +} + +#[cfg(feature = "snapshot-dynamic-buffers")] +impl Display for BufferWriteKind { + fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { + match self { + BufferWriteKind::InitialData => write!(f, "CreateBuffer initial data"), + // D3D11_MAP values; spelled out here so the log doesn't need decoding. + BufferWriteKind::Map(t) => { + let name = match t { + 1 => "READ", + 2 => "WRITE", + 3 => "READ_WRITE", + 4 => "WRITE_DISCARD", + 5 => "WRITE_NO_OVERWRITE", + _ => "?", + }; + write!(f, "Map({})", name) + }, + BufferWriteKind::UpdateSubresource => write!(f, "UpdateSubresource"), + BufferWriteKind::UpdateSubresourceBox(l, r) => + write!(f, "UpdateSubresource[{}..{}]", l, r), + } + } +} + +/// Provenance of one tracked buffer's stored bytes. See `BUFFER_CAPTURE_SEQ` for what `seq` is +/// good for. +#[cfg(feature = "snapshot-dynamic-buffers")] +#[derive(Debug, Clone, Copy)] +pub struct BufferCaptureInfo { + pub kind: BufferWriteKind, + /// Value of `BUFFER_CAPTURE_SEQ` when this buffer was last captured. + pub seq: u64, + /// How many times this buffer has been captured. + pub count: u64, + /// Length in bytes of the stored copy after that capture. + pub bytes: usize, +} + +/// Monotonic counter incremented on every dynamic buffer capture. +/// +/// Its value is meaningless on its own; the point is the differences. Comparing the index and +/// vertex buffers' `seq` at snapshot time says whether the two copies came from the same batch of +/// writes or whether one of them is many captures stale, which is the difference between a +/// coherent mesh and a poly soup. +#[cfg(feature = "snapshot-dynamic-buffers")] +pub static BUFFER_CAPTURE_SEQ: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + pub struct DX11RenderState { /// Current vertex buffer properties, vector of (buf index,byte width,stride). pub vb_state: Vec<(u32,u32,u32)>, @@ -145,26 +228,42 @@ pub struct DX11RenderState { /// structure and you know there aren't any clones. pub device_semantic_string_table: FnvHashMap>, /// When snapshotting this stores all index buffer data, because we can't read it on the fly. + /// Holds copies taken at creation from `pInitialData` and, with the `snapshot-dynamic-buffers` + /// feature, copies captured from dynamic buffers as the game writes them. The two are mixed + /// here on purpose; `buffer_capture_info` records which is which. pub device_index_buffer_data: FnvHashMap>, /// Controls when index data is removed pub device_index_buffer_createtime: Vec<(usize,SystemTime)>, pub device_index_buffer_totalsize_nextlog: (usize,usize), /// When snapshotting this stores all vertex buffer data, because we can't read it on the fly. + /// Mixes creation-time and dynamic copies exactly as `device_index_buffer_data` does. pub device_vertex_buffer_data: FnvHashMap>, /// Controls when vertex data is removed pub device_vertex_buffer_createtime: Vec<(usize,SystemTime)>, pub device_vertex_buffer_totalsize_nextlog: (usize,usize), - /// Metadata for every index/vertex buffer created while precopy is enabled, keyed by buffer - /// pointer. The tuple is `(is_index_buffer, byte_width)`. This lets the Map/Unmap/ - /// UpdateSubresource hooks identify whether a resource being updated is a tracked VB/IB (and - /// its size) without a `GetDesc` call on the hot path. Needed because the game may create a - /// buffer empty and fill it later via Map or UpdateSubresource, in which case + /// What each resource the dynamic buffer hooks have seen turned out to be, keyed by resource + /// pointer. This lets them identify whether a resource being written is a VB/IB we want to + /// capture (and how big it is) without repeating the `GetType`/`GetDesc` calls on the hot + /// path. Needed because the game may create a buffer empty and fill it later, in which case /// `device_*_buffer_data` won't have an entry yet. - pub device_buffer_meta: FnvHashMap, - /// Pending Map->Unmap records keyed by resource pointer. The CPU pointer returned by Map is - /// only known at Map time, but the data must be copied at Unmap (before the real Unmap - /// invalidates the pointer). Tuple is `(cpu_ptr, is_index_buffer, byte_width)`. - pub mapped_buffers: FnvHashMap, + /// + /// Entries are written both by `hook_CreateBuffer` and lazily by the dynamic buffer hooks the + /// first time they see an unknown resource. The lazy path is what makes a runtime precopy + /// enable work on buffers that already existed before the toggle. + /// + /// Only populated with the `snapshot-dynamic-buffers` feature; empty otherwise. + pub device_buffer_meta: FnvHashMap, + /// Writes in progress on a dynamic buffer, keyed by resource pointer. The CPU pointer the + /// game was handed is only known when the write starts, but the data must be copied when it + /// ends (before the pointer is invalidated). Tuple is + /// `(cpu_ptr, is_index_buffer, byte_width, map_type)`. + /// + /// Only populated with the `snapshot-dynamic-buffers` feature; empty otherwise. + pub mapped_buffers: FnvHashMap, + /// Diagnostic provenance for each captured mesh buffer, keyed by buffer pointer. Written by + /// the dynamic buffer capture paths, read and logged by the snapshot code. + #[cfg(feature = "snapshot-dynamic-buffers")] + pub buffer_capture_info: FnvHashMap, } impl DX11RenderState { @@ -183,8 +282,12 @@ impl DX11RenderState { device_vertex_buffer_data: FnvHashMap::with_capacity_and_hasher(1600, Default::default()), device_vertex_buffer_createtime: Vec::new(), device_vertex_buffer_totalsize_nextlog: (0,0), - device_buffer_meta: FnvHashMap::with_capacity_and_hasher(1600, Default::default()), - mapped_buffers: FnvHashMap::with_capacity_and_hasher(16, Default::default()), + // Allocate lazily (no capacity): these stay empty unless the snapshot-dynamic-buffers + // feature is enabled and actively capturing, so this keeps the default build zero-cost. + device_buffer_meta: FnvHashMap::default(), + mapped_buffers: FnvHashMap::default(), + #[cfg(feature = "snapshot-dynamic-buffers")] + buffer_capture_info: FnvHashMap::default(), } } diff --git a/Native/shared_dx/src/types.rs b/Native/shared_dx/src/types.rs index cf99953..20cfc6a 100644 --- a/Native/shared_dx/src/types.rs +++ b/Native/shared_dx/src/types.rs @@ -50,6 +50,20 @@ pub struct DX11Metrics { pub drawn_recently: FnvHashMap<(u32,u32),MetricsDrawStatus>, // (prim,vert) => (mtype,count) pub rehook_time_nanos: u64, pub rehook_calls: u32, + // The `dyn_precopy_` fields below measure only the copies made out of dynamic buffers, which + // is the atypical case: they stay zero unless the `snapshot-dynamic-buffers` feature is + // enabled and a snapshot is actively capturing. The ordinary static path (a copy of + // `pInitialData` taken in `hook_CreateBuffer`) is not counted here, being a one-off per buffer + // rather than a recurring per-update cost. + /// Number of dynamic buffer captures done for precopy. + pub dyn_precopy_captures: u32, + /// Bytes copied by those captures. Watch this: a large buffer that the game refills many + /// times per frame can push it into the GB/sec range, which is what makes precopy crawl. + pub dyn_precopy_bytes: u64, + /// Wall time spent inside those copies. + pub dyn_precopy_nanos: u64, + /// Largest single buffer captured, in bytes. + pub dyn_precopy_largest: u32, } impl DX11Metrics { @@ -61,6 +75,10 @@ impl DX11Metrics { drawn_recently: FnvHashMap::default(), rehook_time_nanos: 0, rehook_calls: 0, + dyn_precopy_captures: 0, + dyn_precopy_bytes: 0, + dyn_precopy_nanos: 0, + dyn_precopy_largest: 0, } } pub fn reset(&mut self) { @@ -70,6 +88,10 @@ impl DX11Metrics { self.drawn_recently.clear(); self.rehook_time_nanos = 0; self.rehook_calls = 0; + self.dyn_precopy_captures = 0; + self.dyn_precopy_bytes = 0; + self.dyn_precopy_nanos = 0; + self.dyn_precopy_largest = 0; } /// Return number of milisecs since last reset pub fn ms_since_reset(&self) -> u64 { diff --git a/Native/shared_dx/src/types_dx11.rs b/Native/shared_dx/src/types_dx11.rs index 36c5154..0ea92fc 100644 --- a/Native/shared_dx/src/types_dx11.rs +++ b/Native/shared_dx/src/types_dx11.rs @@ -23,8 +23,15 @@ pub struct HookDirect3D11Context { pub real_ia_set_input_layout: IASetInputLayoutFn, pub real_ia_set_primitive_topology: IASetPrimitiveTopologyFn, pub real_ps_set_shader_resources: PSSetShaderResourcesFn, + // The real fns behind the dynamic buffer hooks, so only present with + // `snapshot-dynamic-buffers`. This struct is `Copy` and is returned by value from + // `get_hook_context()` on every draw and IA state call, so keep the default build's copy + // exactly the size it was before the feature existed. + #[cfg(feature = "snapshot-dynamic-buffers")] pub real_map: MapFn, + #[cfg(feature = "snapshot-dynamic-buffers")] pub real_unmap: UnmapFn, + #[cfg(feature = "snapshot-dynamic-buffers")] pub real_update_subresource: UpdateSubresourceFn, } #[derive(Clone, Copy)] From 2e5a8d0ed932f609976bc1b1b07a377ac0616e97 Mon Sep 17 00:00:00 2001 From: John Quigley Date: Wed, 26 Aug 2026 16:28:02 -0400 Subject: [PATCH 4/4] Update some comments related to snapshot-dynamic-buffers - Add some comments to devnotes on top of claude's thing which is mostly about details - Update some of the code comments that claude added --- DEVNOTES.md | 14 ++++++++++++++ Native/global_state/src/global_state.rs | 7 ++++--- Native/hook_core/src/input_commands.rs | 12 ++---------- Native/hook_snapshot/src/hook_snapshot.rs | 15 +++++++-------- Native/shared_dx/src/types_dx11.rs | 2 ++ 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/DEVNOTES.md b/DEVNOTES.md index bb6e8af..ef391b8 100644 --- a/DEVNOTES.md +++ b/DEVNOTES.md @@ -94,6 +94,20 @@ tedious to restart the whole game just for those. ### DX11 dynamic buffer snapshots (the `snapshot-dynamic-buffers` feature) +MM assumes that each mesh has its own vertex/index buffer and these are largely static - i.e the game isn't updating them every frame to do, for instance, software animation - which isn't supported. + +At least one game takes this approach for most of its character meshes, so is mostly moddable, but will draw some parts out of a large buffer that is dynamically updated (and animated on the CPU). + +This was more common in older game engines, which had limits on the +number of bone transfers they could squeeze into DX9 shader constants (~256) - so things like a cape might be split out into a separate draw, because the game couldn't fit that into a single draw call with the rest of the character (which can have many bones for limbs, facial and finger animations). + +Normally these parts will be lost in snapshot, and if software-animated (which is likely), they could not be modded anyway. However _if_ the part was snapshotted, in some cases it is possible in practice to "weld" it back to a GPU animated part that is moddable, by hand in blender, and it looks ok despite the host part not having all the weights needed to render it like the original. +The original can then be hidden with a deletion mod, which has the host part as a parent to reduce the chance of misfire. This isn't theoretical as I've done it at least once. 🙃 + +`snapshot-dynamic-buffers` was added to support this case. When built with this (in DX11), the game will track those buffers in an attempt provide the ability to snapshot pieces like this. But since it typically introduces a performance hit when enabled, especially when snapshotting, possibly resulting in missing static parts if the snap window is too short, it is off by default. So normally uou want to turn it on, snapshot what you need, then turn it off. + +Now some details provided by Claude (I skimmed these at least once): + Some DX11 games pack many meshes into a few large buffers (a "megabuffer") that are created empty and filled later via `Map`/`Unmap` or `UpdateSubresource`, which `hook_CreateBuffer` never sees. The `snapshot-dynamic-buffers` Cargo feature captures those buffers and slices the drawn diff --git a/Native/global_state/src/global_state.rs b/Native/global_state/src/global_state.rs index 38e84ed..9a952ec 100644 --- a/Native/global_state/src/global_state.rs +++ b/Native/global_state/src/global_state.rs @@ -114,6 +114,9 @@ pub struct ClrState { pub struct RunConf { pub precopy_data: bool, + /// This only has an effect with the `snapshot-dynamic-buffers` feature: without it the registry + /// value is not even read, and nothing reads this field, so a default build ignores it. + /// /// When true (the default), DX11 dynamic buffer capture only copies buffers while a snapshot /// is actually in progress (`is_snapping`), rather than on every write for the rest of the /// session. Copying a large dynamic buffer on every one of its (often very many) writes per @@ -122,9 +125,7 @@ pub struct RunConf { /// /// Set the `SnapPreCopyAlways` registry dword to 1 to get the old always-on behavior, which /// is the fallback if a game writes its mesh buffers less often than once per snap window. - /// - /// Only has an effect with the `snapshot-dynamic-buffers` feature: without it the registry - /// value is not even read, and nothing reads this field, so a default build ignores it. + /// Note the framerate reduction from doing that may well be severe. pub precopy_only_when_snapping: bool, pub force_tex_cpu_read: bool, /// Game profile data loaded from the profile found for this registry key diff --git a/Native/hook_core/src/input_commands.rs b/Native/hook_core/src/input_commands.rs index 9fdde1a..2f0cd36 100644 --- a/Native/hook_core/src/input_commands.rs +++ b/Native/hook_core/src/input_commands.rs @@ -191,12 +191,9 @@ fn cmd_clear_texture_lists(device: DevicePointer) { hook_snapshot::reset(); // Holding shift additionally drops everything captured from a dynamic buffer. + // (which is only relevant to the `snapshot-dynamic-buffers` feature) // - // That is worth doing after a scene change, since those captures may belong to buffers the - // game has destroyed since (and whose addresses may now hold something else), but it is not - // always wanted: a dropped buffer has to be captured again before it can be snapshotted, and - // a game that writes its mesh buffers rarely may not oblige within a snap window. So the - // plain key keeps whatever has been captured, and shift asks for a clean slate. + // If a dynamic buffer snapshot contains garbage, it may be worth trying this to refresh the buffer contents. #[cfg(feature = "snapshot-dynamic-buffers")] { if input::press_shift_down() { @@ -223,11 +220,6 @@ fn cmd_clear_texture_lists(device: DevicePointer) { write_log_file("==> precopy data now enabled; it was disabled at startup"); #[cfg(feature = "snapshot-dynamic-buffers")] if let DevicePointer::D3D11(_) = device { - // DX11: buffers the game refills via Map/UpdateSubresource are picked up from - // the next update onward without a reload, because the update hooks resolve - // buffer metadata on demand. Buffers that were filled once at creation time - // are a different story: DX11 can't read them back, and the CreateBuffer hook - // was not installed when they were made, so only a reload gets those. write_log_file("==> DX11: dynamically updated meshes will be captured on their next update; \ statically created ones still need a reload of the game data that owns them"); } diff --git a/Native/hook_snapshot/src/hook_snapshot.rs b/Native/hook_snapshot/src/hook_snapshot.rs index fd36fbc..7547489 100644 --- a/Native/hook_snapshot/src/hook_snapshot.rs +++ b/Native/hook_snapshot/src/hook_snapshot.rs @@ -150,9 +150,9 @@ pub unsafe fn take(devptr:&mut DevicePointer, sd:&mut types::interop::SnapshotDa // For an indexed triangle list, the most unique verts a draw can touch is prim_count * 3 // (no shared indices). If num_vertices exceeds that, it most likely came from // vb_size / vert_size (the whole bound VB) rather than this draw's range -- see - // compute_prim_vert_count in hook_render_d3d11.rs. That's a strong hint that either the - // VB is shared across many draws, or the geometry is CPU/software-animated and not - // usefully snapshottable, and may also explain a subsequent missing-index-buffer error. + // compute_prim_vert_count in hook_render_d3d11.rs. That's a strong hint that the + // VB is shared across many draws and/or the geometry is CPU/software-animated and not + // snapshottable except via the `snapshot-dynamic-buffers` feature. let implied_max_verts = sd.prim_count.saturating_mul(3); if sd.num_vertices > implied_max_verts { write_log_file(&format!( @@ -1071,11 +1071,10 @@ pub unsafe fn present_process() { if (*gs).is_snapping { let now = SystemTime::now(); let max_dur = std::time::Duration::from_millis(snap_ms as u64); - // Strictly wall clock, with nothing subtracted from it. Anything that shortens this in - // proportion to work done *during* the window can fail to terminate: buffer capture is - // only running because the window is open, so discounting its cost lets a frame that is - // ~entirely capture advance the clock by ~nothing, which keeps the window open, which - // keeps capture running. Use snap_ms to lengthen the window instead (DX11 generally + // Strictly wall clock, with nothing subtracted from it. So a long running snapshot + // operation (as for a large dynamic buffer captured with `snapshot-dynamic-buffers`) + // could use all available time with nothing left over for other pieces. Solution is + // to use snap_ms to lengthen the window instead (DX11 generally // wants more than the default; see the SNAP_CONFIG comment above). let elapsed = now .duration_since((*gs).snap_start) diff --git a/Native/shared_dx/src/types_dx11.rs b/Native/shared_dx/src/types_dx11.rs index 0ea92fc..eed8044 100644 --- a/Native/shared_dx/src/types_dx11.rs +++ b/Native/shared_dx/src/types_dx11.rs @@ -27,6 +27,8 @@ pub struct HookDirect3D11Context { // `snapshot-dynamic-buffers`. This struct is `Copy` and is returned by value from // `get_hook_context()` on every draw and IA state call, so keep the default build's copy // exactly the size it was before the feature existed. + // JMQ: not sure whether the size of this structure is really a perf factor or not, but its on the verge already of + // tipping over 128 bytes and I am concerned LLVM may drop some related optimization when it exceeds that. #[cfg(feature = "snapshot-dynamic-buffers")] pub real_map: MapFn, #[cfg(feature = "snapshot-dynamic-buffers")]