diff --git a/.github/workflows/job-cmakebuild-linux.yml b/.github/workflows/job-cmakebuild-linux.yml index 15413d6d..a473e8c8 100644 --- a/.github/workflows/job-cmakebuild-linux.yml +++ b/.github/workflows/job-cmakebuild-linux.yml @@ -35,7 +35,7 @@ jobs: run: sudo apt-get update - name: Install Dependencies - run: sudo apt-get install libasound2-dev libgl1-mesa-dev libpulse-dev libdbus-1-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxrandr-dev libxss-dev libxt-dev libxxf86vm-dev cmake apt-transport-https software-properties-common gcc g++ powershell clang-format-22 glslang-tools libwayland-dev libxkbcommon-dev + run: sudo apt-get install libasound2-dev libgl1-mesa-dev libpulse-dev libdbus-1-dev libglib2.0-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxrandr-dev libxss-dev libxt-dev libxxf86vm-dev cmake apt-transport-https software-properties-common gcc g++ powershell clang-format-22 glslang-tools libwayland-dev libxkbcommon-dev pkg-config - name: CMake Build run: .\Scripts\BuildEngine.ps1 -Configurations ${{inputs.configuration}} diff --git a/Obelisk/EntryPoint.cpp b/Obelisk/EntryPoint.cpp index ce957004..fb08fa8b 100644 --- a/Obelisk/EntryPoint.cpp +++ b/Obelisk/EntryPoint.cpp @@ -66,9 +66,6 @@ int applicationEntryPoint(int argc, char* argv[]) app->Run(); app->Shutdown(); - // Step 15 — join worker threads before logger/memory teardown - Helpers::ThreadPoolHelper::Shutdown(); - // Step 16 — flush and dispose logger Logger::Flush(); Logger::Dispose(); diff --git a/Resources/Editor/Settings/FallbackTexture.png b/Resources/Editor/Settings/FallbackTexture.png new file mode 100644 index 00000000..41799f67 Binary files /dev/null and b/Resources/Editor/Settings/FallbackTexture.png differ diff --git a/Resources/Shaders/zui_draw.vert b/Resources/Shaders/zui_draw.vert index cc8fd949..8b942ffb 100644 --- a/Resources/Shaders/zui_draw.vert +++ b/Resources/Shaders/zui_draw.vert @@ -6,10 +6,10 @@ layout(location = 2) in vec4 aColor; // RGBA8 UNORM — hardware unpacks packed layout(push_constant) uniform PC { - vec2 uScale; - vec2 uTranslate; - uint uTexIdx; - float uFbScale; // UIScale = fb/win; snaps vertices to nearest physical pixel + vec2 uScale; + vec2 uTranslate; + vec2 uFbScale; // Actual render-target pixels per logical UI unit, per axis. + uint uTexIdx; } pc; @@ -21,12 +21,12 @@ layout(location = 0) out struct void main() { - Out.Color = aColor; - Out.TexData = vec4(aUV, float(pc.uTexIdx), 0.0); + Out.Color = aColor; + Out.TexData = vec4(aUV, float(pc.uTexIdx), 0.0); // Snap to the nearest physical pixel before projecting. // Prevents sub-pixel drift that blurs glyph quads on non-Retina displays. - float fs = max(pc.uFbScale, 1.0); - vec2 snapped = round(aPos * fs) / fs; - gl_Position = vec4(snapped * pc.uScale + pc.uTranslate, 0.0, 1.0); + vec2 fs = max(pc.uFbScale, vec2(0.0001)); + vec2 snapped = round(aPos * fs) / fs; + gl_Position = vec4(snapped * pc.uScale + pc.uTranslate, 0.0, 1.0); } diff --git a/Tetragrama/Layers/ZUILayer.cpp b/Tetragrama/Layers/ZUILayer.cpp index 2ecc7d85..bf8d3b2b 100644 --- a/Tetragrama/Layers/ZUILayer.cpp +++ b/Tetragrama/Layers/ZUILayer.cpp @@ -298,10 +298,10 @@ namespace Tetragrama::Layers { return false; } - // GLFW cursor callback reports in logical screen coords (same space as - // glfwGetWindowSize / ScreenW). No division needed on any platform. - m_ctx->MousePos[0] = (float) e.GetPosX(); - m_ctx->MousePos[1] = (float) e.GetPosY(); + // GLFW cursor positions are in window coordinates. Convert them into + // the DPI-independent coordinate space used by ZUI layout and rendering. + m_ctx->MousePos[0] = (float) e.GetPosX() * m_ctx->InputScale[0]; + m_ctx->MousePos[1] = (float) e.GetPosY() * m_ctx->InputScale[1]; return false; } diff --git a/Tetragrama/Panels/AssetImporterPanel.cpp b/Tetragrama/Panels/AssetImporterPanel.cpp index bb21cf48..4f2d284f 100644 --- a/Tetragrama/Panels/AssetImporterPanel.cpp +++ b/Tetragrama/Panels/AssetImporterPanel.cpp @@ -1,10 +1,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -32,19 +35,25 @@ namespace Tetragrama::Panels // Initialize - void AssetImporterPanel::Initialize(Tetragrama::Layers::ZUILayer* layer) + void AssetImporterPanel::Initialize(Tetragrama::Layers::ZUILayer* layer, ProjectViewPanel* project_view) { - m_layer = layer; + m_layer = layer; + m_project_view = project_view; if (!layer) return; // Scratch arena for ImportConfiguration strings — carved from layer arena // ZKilo(64): ~8 path strings × ≤512 bytes each — a few KB is all we need layer->LocalArena.CreateSubArena(ZKilo(64), &m_local_arena, "EditorContext/AssetImporterPanelScratch"); - // Importer arenas carved from the engine's ImportPipeline budget so all // import memory — engine importers and editor importers — is budget-tracked. auto* import_arena = &ZEngine::Engine::GetContext()->ImportPipelineArena; + // Publication results can contain one entry per material and texture. Keep + // them outside the 64 KiB configuration scratch arena and reuse storage on + // the next import. The layer's shared local arena is only 4 MiB, so use the + // import budget for this bounded 8 MiB result slab. + m_import_result_slab.Init(import_arena, ZMega(8)); + m_import_task.PublishedPaths.init(&m_import_result_slab, 32); import_arena->CreateSubArena(ZMega(64), &m_gltf_importer_arena, "ImportPipeline/EditorGltfImporter"); import_arena->CreateSubArena(ZMega(128), &m_assimp_importer_arena, "ImportPipeline/EditorAssimpImporter"); @@ -82,9 +91,6 @@ namespace Tetragrama::Panels StartImport(); } - // Consume TriggerScan posted by background thread (actor creation on main thread) - TriggerScan(); - ZUIBox* bg = ZUIBeginColumn(ctx, "##imp_bg", ZFill(), ZFill()); bg->Flags = bg->Flags | ZUI_DrawBackground | ZUI_Scrollable; ZUIBoxSetColorArr(bg, kBg); @@ -595,6 +601,9 @@ namespace Tetragrama::Panels void AssetImporterPanel::StartImport() { + if (m_state.value.load(std::memory_order_acquire) == ImporterState::Importing) + return; + if (!m_gltf_importer || !m_fbx_importer || !m_assimp_importer) return; @@ -643,9 +652,17 @@ namespace Tetragrama::Panels m_state.value.store(ImporterState::Importing, std::memory_order_release); m_progress.value.store(0.f, std::memory_order_relaxed); - m_import_task = {}; - m_import_task.Panel = this; - m_import_task.Configuration = *config; + m_import_task.Outputs.clear(); + m_import_task.PublishedPaths.clear(); + m_import_task.Error.clear(); + m_import_task.Outcome = ImportTaskOutcome::Pending; + m_import_task.Publication = {}; + m_import_task.Panel = this; + m_import_task.Configuration = *config; + m_import_task.Configuration.ArtifactContext = &m_import_task; + m_import_task.Configuration.OnArtifactPublished = &AssetImporterPanel::OnImportArtifactPublished; + if (m_import_task.Configuration.VFS) + m_import_task.Publication = m_import_task.Configuration.VFS->BeginImportPublication(); secure_strncpy(m_import_task.SourcePath, sizeof(m_import_task.SourcePath), m_path_buf, secure_strlen(m_path_buf)); const auto ext = vfs_value.Extension(); @@ -665,6 +682,7 @@ namespace Tetragrama::Panels if (!ZEngine::Helpers::ThreadPoolHelper::Submit(&m_import_task, &AssetImporterPanel::RunImportTask)) { + EndImportPublication(m_import_task); m_state.value.store(ImporterState::Options, std::memory_order_release); PushLog("Import task rejected because the thread pool is shutting down", kRed[0], kRed[1], kRed[2]); } @@ -672,38 +690,74 @@ namespace Tetragrama::Panels void AssetImporterPanel::RunImportTask(void* context) { - auto* task = static_cast(context); + auto* task = static_cast(context); + if (!task) + return; + auto* panel = task->Panel; if (!panel) + { + EndImportPublication(*task); return; + } const auto& config = task->Configuration; switch (task->Kind) { case ImporterKind::Gltf: - panel->m_gltf_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, panel, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); + panel->m_gltf_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, task, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); break; case ImporterKind::Fbx: - panel->m_fbx_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, panel, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); + panel->m_fbx_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, task, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); break; case ImporterKind::Assimp: - panel->m_assimp_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, panel, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); + panel->m_assimp_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, task, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); break; } - } - // TriggerScan (main-thread only) + if (task->Outcome == ImportTaskOutcome::Pending) + { + task->Error = "Importer returned without reporting a result"; + task->Outcome = ImportTaskOutcome::Failed; + } + + // ImportFile's local scratch storage is gone once it returns. The worker + // callbacks have already copied the required output paths into ImportTask. + ZEngine::Core::MainThreadScheduler::Post(task, &AssetImporterPanel::FinalizeImportTask); + } - void AssetImporterPanel::TriggerScan() + void AssetImporterPanel::FinalizeImportTask(void* context) { - if (!m_pending_actor.valid) + auto* task = static_cast(context); + if (!task) return; - auto* app = m_layer ? reinterpret_cast(m_layer->CurrentApp) : nullptr; - auto* scene = app ? reinterpret_cast(app->CurrentScene) : nullptr; - auto* ctx = ZEngine::Engine::GetContext(); + auto* panel = task->Panel; + if (panel) + { + if (task->Outcome == ImportTaskOutcome::Completed) + { + ZEngine::Core::Containers::ArrayView outputs(task->Outputs.data(), task->Outputs.size()); + CompleteImportOnMainThread(panel, outputs); + } + else + { + CompleteImportErrorOnMainThread(panel, task->Error.empty() ? "Importer returned without an error message" : task->Error); + } + } + + // CompleteImportOnMainThread can return early on metadata errors. Release the + // publication afterwards in every case so watcher processing cannot remain + // deferred after an import failure. + EndImportPublication(*task); + task->PublishedPaths.clear(); + } + + void AssetImporterPanel::CreateMeshActor(uuids::uuid mesh_uuid, uint32_t render_instance_id, const char* name) + { + auto* ctx = ZEngine::Engine::GetContext(); - if (scene && ctx && ctx->ActorManager) + if (ctx && ctx->ActorManager) { using namespace ZEngine::ECS::Components; ZEngine::ECS::ActorHandle handle = ctx->ActorManager->Create(); @@ -711,16 +765,15 @@ namespace Tetragrama::Panels if (actor) { NameComponent nc = {}; - secure_strncpy(nc.Value, sizeof(nc.Value), m_pending_actor.name, secure_strlen(m_pending_actor.name)); + secure_strncpy(nc.Value, sizeof(nc.Value), name, secure_strlen(name)); actor->AddComponent(nc); actor->AddComponent({}); MeshComponent mc = {}; - mc.MeshUUID = m_pending_actor.uuid; - mc.RenderInstanceId = m_pending_actor.render_id; + mc.MeshUUID = mesh_uuid; + mc.RenderInstanceId = render_instance_id; actor->AddComponent(mc); } } - m_pending_actor = {}; } // PushLog / PushHistory @@ -737,7 +790,6 @@ namespace Tetragrama::Panels m_log_head = (m_log_head + 1) % kLogMax; if (m_log_count < kLogMax) ++m_log_count; - m_scroll_log = true; } void AssetImporterPanel::PushHistory(const char* name, bool ok, const char* msg) @@ -765,7 +817,66 @@ namespace Tetragrama::Panels void AssetImporterPanel::OnImportFileComplete(void* ctx, ZEngine::Core::Containers::ArrayView outputs) { - auto* self = reinterpret_cast(ctx); + auto* task = static_cast(ctx); + if (!task) + return; + // An importer may have published some artifacts before reporting an error. + // Once it reports that error, completion must not turn the task into a + // success and hide it from the editor. + if (task->Outcome == ImportTaskOutcome::Failed) + return; + + task->Outputs.clear(); + if (outputs.size() > 0) + { + task->Outputs.assign(outputs.data(), outputs.data() + outputs.size()); + for (const auto& output : task->Outputs) + RecordPublishedPath(*task, output.Path.c_str()); + } + task->Error.clear(); + task->Outcome = ImportTaskOutcome::Completed; + } + + void AssetImporterPanel::OnImportArtifactPublished(void* ctx, ZEngine::Importers::AssetFileType, const char* path) + { + auto* task = static_cast(ctx); + if (task) + RecordPublishedPath(*task, path); + } + + void AssetImporterPanel::RecordPublishedPath(ImportTask& task, const char* path) + { + if (!path || path[0] == '\0') + return; + auto parsed = VFSPath::Parse(path); + if (parsed.Failed()) + return; + + for (const ZEngine::Core::VFS::VFSPath& existing : task.PublishedPaths) + if (existing == parsed.Value()) + return; + + task.PublishedPaths.push(parsed.Value()); + } + + void AssetImporterPanel::EndImportPublication(ImportTask& task) + { + auto* vfs = task.Configuration.VFS; + if (!vfs || !task.Publication.IsValid()) + return; + + for (const ZEngine::Core::VFS::VFSPath& published : task.PublishedPaths) + vfs->RecordImportPublicationArtifact(task.Publication, published); + vfs->EndImportPublication(task.Publication); + task.Publication = {}; + } + + void AssetImporterPanel::CompleteImportOnMainThread(void* ctx, ZEngine::Core::Containers::ArrayView outputs) + { + auto* self = reinterpret_cast(ctx); + + if (self->m_project_view) + self->m_project_view->RequestRefresh(); bool has_mesh = false; cstring mesh_path = nullptr; @@ -799,21 +910,30 @@ namespace Tetragrama::Panels if (!rel.Succeeded()) continue; - char native_mat_path[MAX_FILE_PATH_COUNT] = {}; - rel.Value().ResolveNative(ws, native_mat_path, sizeof(native_mat_path)); - - ZEngine::Importers::AssetMaterial material{}; - ZEngine::Importers::AssetCodec::DeserializeMaterialAssetFile(&self->m_local_arena, native_mat_path, material); - if (material.MaterialUUID.is_nil()) - continue; + auto material_uuid = ReadEmbeddedAssetUUID(*vfs, rel.Value()); + if (material_uuid.Failed()) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to read material identity for '{}' (VFS error {})", mat_path, static_cast(material_uuid.Error()))); + return; + } - auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); - ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; - meta.AssetUUID = material.MaterialUUID; + auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); + if (meta_result.Failed() && meta_result.Error() != ZEngine::Core::VFS::VFSError::NotFound && meta_result.Error() != ZEngine::Core::VFS::VFSError::Corrupted) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to read metadata for '{}' (VFS error {})", mat_path, static_cast(meta_result.Error()))); + return; + } + ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; + meta.AssetUUID = material_uuid.Value(); secure_strncpy(meta.SourcePath, sizeof(meta.SourcePath), self->m_path_buf, sizeof(meta.SourcePath) - 1); secure_strncpy(meta.ArtifactPath, sizeof(meta.ArtifactPath), mat_path, sizeof(meta.ArtifactPath) - 1); secure_strncpy(meta.ImporterName, sizeof(meta.ImporterName), "GltfImporter/AssimpImporter", sizeof(meta.ImporterName) - 1); - ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + auto write = ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + if (write.Failed()) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to persist metadata for '{}' (VFS error {})", mat_path, static_cast(write.Error()))); + return; + } } } } @@ -850,8 +970,13 @@ namespace Tetragrama::Panels auto rel = VFSPath::Parse(mesh_path); if (rel.Succeeded()) { - auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); - ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; + auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); + if (meta_result.Failed() && meta_result.Error() != ZEngine::Core::VFS::VFSError::NotFound && meta_result.Error() != ZEngine::Core::VFS::VFSError::Corrupted) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to read metadata for '{}' (VFS error {})", mesh_path, static_cast(meta_result.Error()))); + return; + } + ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; // Sync to the file's own embedded UUID — otherwise a nil // AssetUUID gets locked in forever (#755). if (has_header) @@ -859,7 +984,12 @@ namespace Tetragrama::Panels secure_strncpy(meta.SourcePath, sizeof(meta.SourcePath), self->m_path_buf, sizeof(meta.SourcePath) - 1); secure_strncpy(meta.ArtifactPath, sizeof(meta.ArtifactPath), mesh_path, sizeof(meta.ArtifactPath) - 1); secure_strncpy(meta.ImporterName, sizeof(meta.ImporterName), "GltfImporter/AssimpImporter", sizeof(meta.ImporterName) - 1); - ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + auto write = ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + if (write.Failed()) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to persist metadata for '{}' (VFS error {})", mesh_path, static_cast(write.Error()))); + return; + } } } @@ -882,11 +1012,7 @@ namespace Tetragrama::Panels snprintf(iname, sizeof(iname), "%.*s", (int) s.Length, s.Data); } } - uint32_t render_id = scene->AddMeshInstance(header.Id, iname); - self->m_pending_actor.uuid = header.Id; - self->m_pending_actor.render_id = render_id; - self->m_pending_actor.valid = true; - secure_strncpy(self->m_pending_actor.name, sizeof(self->m_pending_actor.name), iname, sizeof(self->m_pending_actor.name) - 1); + self->CreateMeshActor(header.Id, scene->AddMeshInstance(header.Id, iname), iname); } } self->m_add_to_scene = false; @@ -923,13 +1049,16 @@ namespace Tetragrama::Panels self->m_progress.value.store(1.f, std::memory_order_relaxed); self->m_state.value.store(ImporterState::Idle, std::memory_order_release); - ZEngine::Core::MainThreadScheduler::Post(self, [](void* c) { reinterpret_cast(c)->TriggerScan(); }); } void AssetImporterPanel::OnImportProgress(void* ctx, float pct) { - auto* self = reinterpret_cast(ctx); - char msg[128]; + auto* task = static_cast(ctx); + auto* self = task ? task->Panel : nullptr; + if (!self) + return; + + char msg[128]; snprintf(msg, sizeof(msg), "Processing... %.0f%%", pct * 100.f); self->PushLog(msg, kWhite[0], kWhite[1], kWhite[2]); self->m_progress.value.store(pct, std::memory_order_relaxed); @@ -937,8 +1066,21 @@ namespace Tetragrama::Panels void AssetImporterPanel::OnImportError(void* ctx, std::string_view err) { - auto* self = reinterpret_cast(ctx); - char msg[512]; + auto* task = static_cast(ctx); + if (!task) + return; + + task->Outputs.clear(); + task->Error.assign(err.data(), err.size()); + task->Outcome = ImportTaskOutcome::Failed; + } + + void AssetImporterPanel::CompleteImportErrorOnMainThread(void* ctx, std::string_view err) + { + auto* self = reinterpret_cast(ctx); + self->m_add_to_scene = false; + self->m_instance_name[0] = '\0'; + char msg[512]; snprintf(msg, sizeof(msg), "Error: %.*s", (int) err.size(), err.data()); self->PushLog(msg, kRed[0], kRed[1], kRed[2]); char fn[256] = {}; @@ -952,13 +1094,16 @@ namespace Tetragrama::Panels } self->PushHistory(fn, false, msg); self->m_state.value.store(ImporterState::Idle, std::memory_order_release); - ZEngine::Core::MainThreadScheduler::Post(self, [](void* c) { reinterpret_cast(c)->TriggerScan(); }); } void AssetImporterPanel::OnImportLog(void* ctx, std::string_view msg) { - auto* self = reinterpret_cast(ctx); - char buf[256]; + auto* task = static_cast(ctx); + auto* self = task ? task->Panel : nullptr; + if (!self) + return; + + char buf[256]; snprintf(buf, sizeof(buf), "%.*s", (int) msg.size(), msg.data()); self->PushLog(buf, kWhite[0], kWhite[1], kWhite[2]); } diff --git a/Tetragrama/Panels/AssetImporterPanel.h b/Tetragrama/Panels/AssetImporterPanel.h index e94c951c..cbcf4563 100644 --- a/Tetragrama/Panels/AssetImporterPanel.h +++ b/Tetragrama/Panels/AssetImporterPanel.h @@ -1,7 +1,11 @@ #pragma once #include #include +#include +#include #include +#include +#include #include #include #include @@ -9,13 +13,16 @@ #include #include #include +#include +#include namespace Tetragrama::Panels { + struct ProjectViewPanel; + /// @brief Three-state asset importer panel (Idle → Options → Importing). /// Supports glTF/GLB, FBX, and Assimp-backed formats. Import runs - /// on a background thread; ECS actor creation is posted back to the - /// main thread via TriggerScan(). + /// on a background thread; completion is finalized on the main thread. struct AssetImporterPanel : ZEngine::UI::ZUIPanelView { AssetImporterPanel() @@ -25,7 +32,8 @@ namespace Tetragrama::Panels /// @brief Allocates importers and arenas; must be called once before first use. /// @param layer Owning ZUI layer (provides arena and app pointers). - void Initialize(Tetragrama::Layers::ZUILayer* layer); + /// @param project_view Content browser refreshed after a successful import. + void Initialize(Tetragrama::Layers::ZUILayer* layer, ProjectViewPanel* project_view); /// @brief Dispatches to BuildIdle/BuildOptions/BuildImporting based on current state. /// @param ctx ZUI context for the current frame. @@ -34,9 +42,14 @@ namespace Tetragrama::Panels private: Tetragrama::Layers::ZUILayer* m_layer = nullptr; + ProjectViewPanel* m_project_view = nullptr; // Scratch arena for ImportConfiguration strings (cleared before each import) ZEngine::Core::Memory::ArenaAllocator m_local_arena = {}; + // Task results must outlive the worker callback, but must not consume the + // 64 KiB configuration scratch arena. TLSF reuses and reclaims Array growth + // between imports without a per-import arena leak. + ZEngine::Core::Memory::TLSFSlab m_import_result_slab = {}; // Importer-dedicated arenas carved from the engine ImportPipeline budget ZEngine::Core::Memory::ArenaAllocator m_gltf_importer_arena = {}; ZEngine::Core::Memory::ArenaAllocator m_assimp_importer_arena = {}; @@ -53,12 +66,26 @@ namespace Tetragrama::Panels Assimp, }; + enum class ImportTaskOutcome : uint8_t + { + Pending, + Completed, + Failed, + }; + struct ImportTask { - AssetImporterPanel* Panel = nullptr; - ZEngine::Importers::AssetCodec::ImportConfiguration Configuration = {}; - ImporterKind Kind = ImporterKind::Gltf; - char SourcePath[MAX_FILE_PATH_COUNT] = {}; + AssetImporterPanel* Panel = nullptr; + ZEngine::Importers::AssetCodec::ImportConfiguration Configuration = {}; + ImporterKind Kind = ImporterKind::Gltf; + char SourcePath[MAX_FILE_PATH_COUNT] = {}; + // The importer releases its scratch arena after it calls its callback. + // Retain the small output list until the main thread consumes it. + std::vector Outputs = {}; + ZEngine::Core::Containers::Array PublishedPaths = {}; + ZEngine::Core::VFS::VFSImportPublication Publication = {}; + std::string Error = {}; + ImportTaskOutcome Outcome = ImportTaskOutcome::Pending; }; ImportTask m_import_task = {}; @@ -87,17 +114,8 @@ namespace Tetragrama::Panels bool m_same_settings = false; int m_options_filter = 5; // 0=General 1=Mesh 2=Material 3=Anim 4=LOD 5=All - // Pending ECS actor creation — written on bg thread, consumed on main thread - struct PendingActor - { - uuids::uuid uuid = {}; - uint32_t render_id = UINT32_MAX; - char name[128] = {}; - bool valid = false; - } m_pending_actor = {}; - // Import history ring buffer (Idle state, newest first) - static constexpr int kHistMax = 16; + static constexpr int kHistMax = 16; struct HistEntry { char name[256] = {}; @@ -117,7 +135,6 @@ namespace Tetragrama::Panels LogEntry m_log[kLogMax] = {}; int m_log_head = 0; int m_log_count = 0; - bool m_scroll_log = false; std::mutex m_log_mutex; // Build helpers @@ -134,25 +151,31 @@ namespace Tetragrama::Panels std::future BrowseFileAsync(); /// @brief Begin the import process with the current settings. void StartImport(); - /// @brief Trigger a directory scan for importable assets. - void TriggerScan(); // main-thread only + /// @brief Add an imported mesh instance to the ECS scene on the main thread. + void CreateMeshActor(uuids::uuid mesh_uuid, uint32_t render_instance_id, const char* name); /// @brief Append a log message to the import log. void PushLog(const char* text, float r, float g, float b); /// @brief Record the completed import in the history list. void PushHistory(const char* name, bool ok, const char* msg); - /// @brief Runs one import on a worker through the allocation-free task API. + /// @brief Runs one import on a worker through the C-style task API. static void RunImportTask(void* context); + /// @brief Consumes a completed worker task on the main thread. + static void FinalizeImportTask(void* context); - // Static callbacks for ImportFile (called from background thread) - /// @brief Callback invoked when the import file step finishes. + // Static callbacks for ImportFile. They execute on a worker, so they only + // copy the result into ImportTask; FinalizeImportTask performs editor work. static void OnImportFileComplete(void* ctx, ZEngine::Core::Containers::ArrayView outputs); - /// @brief Callback invoked with progress updates during import. + static void OnImportArtifactPublished(void* ctx, ZEngine::Importers::AssetFileType type, const char* path); static void OnImportProgress(void* ctx, float pct); - /// @brief Callback invoked when an import error occurs. static void OnImportError(void* ctx, std::string_view err); - /// @brief Callback invoked for each import log message. static void OnImportLog(void* ctx, std::string_view msg); + + // Main-thread-only completion handlers. + static void CompleteImportOnMainThread(void* ctx, ZEngine::Core::Containers::ArrayView outputs); + static void CompleteImportErrorOnMainThread(void* ctx, std::string_view err); + static void RecordPublishedPath(ImportTask& task, const char* path); + static void EndImportPublication(ImportTask& task); }; } // namespace Tetragrama::Panels diff --git a/Tetragrama/Panels/ProjectViewPanel.cpp b/Tetragrama/Panels/ProjectViewPanel.cpp index 38571420..75b1d5b5 100644 --- a/Tetragrama/Panels/ProjectViewPanel.cpp +++ b/Tetragrama/Panels/ProjectViewPanel.cpp @@ -211,6 +211,12 @@ namespace Tetragrama::Panels Title = "Project"; } + void ProjectViewPanel::RequestRefresh() + { + m_needs_refresh = true; + m_tree_cache_count = 0; + } + // VFS listing void ProjectViewPanel::RefreshListing(IVFSContext* vfs) { diff --git a/Tetragrama/Panels/ProjectViewPanel.h b/Tetragrama/Panels/ProjectViewPanel.h index 087a9dd7..1bad8377 100644 --- a/Tetragrama/Panels/ProjectViewPanel.h +++ b/Tetragrama/Panels/ProjectViewPanel.h @@ -27,6 +27,9 @@ namespace Tetragrama::Panels /// @param rect Panel bounding rect [x0, y0, x1, y1]. void BuildContent(ZEngine::UI::ZUIContext* ctx, float rect[4]) override; + /// @brief Invalidates cached listings after a main-thread file operation. + void RequestRefresh(); + private: // Navigation ZEngine::Core::VFS::VFSPath m_current_dir = {}; diff --git a/Tetragrama/Panels/ViewportPanel.cpp b/Tetragrama/Panels/ViewportPanel.cpp index b6f26e59..cf9cc0ef 100644 --- a/Tetragrama/Panels/ViewportPanel.cpp +++ b/Tetragrama/Panels/ViewportPanel.cpp @@ -163,13 +163,11 @@ namespace Tetragrama::Panels ZUISignal img_sig = ZUISignalFromBox(ctx, img); ZUIPopBox(ctx); - // Feed the viewport rect to the camera controller every frame so it can - // self-gate on cursor position without depending on the ZUI hit-test chain. + // The camera consumes raw GLFW cursor positions, not UI-space positions. + // Convert both the hover bounds and the extent used by camera rays/panning. if (app->CameraController) { - app->CameraController->SetViewportRect(rect[0], rect[1], rect[2], rect[3]); - if (sw > 0.0f && sh > 0.0f) - app->CameraController->SetViewport(sw, sh); + app->CameraController->SetViewportFromUI(rect, ctx->InputScale); } // Keep ViewportHovered for ZUI-level concerns (drag-drop, scroll routing). diff --git a/Tetragrama/Panels/ZUIPanelManagerComponent.cpp b/Tetragrama/Panels/ZUIPanelManagerComponent.cpp index 03e8f509..10fd2c7e 100644 --- a/Tetragrama/Panels/ZUIPanelManagerComponent.cpp +++ b/Tetragrama/Panels/ZUIPanelManagerComponent.cpp @@ -64,12 +64,14 @@ namespace Tetragrama::Panels auto* p_proj = Manager.AddPanel(ZUIDockHashName("Project")); Manager.AddView(p_proj, &project); - importer.Initialize(parent); // allocate importers from ImportPipeline budget + importer.Initialize(parent, &project); // allocate importers from ImportPipeline budget - // EngineAssetsBackend is rooted at /ZodiacEngine (mounted at /ZodiacEngine). + // EngineAssetsBackend is rooted at the packaged ZodiacEngine directory + // beside the executable (mounted at /ZodiacEngine). // Settings/ is created by the engine build. Verify it exists via the backend, // then use VFSPath::ResolveNative to produce the correct native path — no - // manual string concatenation. Falls back to the working directory if absent. + // manual string concatenation. A missing settings directory uses the local + // layout filename so the editor can still start with its default layout. char layout_path[512] = {}; bool settings_ok = false; auto* eng = ZEngine::Engine::GetContext(); diff --git a/Tetragrama/Serializers/EditorSceneSerializer.cpp b/Tetragrama/Serializers/EditorSceneSerializer.cpp index ca6b5116..76b3e174 100644 --- a/Tetragrama/Serializers/EditorSceneSerializer.cpp +++ b/Tetragrama/Serializers/EditorSceneSerializer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -106,12 +107,16 @@ namespace Tetragrama::Serializers cook_cfg.VFS = reinterpret_cast(ZEngine::Engine::GetContext()->VFS); auto output = ZEngine::Importers::AssetCodec::SerializeMeshAssetFile(scratch.Arena, *mesh, *hierarchy, cook_cfg); - if (!output.Path.empty()) + if (output.Succeeded()) { // Register the artifact path so future saves don't re-cook - ZEngine::Helpers::secure_strncpy(const_cast(rec)->Meta.ArtifactPath, MAX_FILE_PATH_COUNT, output.Path.c_str(), output.Path.size()); + ZEngine::Helpers::secure_strncpy(const_cast(rec)->Meta.ArtifactPath, MAX_FILE_PATH_COUNT, output.Value().Path.c_str(), output.Value().Path.size()); - scene->PushAssetFile(output); + scene->PushAssetFile(output.Value()); + } + else + { + ZENGINE_CORE_ERROR("Failed to cook '{}/{}' (VFS error {})", cook_cfg.OutputAssetsPath.c_str(), output_file, static_cast(output.Error())) } } ZReleaseScratch(scratch); diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp index 866ad1e3..87f17ca6 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include using namespace ZEngine::Core::Containers; @@ -132,10 +133,13 @@ namespace ZEngine::Applications auto swapchain = Device->SwapchainPtr; swapchain->AcquireNextImage(CurrentFrameContextIndex); + if (!swapchain->IsFrameValid()) + return false; if (Device->RRM) static_cast(Device->RRM)->BeginFrame(swapchain->CurrentFrame->Index); swapchain->FrameAsyncOperations.clear(); + swapchain->TraceSubmission = false; swapchain->CollectAsyncGPUOperations(); Managers::AssetManager::FlushTextureReleases(); @@ -159,6 +163,11 @@ namespace ZEngine::Applications void AppRenderPipeline::EndFrame() { + if (!Device->SwapchainPtr->IsFrameValid()) + { + Device->SwapchainPtr->Present(); + return; + } if (Device->RRM) static_cast(Device->RRM)->EndFrame(); @@ -318,9 +327,11 @@ namespace ZEngine::Applications { if (ZUICtx) { - // Use the logical window size (glfwGetWindowSize) not the physical swapchain - // size. Mouse positions from GLFW cursor callbacks are also in logical pixels, - // so panel positions and hit-testing must use the same coordinate space. + const uint32_t previous_screen_w = ZUICtx->ScreenW; + const uint32_t previous_screen_h = ZUICtx->ScreenH; + const float previous_scale = ZUICtx->UIScale; + // ZUI uses DPI-independent coordinates. The render pass projects them into + // the physical swapchain, while input is converted to the same UI space. if (Device->CurrentWindow) { auto* native = static_cast(Device->CurrentWindow->GetNativeWindow()); @@ -334,46 +345,48 @@ namespace ZEngine::Applications content_scale = 1.f; } - // Exact ImGui approach (imgui_impl_glfw.cpp GetWindowSizeAndFramebufferScale): - // ScreenW/H = glfwGetWindowSize → logical screen coords (1512 on Retina) - // UIScale = glfwGetFramebufferSize / glfwGetWindowSize → physical/logical ratio (2.0 on Retina) - // Cursor from GLFW callback is already in logical space — no division needed. - // This is identical on all platforms; macOS just happens to have UIScale=2 on Retina. + // GLFW's content scale is the UI-metric scale. On Wayland and macOS it + // normally matches framebuffer/window, while X11 can report a scaled + // desktop with a 1:1 framebuffer. Build one DPI-independent ZUI space + // from the larger factor so text and controls remain readable on all + // three platforms. + int win_w = (int) Device->CurrentWindow->GetWidth(); + int win_h = (int) Device->CurrentWindow->GetHeight(); + int fb_w = win_w; + int fb_h = win_h; if (native) { - int win_w = 0, win_h = 0, fb_w = 0, fb_h = 0; glfwGetWindowSize(native, &win_w, &win_h); glfwGetFramebufferSize(native, &fb_w, &fb_h); - if (win_w > 0 && win_h > 0) - { - ZUICtx->ScreenW = (uint32_t) win_w; - ZUICtx->ScreenH = (uint32_t) win_h; - float s = (fb_w > 0) ? (float) fb_w / (float) win_w : 1.f; - ZUICtx->UIScale = (s > 0.5f) ? s : 1.f; - } - else - { - ZUICtx->ScreenW = Device->CurrentWindow->GetWidth(); - ZUICtx->ScreenH = Device->CurrentWindow->GetHeight(); - ZUICtx->UIScale = content_scale; - } - } - else - { - ZUICtx->ScreenW = Device->CurrentWindow->GetWidth(); - ZUICtx->ScreenH = Device->CurrentWindow->GetHeight(); - ZUICtx->UIScale = content_scale; } - if (!ZUICtx->UIScaleLogged) + + win_w = win_w > 0 ? win_w : 1; + win_h = win_h > 0 ? win_h : 1; + fb_w = fb_w > 0 ? fb_w : win_w; + fb_h = fb_h > 0 ? fb_h : win_h; + + const float framebuffer_scale_x = (float) fb_w / (float) win_w; + const float framebuffer_scale_y = (float) fb_h / (float) win_h; + const float ui_scale = fmaxf(0.5f, fmaxf(content_scale, fmaxf(framebuffer_scale_x, framebuffer_scale_y))); + + ZUICtx->ScreenW = (uint32_t) fmaxf(1.f, roundf((float) fb_w / ui_scale)); + ZUICtx->ScreenH = (uint32_t) fmaxf(1.f, roundf((float) fb_h / ui_scale)); + ZUICtx->UIScale = (float) fb_w / (float) ZUICtx->ScreenW; + ZUICtx->InputScale[0] = (float) ZUICtx->ScreenW / (float) win_w; + ZUICtx->InputScale[1] = (float) ZUICtx->ScreenH / (float) win_h; + if (!ZUICtx->UIScaleLogged || previous_screen_w != ZUICtx->ScreenW || previous_screen_h != ZUICtx->ScreenH || previous_scale != ZUICtx->UIScale) { - ZENGINE_CORE_INFO("[ZUI] UIScale={:.2f} Screen={}x{} (logical) ContentScale={:.2f}", ZUICtx->UIScale, ZUICtx->ScreenW, ZUICtx->ScreenH, content_scale); + ZENGINE_CORE_INFO("[ZUI] UIScale={:.2f} Screen={}x{} Window={}x{} Framebuffer={}x{} ContentScale={:.2f} InputScale={:.2f}x{:.2f}", ZUICtx->UIScale, ZUICtx->ScreenW, ZUICtx->ScreenH, win_w, win_h, fb_w, fb_h, content_scale, ZUICtx->InputScale[0], ZUICtx->InputScale[1]); ZUICtx->UIScaleLogged = true; } } else { - ZUICtx->ScreenW = Device->SwapchainPtr->SwapchainImageWidth; - ZUICtx->ScreenH = Device->SwapchainPtr->SwapchainImageHeight; + ZUICtx->ScreenW = Device->SwapchainPtr->SwapchainImageWidth; + ZUICtx->ScreenH = Device->SwapchainPtr->SwapchainImageHeight; + ZUICtx->UIScale = 1.f; + ZUICtx->InputScale[0] = 1.f; + ZUICtx->InputScale[1] = 1.f; } ZEngine::UI::ZUIBeginFrame(ZUICtx, dt); } diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.h b/ZEngine/ZEngine/Applications/AppRenderPipeline.h index bd7acdf1..4f140d7e 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.h +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.h @@ -32,6 +32,9 @@ namespace ZEngine::Applications uint32_t RenderTargetW = 0; uint32_t RenderTargetH = 0; uint64_t ResizeSequence = 0; + // Physical window size, captured on the main thread with platform events. + uint32_t FramebufferW = 0; + uint32_t FramebufferH = 0; bool RenderOverlay = false; }; diff --git a/ZEngine/ZEngine/CMakeLists.txt b/ZEngine/ZEngine/CMakeLists.txt index 230fb4cb..e0ac45f9 100644 --- a/ZEngine/ZEngine/CMakeLists.txt +++ b/ZEngine/ZEngine/CMakeLists.txt @@ -185,7 +185,32 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") target_compile_definitions(zEngineLib PUBLIC NOMINMAX GLFW_EXPOSE_NATIVE_WIN32) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") target_link_libraries(zEngineLib PUBLIC stdc++fs dl) - target_compile_definitions(zEngineLib PUBLIC GLFW_EXPOSE_NATIVE_WAYLAND) + # GLFW 3.5 includes both backends and selects the active session at runtime. + # The file-dialog layer needs both native window types for correct parenting. + target_compile_definitions(zEngineLib PUBLIC GLFW_EXPOSE_NATIVE_X11 GLFW_EXPOSE_NATIVE_WAYLAND) + + find_package(PkgConfig REQUIRED) + pkg_check_modules(GIO REQUIRED IMPORTED_TARGET gio-2.0) + pkg_check_modules(WAYLAND_CLIENT REQUIRED IMPORTED_TARGET wayland-client) + target_link_libraries(zEngineLib PRIVATE PkgConfig::GIO PkgConfig::WAYLAND_CLIENT) + + set(ZENGINE_XDG_FOREIGN_PROTOCOL ${CMAKE_CURRENT_SOURCE_DIR}/Windows/Platform/xdg-foreign-unstable-v2.xml) + set(ZENGINE_XDG_FOREIGN_HEADER ${CMAKE_CURRENT_BINARY_DIR}/xdg-foreign-unstable-v2-client-protocol.h) + set(ZENGINE_XDG_FOREIGN_CODE ${CMAKE_CURRENT_BINARY_DIR}/xdg-foreign-unstable-v2-client-protocol.c) + + find_program(WAYLAND_SCANNER_EXECUTABLE NAMES wayland-scanner REQUIRED) + add_custom_command( + OUTPUT ${ZENGINE_XDG_FOREIGN_HEADER} ${ZENGINE_XDG_FOREIGN_CODE} + COMMAND ${WAYLAND_SCANNER_EXECUTABLE} client-header ${ZENGINE_XDG_FOREIGN_PROTOCOL} ${ZENGINE_XDG_FOREIGN_HEADER} + COMMAND ${WAYLAND_SCANNER_EXECUTABLE} private-code ${ZENGINE_XDG_FOREIGN_PROTOCOL} ${ZENGINE_XDG_FOREIGN_CODE} + DEPENDS ${ZENGINE_XDG_FOREIGN_PROTOCOL} + VERBATIM + ) + target_sources(zEngineLib PRIVATE + ${ZENGINE_XDG_FOREIGN_HEADER} + ${ZENGINE_XDG_FOREIGN_CODE} + ) + target_include_directories(zEngineLib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_libraries(zEngineLib PRIVATE "-lobjc" "-framework AppKit" "-framework CoreGraphics" "-framework CoreServices") target_compile_definitions(zEngineLib PUBLIC GLFW_EXPOSE_NATIVE_COCOA) diff --git a/ZEngine/ZEngine/Controllers/ICameraController.h b/ZEngine/ZEngine/Controllers/ICameraController.h index c9ae71ab..3548ffb0 100644 --- a/ZEngine/ZEngine/Controllers/ICameraController.h +++ b/ZEngine/ZEngine/Controllers/ICameraController.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace ZEngine::Controllers { @@ -22,19 +23,33 @@ namespace ZEngine::Controllers virtual void SetViewportOrigin(float x, float y) = 0; /// @brief Notify the controller of the viewport's screen rect each frame. /// Used for self-contained hover detection — no ZUI hit-test dependency. - /// @param x0 Left edge in logical pixels. - /// @param y0 Top edge in logical pixels. - /// @param x1 Right edge in logical pixels. - /// @param y1 Bottom edge in logical pixels. + /// @param x0 Left edge in GLFW window coordinates. + /// @param y0 Top edge in GLFW window coordinates. + /// @param x1 Right edge in GLFW window coordinates. + /// @param y1 Bottom edge in GLFW window coordinates. virtual void SetViewportRect(float x0, float y0, float x1, float y1) = 0; + /// @brief Converts UI bounds and extent to the same window coordinates as cursor input. + /// @param input_scale UI coordinates per GLFW window coordinate, independently per axis. + void SetViewportFromUI(const float rect[4], const float input_scale[2]) + { + const float sx = std::isfinite(input_scale[0]) && input_scale[0] > 0.f ? input_scale[0] : 1.f; + const float sy = std::isfinite(input_scale[1]) && input_scale[1] > 0.f ? input_scale[1] : 1.f; + const float x0 = rect[0] / sx; + const float y0 = rect[1] / sy; + const float x1 = rect[2] / sx; + const float y1 = rect[3] / sy; + SetViewportRect(x0, y0, x1, y1); + if (x1 > x0 && y1 > y0) + SetViewport(x1 - x0, y1 - y0); + } /// @brief Inform the controller that the UI owns one or both input channels. /// @param pointer_captured True while a non-viewport UI control owns pointer input. /// @param keyboard_captured True while a UI control, popup, or modal owns keyboard input. - virtual void SetInputCapture(bool pointer_captured, bool keyboard_captured) {} - virtual void ResumeEventProcessing() = 0; - virtual void PauseEventProcessing() = 0; + virtual void SetInputCapture(bool pointer_captured, bool keyboard_captured) {} + virtual void ResumeEventProcessing() = 0; + virtual void PauseEventProcessing() = 0; - CameraControllerType GetControllerType() const + CameraControllerType GetControllerType() const { return m_controller_type; } diff --git a/ZEngine/ZEngine/Core/EngineAssetRoot.cpp b/ZEngine/ZEngine/Core/EngineAssetRoot.cpp new file mode 100644 index 00000000..08dc95b5 --- /dev/null +++ b/ZEngine/ZEngine/Core/EngineAssetRoot.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#elif defined(__APPLE__) +#include +#elif defined(__linux__) +#include +#endif + +namespace ZEngine::Core +{ + namespace + { + constexpr const char* kEngineAssetDirectory = "ZodiacEngine"; + + std::filesystem::path NormalizePath(const std::filesystem::path& path) + { + std::error_code error; + std::filesystem::path absolute = std::filesystem::absolute(path, error); + return error ? path.lexically_normal() : absolute.lexically_normal(); + } + } // namespace + + std::filesystem::path GetExecutablePath() + { +#if defined(_WIN32) + std::vector buffer(MAX_PATH); + for (;;) + { + const DWORD length = GetModuleFileNameW(nullptr, buffer.data(), static_cast(buffer.size())); + if (length == 0) + return {}; + if (length < buffer.size()) + return std::filesystem::path(std::wstring(buffer.data(), length)); + buffer.resize(buffer.size() * 2); + } +#elif defined(__APPLE__) + uint32_t size = 0; + if (_NSGetExecutablePath(nullptr, &size) != -1 || size == 0) + return {}; + + std::vector buffer(size); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) + return {}; + return std::filesystem::path(buffer.data()); +#elif defined(__linux__) + std::vector buffer(1024); + for (;;) + { + const ssize_t length = readlink("/proc/self/exe", buffer.data(), buffer.size()); + if (length < 0) + return {}; + if (static_cast(length) < buffer.size()) + return std::filesystem::path(std::string(buffer.data(), static_cast(length))); + buffer.resize(buffer.size() * 2); + } +#else + return {}; +#endif + } + + EngineAssetRootResolution ResolveEngineAssetRoot(const std::filesystem::path& executable_path, const std::filesystem::path& override_root) + { + if (!override_root.empty()) + return {.Root = NormalizePath(override_root)}; + + if (executable_path.empty()) + return {.Diagnostic = "Unable to determine the executable path while resolving the engine asset package. Set ZENGINE_ASSET_ROOT to the ZodiacEngine directory."}; + + return {.Root = NormalizePath(executable_path.parent_path() / kEngineAssetDirectory)}; + } + + EngineAssetRootResolution ResolveEngineAssetRoot() + { + const char* const override_root = std::getenv(EngineAssetRootOverrideEnvironment); + return ResolveEngineAssetRoot(GetExecutablePath(), override_root && override_root[0] != '\0' ? std::filesystem::path(override_root) : std::filesystem::path{}); + } + + EngineAssetRootResolution ValidateEngineAssetRoot(const std::filesystem::path& root) + { + if (root.empty()) + return {.Diagnostic = "Engine asset root is empty"}; + + EngineAssetRootResolution result = {.Root = NormalizePath(root)}; + std::error_code error; + if (!std::filesystem::is_directory(result.Root, error)) + { + result.Diagnostic = "Engine asset root is missing or not a directory: " + result.Root.string(); + } + return result; + } + +} // namespace ZEngine::Core diff --git a/ZEngine/ZEngine/Core/EngineAssetRoot.h b/ZEngine/ZEngine/Core/EngineAssetRoot.h new file mode 100644 index 00000000..b9e0fb9f --- /dev/null +++ b/ZEngine/ZEngine/Core/EngineAssetRoot.h @@ -0,0 +1,34 @@ +#pragma once +#include +#include + +namespace ZEngine::Core +{ + inline constexpr const char* EngineAssetRootOverrideEnvironment = "ZENGINE_ASSET_ROOT"; + + struct EngineAssetRootResolution + { + std::filesystem::path Root = {}; + std::string Diagnostic = {}; + + [[nodiscard]] bool Succeeded() const + { + return Diagnostic.empty(); + } + }; + + // Returns the executable's native path, independent of the process working directory. + [[nodiscard]] std::filesystem::path GetExecutablePath(); + + // Resolves the directory containing packaged engine files. An explicit override + // takes precedence; otherwise assets live in /ZodiacEngine. + [[nodiscard]] EngineAssetRootResolution ResolveEngineAssetRoot(const std::filesystem::path& executable_path, const std::filesystem::path& override_root = {}); + + // Uses the current executable and the optional ZENGINE_ASSET_ROOT environment + // override. The override exists for development tools and package tests. + [[nodiscard]] EngineAssetRootResolution ResolveEngineAssetRoot(); + + // Checks that the native package directory exists before it is mounted. + [[nodiscard]] EngineAssetRootResolution ValidateEngineAssetRoot(const std::filesystem::path& root); + +} // namespace ZEngine::Core diff --git a/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp b/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp index a6388ca6..7950ceab 100644 --- a/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp +++ b/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp @@ -106,13 +106,10 @@ namespace ZEngine::Core::Memory // is this domain's dominant consumer. The generic one-off staging buffers // (AllocateBuffer with domain=HostStaging) share this same pool. // - // blockSize=0 (auto-sized), not a fixed StagingBytes block: a block needs some - // alignment/bookkeeping headroom beyond its raw byte count, so a fixed block - // exactly equal to the ring's own StagingBytes allocation can't actually fit it - // (confirmed: fails with VK_ERROR_OUT_OF_DEVICE_MEMORY). Same underlying reason as - // DeviceTexture's blockSize=0 below, just triggered by exact-equality instead of - // exceeding the block. - create_buffer_pool(GpuMemoryDomain::HostStaging, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VmaAllocationCreateInfo{.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}, 0, 1); + // blockSize=0 and maxBlockCount=0 let VMA add a block for one-off staging + // uploads after the ring's block is full. A one-block pool made every geometry + // staging allocation fall back to VMA's default allocator on discrete GPUs. + create_buffer_pool(GpuMemoryDomain::HostStaging, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VmaAllocationCreateInfo{.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}, 0, 0); // HostReadback has the inverse host-access contract of HostStaging. It is // deliberately a separate pool because readback allocations can stay leased diff --git a/ZEngine/ZEngine/Core/VFS/IVFSContext.h b/ZEngine/ZEngine/Core/VFS/IVFSContext.h index 8ba5f5b9..1b7900e4 100644 --- a/ZEngine/ZEngine/Core/VFS/IVFSContext.h +++ b/ZEngine/ZEngine/Core/VFS/IVFSContext.h @@ -4,10 +4,21 @@ #include #include #include +#include namespace ZEngine::Core::VFS { + struct VFSImportPublication + { + uint64_t Id = 0; + + [[nodiscard]] bool IsValid() const + { + return Id != 0; + } + }; + struct IVFSContext { virtual ~IVFSContext() = default; @@ -34,7 +45,17 @@ namespace ZEngine::Core::VFS virtual void Close(IVFSFile* const file) = 0; - virtual void Shutdown() = 0; + // Editor importers use this scope to publish a group of generated files without + // letting watcher callbacks index the group while it is incomplete. Default + // no-ops preserve the lightweight VFS contexts used by tools and tests. + [[nodiscard]] virtual VFSImportPublication BeginImportPublication() + { + return {}; + } + virtual void RecordImportPublicationArtifact(VFSImportPublication, const VFSPath&) {} + virtual void EndImportPublication(VFSImportPublication) {} + + virtual void Shutdown() = 0; }; } // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp index 9f938184..d221d3df 100644 --- a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp +++ b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp @@ -1,20 +1,20 @@ #include +#include #include #include #include #include +#include #include -#include +#include #include +#include namespace ZEngine::Core::VFS { namespace { - // .meta files are small JSON — 8 KB is a generous upper bound. - static constexpr size_t k_meta_read_cap = 8192; - - static int64_t NowNs() + static int64_t NowNs() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } @@ -22,33 +22,26 @@ namespace ZEngine::Core::VFS static void CopyStr(const std::string& src, char* dst, size_t cap) { size_t n = src.size() < cap - 1 ? src.size() : cap - 1; - std::memcpy(dst, src.data(), n); + Helpers::secure_memcpy(dst, cap, src.data(), n); dst[n] = '\0'; } } // namespace // MetaFileIO - VFSPath MetaFileIO::MetaPathFor(const VFSPath& asset_path) + VFSResult MetaFileIO::MetaPathFor(const VFSPath& asset_path) { - char buf[MAX_FILE_PATH_COUNT] = {}; - const char* raw = asset_path.CStr(); - size_t len = Helpers::secure_strlen(raw); - size_t avail = MAX_FILE_PATH_COUNT - 1; - size_t copy = len < avail ? len : avail; - Helpers::secure_memcpy(buf, MAX_FILE_PATH_COUNT, raw, copy); - const char suffix[] = ".meta"; - size_t suf_len = sizeof(suffix) - 1; - if (copy + suf_len < MAX_FILE_PATH_COUNT) - Helpers::secure_memcpy(buf + copy, MAX_FILE_PATH_COUNT - copy, suffix, suf_len + 1); - return VFSPath::Parse(buf).Value(); + if (!asset_path.IsValid() || asset_path.IsRoot()) + return VFSResult::Fail(VFSError::InvalidPath); + return VFSPath::Parse((std::string(asset_path.CStr()) + ".meta").c_str()); } VFSResult MetaFileIO::Read(IVFSContext& ctx, const VFSPath& asset_path) { - VFSPath meta_path = MetaPathFor(asset_path); - - auto open_result = ctx.Open(meta_path, VFSOpenFlags::Read); + auto meta_path = MetaPathFor(asset_path); + if (meta_path.Failed()) + return VFSResult::Fail(meta_path.Error()); + auto open_result = ctx.Open(meta_path.Value(), VFSOpenFlags::Read); if (open_result.Failed()) return VFSResult::Fail(open_result.Error()); @@ -62,33 +55,33 @@ namespace ZEngine::Core::VFS } uint64_t size = size_result.Value(); - if (size >= k_meta_read_cap) + if (size > MaxFileSize) { ctx.Close(file); - return VFSResult::Fail(VFSError::Corrupted); + return VFSResult::Fail(VFSError::SizeLimitExceeded); } - uint8_t buf[k_meta_read_cap]; - auto read_result = file->ReadAll({buf, (size_t) size}); + std::vector buf(static_cast(size)); + auto read_result = file->ReadAll({buf.data(), buf.size()}); ctx.Close(file); if (read_result.Failed()) return VFSResult::Fail(read_result.Error()); + if (read_result.Value() != size) + return VFSResult::Fail(VFSError::IOError); - buf[size] = '\0'; - - auto j = nlohmann::json::parse(reinterpret_cast(buf), nullptr, /*allow_exceptions=*/false); - if (j.is_discarded()) + auto j = nlohmann::json::parse(buf.begin(), buf.end(), nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) return VFSResult::Fail(VFSError::Corrupted); MetaFileData out{}; - if (j.contains("uuid") && j["uuid"].is_string()) - { - auto parsed = uuids::uuid::from_string(j["uuid"].get()); - if (parsed.has_value()) - out.AssetUUID = parsed.value(); - } + if (!j.contains("uuid") || !j["uuid"].is_string()) + return VFSResult::Fail(VFSError::Corrupted); + const auto parsed = uuids::uuid::from_string(j["uuid"].get()); + if (!parsed.has_value() || parsed->is_nil()) + return VFSResult::Fail(VFSError::Corrupted); + out.AssetUUID = *parsed; if (j.contains("importer") && j["importer"].is_string()) CopyStr(j["importer"].get(), out.ImporterName, sizeof(out.ImporterName)); @@ -111,7 +104,7 @@ namespace ZEngine::Core::VFS { if (out.SettingsCount >= META_MAX_SETTINGS) break; - if (!s.contains("key") || !s.contains("value")) + if (!s.contains("key") || !s.contains("value") || !s["key"].is_string() || !s["value"].is_string()) continue; auto& kv = out.Settings[out.SettingsCount++]; CopyStr(s["key"].get(), kv.Key, sizeof(kv.Key)); @@ -125,6 +118,11 @@ namespace ZEngine::Core::VFS VFSResult MetaFileIO::Write(IVFSContext& ctx, const VFSPath& asset_path, const MetaFileData& data) { + auto meta_path = MetaPathFor(asset_path); + if (meta_path.Failed()) + return VFSResult::Fail(meta_path.Error()); + if (data.AssetUUID.is_nil() || data.SettingsCount > META_MAX_SETTINGS) + return VFSResult::Fail(VFSError::Corrupted); nlohmann::json j; j["uuid"] = uuids::to_string(data.AssetUUID); j["importer"] = data.ImporterName; @@ -142,44 +140,19 @@ namespace ZEngine::Core::VFS }); } - std::string serialized = j.dump(4); - - // Build tmp path: .tmp - VFSPath meta_path = MetaPathFor(asset_path); - char tmp_buf[MAX_FILE_PATH_COUNT] = {}; - const char* meta_raw = meta_path.CStr(); - size_t meta_len = Helpers::secure_strlen(meta_raw); - size_t avail = MAX_FILE_PATH_COUNT - 1; - size_t copy = meta_len < avail ? meta_len : avail; - Helpers::secure_memcpy(tmp_buf, MAX_FILE_PATH_COUNT, meta_raw, copy); - const char tmp_suffix[] = ".tmp"; - size_t suf_len = sizeof(tmp_suffix) - 1; - if (copy + suf_len < MAX_FILE_PATH_COUNT) - Helpers::secure_memcpy(tmp_buf + copy, MAX_FILE_PATH_COUNT - copy, tmp_suffix, suf_len + 1); - VFSPath tmp_path = VFSPath::Parse(tmp_buf).Value(); - - auto open_result = ctx.Open(tmp_path, VFSOpenFlags::Write | VFSOpenFlags::Create | VFSOpenFlags::Truncate); - if (open_result.Failed()) - return VFSResult::Fail(open_result.Error()); - - IVFSFile* file = open_result.Value(); - const auto* bytes = reinterpret_cast(serialized.data()); - auto write_result = file->Write({bytes, serialized.size()}, 0); - auto flush_result = file->Flush(); - file->Close(); - ctx.Close(file); - - if (write_result.Failed()) - return VFSResult::Fail(write_result.Error()); - if (flush_result.Failed()) - return VFSResult::Fail(flush_result.Error()); - - return ctx.Rename(tmp_path, meta_path); + const std::string serialized = j.dump(4); + if (serialized.size() > MaxFileSize) + return VFSResult::Fail(VFSError::SizeLimitExceeded); + return WriteFileAtomically(ctx, meta_path.Value(), {reinterpret_cast(serialized.data()), serialized.size()}); } - VFSResult MetaFileIO::GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash) + VFSResult MetaFileIO::GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash, bool use_embedded_identity) { - auto read_result = Read(ctx, asset_path); + // Scanner and importer callers must not mint different identities for + // the same absent sidecar. Writes remain atomic; serialize this RMW. + static std::mutex identity_mutex; + std::lock_guard identity_lock(identity_mutex); + auto read_result = Read(ctx, asset_path); if (read_result.Succeeded()) { @@ -194,12 +167,26 @@ namespace ZEngine::Core::VFS existing.SourceHash = current_hash; existing.LastImportTimeNs = NowNs(); existing.Status = ImportStatus::Stale; - Write(ctx, asset_path, existing); // best-effort; ignore failure + auto write = Write(ctx, asset_path, existing); + if (write.Failed()) + return VFSResult::Fail(write.Error()); return VFSResult::Ok(existing); } - // No .meta or corrupt .meta — generate a fresh UUID. + if (read_result.Error() != VFSError::NotFound && read_result.Error() != VFSError::Corrupted) + return VFSResult::Fail(read_result.Error()); + + // Resolve and persist identity under the same lock; scanners must not seed + // sidecars separately or mint random identities for unreadable cooked assets. MetaFileData fresh{}; + if (use_embedded_identity && (asset_path.Extension().Equals(".zemesh") || asset_path.Extension().Equals(".zematerial"))) + { + auto embedded = ReadEmbeddedAssetUUID(ctx, asset_path); + if (embedded.Failed()) + return VFSResult::Fail(embedded.Error()); + fresh.AssetUUID = embedded.Value(); + } + else { std::random_device rd; std::mt19937 generator(rd()); @@ -210,7 +197,9 @@ namespace ZEngine::Core::VFS fresh.SourceHash = current_hash; fresh.LastImportTimeNs = NowNs(); fresh.Status = ImportStatus::New; - Write(ctx, asset_path, fresh); + auto write = Write(ctx, asset_path, fresh); + if (write.Failed()) + return VFSResult::Fail(write.Error()); return VFSResult::Ok(fresh); } @@ -220,26 +209,37 @@ namespace ZEngine::Core::VFS if (open_result.Failed()) return VFSResult::Fail(open_result.Error()); - IVFSFile* file = open_result.Value(); + IVFSFile* file = open_result.Value(); + + auto size_result = file->Size(); + if (size_result.Failed()) + { + ctx.Close(file); + return VFSResult::Fail(size_result.Error()); + } // Stream through the file in 4 KB chunks, chaining rapidhash via the seed parameter. - uint8_t chunk[4096]; - uint64_t hash = 0; - uint64_t offset = 0; + uint8_t chunk[4096]; + uint64_t hash = 0; + uint64_t offset = 0; - while (true) + while (offset < size_result.Value()) { - auto read_result = file->Read({chunk, sizeof(chunk)}, offset); - if (read_result.Failed()) + const size_t count = static_cast(std::min(sizeof(chunk), size_result.Value() - offset)); + size_t filled = 0; + while (filled < count) { - ctx.Close(file); - return VFSResult::Fail(read_result.Error()); + auto read = file->Read({chunk + filled, count - filled}, offset + filled); + if (read.Failed() || read.Value() == 0 || read.Value() > count - filled) + { + ctx.Close(file); + return VFSResult::Fail(read.Failed() ? read.Error() : VFSError::IOError); + } + filled += read.Value(); } - size_t n = read_result.Value(); - if (n == 0) - break; - hash = rapidhash_withSeed(chunk, n, hash); - offset += (uint64_t) n; + // Keep chunk boundaries stable even when the backend returns partial reads. + hash = rapidhash_withSeed(chunk, count, hash); + offset += count; } ctx.Close(file); return VFSResult::Ok(hash); diff --git a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h index f7bc262f..02b87476 100644 --- a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h +++ b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h @@ -8,8 +8,12 @@ namespace ZEngine::Core::VFS { struct MetaFileIO { + // Covers all fixed-capacity fields, including worst-case JSON escaping. + static constexpr size_t MaxFileSize = 64 * 1024; + static_assert(MaxFileSize >= 6 * sizeof(MetaFileData) + 4096, "Metadata capacity must cover escaped fields and JSON formatting"); + // Returns the sidecar path for an asset: "/project/mesh.glb" -> "/project/mesh.glb.meta" - static VFSPath MetaPathFor(const VFSPath& asset_path); + static VFSResult MetaPathFor(const VFSPath& asset_path); // Reads the .meta sidecar from the VFS. Returns Fail if the file is absent or malformed. // Status on the returned MetaFileData is always ImportStatus::Unknown — caller sets it. @@ -22,8 +26,9 @@ namespace ZEngine::Core::VFS // - No .meta or corrupt .meta -> generate UUID, write, return New // - .meta exists, hash matches -> return UpToDate (no write) // - .meta exists, hash differs -> update hash + timestamp, write, return Stale - // The UUID in an existing .meta is never replaced. - static VFSResult GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash); + // Valid existing identities are preserved; missing/corrupt cooked sidecars can use the embedded identity. + // Access and size-limit errors are returned without changing the sidecar. + static VFSResult GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash, bool use_embedded_identity = false); // Computes the rapidhash-64 digest of the asset file content. static VFSResult ComputeHash(IVFSContext& ctx, const VFSPath& asset_path); diff --git a/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm b/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm index f3a58013..3070ecb1 100644 --- a/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm +++ b/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm @@ -21,6 +21,7 @@ Helpers::secure_strncpy(ev.Path, sizeof(ev.Path), path, ClampedLength(path)); ev.Kind = kind; ev.IsDirectory = is_directory; + ev.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); return ev; } @@ -103,6 +104,7 @@ { VFSWatchEvent overflow{}; overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); self->PushEvent(overflow); continue; } @@ -264,6 +266,11 @@ { std::lock_guard lock(m_watch_mutex); m_run_loop = CFRunLoopGetCurrent(); + // The final watch can be removed while the thread is running, + // leaving the run loop with no sources. CFRunLoopRun then returns + // before StopThread joins us, so keep an owned reference until the + // joining thread has finished using it. + CFRetain(m_run_loop); RebuildStream(); } // Signal that FSEventStreamStart has been called and the run loop @@ -299,6 +306,8 @@ m_thread.join(); } std::lock_guard lock(m_watch_mutex); + if (m_run_loop) + CFRelease(m_run_loop); m_run_loop = nullptr; } diff --git a/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp b/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp index 9f066e82..fc4c661c 100644 --- a/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp +++ b/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp @@ -57,8 +57,9 @@ namespace ZEngine::Core::VFS { VFSWatchEvent ev{}; Helpers::secure_strncpy(ev.Path, sizeof(ev.Path), path, ClampedLength(path)); - ev.Kind = kind; - ev.IsDirectory = is_directory; + ev.Kind = kind; + ev.IsDirectory = is_directory; + ev.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); return ev; } @@ -112,7 +113,8 @@ namespace ZEngine::Core::VFS if (wd < 0) { VFSWatchEvent overflow{}; - overflow.Kind = WatchEventKind::Overflow; + overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); PushEvent(overflow); return -1; } @@ -236,7 +238,8 @@ namespace ZEngine::Core::VFS if (raw->mask & IN_Q_OVERFLOW) { VFSWatchEvent overflow{}; - overflow.Kind = WatchEventKind::Overflow; + overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); m_batch.push(overflow); continue; } diff --git a/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp b/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp index 69fc450b..8be12734 100644 --- a/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp +++ b/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp @@ -59,7 +59,8 @@ namespace ZEngine::Core::VFS { VFSWatchEvent ev{}; Helpers::secure_strncpy(ev.Path, sizeof(ev.Path), path, ClampedLength(path)); - ev.Kind = kind; + ev.Kind = kind; + ev.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); return ev; } @@ -172,7 +173,8 @@ namespace ZEngine::Core::VFS if (bytes == 0) { VFSWatchEvent overflow{}; - overflow.Kind = WatchEventKind::Overflow; + overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); PushEvent(overflow); return; } diff --git a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp index f694c4c6..eac1eb18 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp @@ -20,6 +20,12 @@ namespace ZEngine::Core::VFS { m_arena = arena; m_mount_table.Initialize(m_arena, mount_table_capacity); + + m_active_import_publication.Artifacts.init(m_arena, 16); + m_active_import_publication.DirtyDirectories.init(m_arena, 8); + m_active_import_publication.DeferredEvents.init(m_arena, 32); + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + m_completed_import_publications[i].Artifacts.init(m_arena, 16); } void VFSContext::InitWatcher(const char* project_root_native, VFSDirectoryCache* cache, VFSScanner* scanner, AssetRegistry* registry, Importers::ImportCoordinator* coordinator, FileChangeListener file_change_listener, void* file_change_context) @@ -119,97 +125,271 @@ namespace ZEngine::Core::VFS // simultaneous pending debounce entries than that before any flush. m_file_watcher->Initialize(m_arena, 256); - // ev.Path/ev.OldPath are native absolute paths from the OS watcher, but every - // downstream consumer expects a workspace-relative VFSPath and prepends the - // workspace root itself — VFSPath::FromNative is just Parse, so it wouldn't strip - // that root. Do it here, once, instead of double-prefixing everywhere downstream. - auto to_relative_vfs_path = [this](cstring native) -> VFSResult { - const size_t root_len = Helpers::secure_strlen(m_project_root_native); - const size_t native_len = Helpers::secure_strlen(native); - if (native_len >= root_len && strncmp(native, m_project_root_native, root_len) == 0 && (native[root_len] == '\0' || native[root_len] == PLATFORM_OS_BACKSLASH)) - return VFSPath::Parse(native[root_len] != '\0' ? native + root_len : "/"); - return VFSPath::FromNative(native); - }; + const WatchHandle root_handle = m_file_watcher->Watch(m_project_root_native, /*recursive=*/true, [this](const VFSWatchEvent& event) { HandleWatchEvent(event); }); + + if (root_handle == INVALID_WATCH_HANDLE) + { + ZENGINE_LOG_VFS_ERR("InitWatcher: failed to watch root path '{}'", m_project_root_native); + m_file_watcher->~VFSFileWatcher(); + m_file_watcher = nullptr; + m_platform_watcher->~IVFSPlatformWatcher(); + m_platform_watcher = nullptr; + return; + } - const WatchHandle root_handle = m_file_watcher->Watch(m_project_root_native, /*recursive=*/true, [this, to_relative_vfs_path](const VFSWatchEvent& ev) { - const bool full_rescan = (ev.Kind == WatchEventKind::Overflow); + m_platform_watcher->StartThread(); + } - VFSResult path = full_rescan ? to_relative_vfs_path(m_project_root_native) : to_relative_vfs_path(ev.Path); - if (path.Failed()) + VFSImportPublication VFSContext::BeginImportPublication() + { + if (m_active_import_publication.Publication.IsValid()) + { + ++m_active_import_publication.Depth; + return m_active_import_publication.Publication; + } + + m_active_import_publication.Publication = {.Id = m_next_import_publication_id++}; + m_active_import_publication.StartedAtNs = VFSWatchTimestampNowNanoseconds(); + m_active_import_publication.Depth = 1; + m_active_import_publication.RequiresScan = false; + m_active_import_publication.Artifacts.clear(); + m_active_import_publication.DirtyDirectories.clear(); + m_active_import_publication.DeferredEvents.clear(); + return m_active_import_publication.Publication; + } + + void VFSContext::RecordImportPublicationArtifact(VFSImportPublication publication, const VFSPath& path) + { + if (!publication.IsValid() || publication.Id != m_active_import_publication.Publication.Id || !path.IsValid()) + return; + + for (size_t i = 0; i < m_active_import_publication.Artifacts.size(); ++i) + if (m_active_import_publication.Artifacts[i] == path) + return; + + m_active_import_publication.Artifacts.push(path); + + const VFSPath parent = path.Parent(); + for (size_t i = 0; i < m_active_import_publication.DirtyDirectories.size(); ++i) + if (m_active_import_publication.DirtyDirectories[i] == parent) return; + m_active_import_publication.DirtyDirectories.push(parent); + } - const VFSPath target = (ev.IsDirectory || full_rescan) ? path.Value() : path.Value().Parent(); - const VFSPath file_path = path.Value(); + void VFSContext::EndImportPublication(VFSImportPublication publication) + { + if (!publication.IsValid() || publication.Id != m_active_import_publication.Publication.Id) + return; + if (m_active_import_publication.Depth > 1) + { + --m_active_import_publication.Depth; + return; + } + + const uint64_t ended_at_ns = VFSWatchTimestampNowNanoseconds(); - // Invalidate directory cache - if (m_directory_cache) + // Events arriving during a publication are held until all artifact paths are + // known. Generated paths are coalesced into the deferred scan; unrelated + // paths retain the normal registry/coordinator behavior. + for (size_t i = 0; i < m_active_import_publication.DeferredEvents.size(); ++i) + { + const VFSWatchEvent& event = m_active_import_publication.DeferredEvents[i]; + if (event.Kind == WatchEventKind::Overflow || IsPublicationEvent(m_active_import_publication.StartedAtNs, m_active_import_publication.Artifacts, event)) { - m_directory_cache->Invalidate(target); - if (ev.Kind == WatchEventKind::Renamed && ev.OldPath[0] != '\0') - { - VFSResult old_path = to_relative_vfs_path(ev.OldPath); - if (old_path.Succeeded()) - m_directory_cache->Invalidate(ev.IsDirectory ? old_path.Value() : old_path.Value().Parent()); - } + m_active_import_publication.RequiresScan = true; + continue; } + ProcessWatchEvent(event); + } + m_active_import_publication.DeferredEvents.clear(); + + if (!m_active_import_publication.Artifacts.empty()) + { + for (size_t i = 0; i < m_active_import_publication.DirtyDirectories.size(); ++i) + if (m_directory_cache) + m_directory_cache->Invalidate(m_active_import_publication.DirtyDirectories[i]); + m_active_import_publication.RequiresScan = true; + } + + // Retain the exact output set until watcher delivery has settled. A raw + // inotify/FSEvents event may arrive after the importer returns, so using + // EndImportPublication's timestamp as a causal boundary is not reliable. + CompletedImportPublication& completed = m_completed_import_publications[m_next_completed_import_publication]; + completed.StartedAtNs = m_active_import_publication.StartedAtNs; + completed.SettleUntilNs = ended_at_ns + IMPORT_PUBLICATION_SETTLE_NS; + completed.RequiresScan = m_active_import_publication.RequiresScan; + completed.Artifacts.clear(); + for (size_t i = 0; i < m_active_import_publication.Artifacts.size(); ++i) + completed.Artifacts.push(m_active_import_publication.Artifacts[i]); + m_next_completed_import_publication = (m_next_completed_import_publication + 1) % COMPLETED_IMPORT_PUBLICATION_COUNT; + + m_active_import_publication.Publication = {}; + m_active_import_publication.StartedAtNs = 0; + m_active_import_publication.Depth = 0; + m_active_import_publication.RequiresScan = false; + m_active_import_publication.Artifacts.clear(); + m_active_import_publication.DirtyDirectories.clear(); + } + + VFSResult VFSContext::ToRelativeVFSPath(cstring native) const + { + if (!native) + return VFSResult::Fail(VFSError::InvalidPath); + + const size_t root_len = Helpers::secure_strlen(m_project_root_native); + const size_t native_len = Helpers::secure_strlen(native); + if (native_len >= root_len && strncmp(native, m_project_root_native, root_len) == 0 && (native[root_len] == '\0' || native[root_len] == PLATFORM_OS_BACKSLASH)) + return VFSPath::Parse(native[root_len] != '\0' ? native + root_len : "/"); + return VFSPath::FromNative(native); + } + + bool VFSContext::IsPublicationArtifact(const Containers::Array& artifacts, const VFSPath& path) const + { + for (size_t i = 0; i < artifacts.size(); ++i) + { + const VFSPath& artifact = artifacts[i]; + if (artifact == path) + return true; + // Metadata is stored beside its registered artifact as .meta. + if (path.Length() == artifact.Length() + 5 && strncmp(path.CStr(), artifact.CStr(), artifact.Length()) == 0 && strncmp(path.CStr() + artifact.Length(), ".meta", 5) == 0) + return true; + } + return false; + } + + bool VFSContext::IsPublicationEvent(uint64_t started_at_ns, const Containers::Array& artifacts, const VFSWatchEvent& event) const + { + if (event.Kind == WatchEventKind::Overflow || event.ObservedAtNanoseconds == 0 || event.ObservedAtNanoseconds < started_at_ns) + return false; + + VFSResult path = ToRelativeVFSPath(event.Path); + return path.Succeeded() && IsPublicationArtifact(artifacts, path.Value()); + } - // Notify AssetRegistry and ImportCoordinator for file (non-directory) events - // Skip .meta sidecar files — they are written by the coordinator itself and must - // not be fed back into the import queue. - auto is_meta = [](const VFSPath& p) -> bool { - auto ext = p.Extension(); - return ext.Data && ext.Length == 5 && // ".meta" - ext.Data[0] == '.' && ext.Data[1] == 'm' && ext.Data[2] == 'e' && ext.Data[3] == 't' && ext.Data[4] == 'a'; - }; - if (!ev.IsDirectory && !full_rescan && !is_meta(file_path)) + void VFSContext::HandleWatchEvent(const VFSWatchEvent& event) + { + if (m_active_import_publication.Publication.IsValid() && event.ObservedAtNanoseconds >= m_active_import_publication.StartedAtNs) + { + m_active_import_publication.DeferredEvents.push(event); + return; + } + + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + CompletedImportPublication& completed = m_completed_import_publications[i]; + if (completed.SettleUntilNs != 0 && VFSWatchTimestampNowNanoseconds() <= completed.SettleUntilNs && IsPublicationEvent(completed.StartedAtNs, completed.Artifacts, event)) { - switch (ev.Kind) - { - case WatchEventKind::Modified: - if (m_registry) - m_registry->OnAssetModified(file_path); - if (m_coordinator) - m_coordinator->Enqueue(file_path, Importers::ImportPriority::Immediate); - break; + completed.RequiresScan = true; + return; + } + } - case WatchEventKind::Deleted: - if (m_registry) - m_registry->OnAssetDeleted(file_path); - break; + ProcessWatchEvent(event); + } - case WatchEventKind::Renamed: - if (m_registry && ev.OldPath[0] != '\0') - { - VFSResult old_path = to_relative_vfs_path(ev.OldPath); - if (old_path.Succeeded()) - m_registry->OnAssetRenamed(old_path.Value(), file_path); - } - break; + void VFSContext::ProcessWatchEvent(const VFSWatchEvent& event) + { + const bool full_rescan = (event.Kind == WatchEventKind::Overflow); + VFSResult path = full_rescan ? ToRelativeVFSPath(m_project_root_native) : ToRelativeVFSPath(event.Path); + if (path.Failed()) + return; - default: - break; - } + const VFSPath target = (event.IsDirectory || full_rescan) ? path.Value() : path.Value().Parent(); + const VFSPath file_path = path.Value(); - if (m_file_change_listener) - m_file_change_listener(m_file_change_context, file_path, ev.Kind); + if (m_directory_cache) + { + m_directory_cache->Invalidate(target); + if (event.Kind == WatchEventKind::Renamed && event.OldPath[0] != '\0') + { + VFSResult old_path = ToRelativeVFSPath(event.OldPath); + if (old_path.Succeeded()) + m_directory_cache->Invalidate(event.IsDirectory ? old_path.Value() : old_path.Value().Parent()); } + } - // Rescan directory - if (m_scanner && m_directory_cache && !m_scanner->IsScanning()) - m_scanner->Scan(this, target, m_directory_cache); - }); + auto is_temporary = [](const VFSPath& path) -> bool { + const VFSPathComponent extension = path.Extension(); + return extension.Data && extension.Length == 4 && extension.Data[0] == '.' && extension.Data[1] == 't' && extension.Data[2] == 'm' && extension.Data[3] == 'p'; + }; + auto is_meta = [](const VFSPath& path) -> bool { + const VFSPathComponent extension = path.Extension(); + return extension.Data && extension.Length == 5 && extension.Data[0] == '.' && extension.Data[1] == 'm' && extension.Data[2] == 'e' && extension.Data[3] == 't' && extension.Data[4] == 'a'; + }; - if (root_handle == INVALID_WATCH_HANDLE) + if (!event.IsDirectory && !full_rescan && is_temporary(file_path)) + return; + if (!event.IsDirectory && !full_rescan && !is_meta(file_path)) { - ZENGINE_LOG_VFS_ERR("InitWatcher: failed to watch root path '{}'", m_project_root_native); - m_file_watcher->~VFSFileWatcher(); - m_file_watcher = nullptr; - m_platform_watcher->~IVFSPlatformWatcher(); - m_platform_watcher = nullptr; + switch (event.Kind) + { + case WatchEventKind::Modified: + if (m_registry) + m_registry->OnAssetModified(file_path); + if (m_coordinator) + m_coordinator->Enqueue(file_path, Importers::ImportPriority::Immediate); + break; + + case WatchEventKind::Deleted: + if (m_registry) + m_registry->OnAssetDeleted(file_path); + break; + + case WatchEventKind::Renamed: + if (m_registry && event.OldPath[0] != '\0') + { + VFSResult old_path = ToRelativeVFSPath(event.OldPath); + if (old_path.Succeeded()) + m_registry->OnAssetRenamed(old_path.Value(), file_path); + } + break; + + default: + break; + } + + if (m_file_change_listener) + m_file_change_listener(m_file_change_context, file_path, event.Kind); + } + + if (m_scanner && m_directory_cache && !m_scanner->IsScanning()) + m_scanner->Scan(this, target, m_directory_cache); + } + + void VFSContext::FlushDeferredImportScan() + { + if (!m_scanner || !m_directory_cache) return; + + const uint64_t now_ns = VFSWatchTimestampNowNanoseconds(); + bool should_scan = false; + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + const CompletedImportPublication& completed = m_completed_import_publications[i]; + if (completed.SettleUntilNs != 0 && completed.RequiresScan && now_ns >= completed.SettleUntilNs) + { + should_scan = true; + break; + } } + if (!should_scan || m_scanner->IsScanning()) + return; - m_platform_watcher->StartThread(); + // Start one scan for every completed publication whose watcher grace period + // elapsed. The scan observes the final filesystem state, including a real + // external edit that happened during the grace period. + ScanProject(); + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + CompletedImportPublication& completed = m_completed_import_publications[i]; + if (completed.SettleUntilNs != 0 && now_ns >= completed.SettleUntilNs) + { + completed.StartedAtNs = 0; + completed.SettleUntilNs = 0; + completed.RequiresScan = false; + completed.Artifacts.clear(); + } + } } void VFSContext::ScanProject() @@ -224,6 +404,7 @@ namespace ZEngine::Core::VFS { m_file_watcher->Tick(); } + FlushDeferredImportScan(); } void VFSContext::ShutdownWatcher() @@ -242,8 +423,19 @@ namespace ZEngine::Core::VFS m_platform_watcher->~IVFSPlatformWatcher(); m_platform_watcher = nullptr; } - m_directory_cache = nullptr; - m_scanner = nullptr; + m_directory_cache = nullptr; + m_scanner = nullptr; + m_active_import_publication.Publication = {}; + m_active_import_publication.Artifacts.clear(); + m_active_import_publication.DirtyDirectories.clear(); + m_active_import_publication.DeferredEvents.clear(); + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + m_completed_import_publications[i].StartedAtNs = 0; + m_completed_import_publications[i].SettleUntilNs = 0; + m_completed_import_publications[i].RequiresScan = false; + m_completed_import_publications[i].Artifacts.clear(); + } } VFSResult VFSContext::Open(const VFSPath& absolute_path, VFSOpenFlags flags) diff --git a/ZEngine/ZEngine/Core/VFS/VFSContext.h b/ZEngine/ZEngine/Core/VFS/VFSContext.h index 7cabeaa0..bc8aa793 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSContext.h +++ b/ZEngine/ZEngine/Core/VFS/VFSContext.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -26,6 +27,7 @@ namespace ZEngine namespace ZEngine::Core::VFS { struct VFSScanner; + struct VFSContextTestAccess; using FileChangeListener = void (*)(void* context, const VFSPath& path, WatchEventKind kind); @@ -50,6 +52,10 @@ namespace ZEngine::Core::VFS void ShutdownWatcher(); + [[nodiscard]] VFSImportPublication BeginImportPublication() override; + void RecordImportPublicationArtifact(VFSImportPublication publication, const VFSPath& path) override; + void EndImportPublication(VFSImportPublication publication) override; + [[nodiscard]] VFSResult Open(const VFSPath& absolute_path, VFSOpenFlags flags) override; void Close(IVFSFile* file) override; [[nodiscard]] VFSResult> List(const VFSPath& absolute_dir, Memory::ArenaAllocator* out_arena) override; @@ -64,21 +70,59 @@ namespace ZEngine::Core::VFS void Shutdown() override; private: + friend struct VFSContextTestAccess; + + struct ImportPublicationState + { + VFSImportPublication Publication = {}; + uint64_t StartedAtNs = 0; + uint32_t Depth = 0; + bool RequiresScan = false; + Containers::Array Artifacts = {}; + Containers::Array DirtyDirectories = {}; + Containers::Array DeferredEvents = {}; + }; + + struct CompletedImportPublication + { + uint64_t StartedAtNs = 0; + uint64_t SettleUntilNs = 0; + bool RequiresScan = false; + Containers::Array Artifacts = {}; + }; + [[nodiscard]] VFSResult ResolveWritable(const VFSPath& path) const; + [[nodiscard]] VFSResult ToRelativeVFSPath(cstring native) const; + [[nodiscard]] bool IsPublicationEvent(uint64_t started_at_ns, const Containers::Array& artifacts, const VFSWatchEvent& event) const; + [[nodiscard]] bool IsPublicationArtifact(const Containers::Array& artifacts, const VFSPath& path) const; + void HandleWatchEvent(const VFSWatchEvent& event); + void ProcessWatchEvent(const VFSWatchEvent& event); + void FlushDeferredImportScan(); VFSMountTable m_mount_table = {}; Memory::ArenaAllocator* m_arena = nullptr; mutable std::mutex m_arena_mutex; - IVFSPlatformWatcher* m_platform_watcher = nullptr; - VFSFileWatcher* m_file_watcher = nullptr; - VFSDirectoryCache* m_directory_cache = nullptr; - VFSScanner* m_scanner = nullptr; - AssetRegistry* m_registry = nullptr; - Importers::ImportCoordinator* m_coordinator = nullptr; - FileChangeListener m_file_change_listener = nullptr; - void* m_file_change_context = nullptr; - char m_project_root_native[MAX_FILE_PATH_COUNT] = {}; + IVFSPlatformWatcher* m_platform_watcher = nullptr; + VFSFileWatcher* m_file_watcher = nullptr; + VFSDirectoryCache* m_directory_cache = nullptr; + VFSScanner* m_scanner = nullptr; + AssetRegistry* m_registry = nullptr; + Importers::ImportCoordinator* m_coordinator = nullptr; + FileChangeListener m_file_change_listener = nullptr; + void* m_file_change_context = nullptr; + char m_project_root_native[MAX_FILE_PATH_COUNT] = {}; + + // Native watchers observe filesystem changes asynchronously. Keep each + // completed publication long enough for the raw event and its debounce + // window to drain before a single reconciliation scan is allowed through. + static constexpr uint64_t IMPORT_PUBLICATION_SETTLE_NS = 500'000'000ULL; + static constexpr uint32_t COMPLETED_IMPORT_PUBLICATION_COUNT = 8; + + ImportPublicationState m_active_import_publication = {}; + CompletedImportPublication m_completed_import_publications[COMPLETED_IMPORT_PUBLICATION_COUNT] = {}; + uint32_t m_next_completed_import_publication = 0; + uint64_t m_next_import_publication_id = 1; }; } // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp b/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp index 42c3d3ec..79cd90a2 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -328,10 +329,12 @@ namespace ZEngine::Core::VFS file->m_handle = CreateFileA(native, access, FILE_SHARE_READ, nullptr, creation, FILE_ATTRIBUTE_NORMAL, nullptr); if (file->m_handle == INVALID_HANDLE_VALUE) { + const DWORD native_error = GetLastError(); + const auto error = native_error == ERROR_FILE_NOT_FOUND || native_error == ERROR_PATH_NOT_FOUND ? VFSError::NotFound : native_error == ERROR_ACCESS_DENIED ? VFSError::PermissionDenied : VFSError::IOError; file->~VFSDiskFile(); std::lock_guard lock(m_file_pool_mutex); m_file_pool.Free(file); - return VFSResult::Fail(VFSError::NotFound); + return VFSResult::Fail(error); } LARGE_INTEGER sz; if (GetFileSizeEx(file->m_handle, &sz)) @@ -349,10 +352,12 @@ namespace ZEngine::Core::VFS file->m_fd = ::open(native, oflags, 0644); if (file->m_fd < 0) { + const int native_error = errno; + const auto error = native_error == ENOENT ? VFSError::NotFound : native_error == EACCES || native_error == EPERM ? VFSError::PermissionDenied : native_error == ENOTDIR ? VFSError::NotADirectory : native_error == EISDIR ? VFSError::NotAFile : VFSError::IOError; file->~VFSDiskFile(); std::lock_guard lock(m_file_pool_mutex); m_file_pool.Free(file); - return VFSResult::Fail(VFSError::NotFound); + return VFSResult::Fail(error); } struct stat st; if (::fstat(file->m_fd, &st) == 0) diff --git a/ZEngine/ZEngine/Core/VFS/VFSError.h b/ZEngine/ZEngine/Core/VFS/VFSError.h index 4083939d..386e8611 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSError.h +++ b/ZEngine/ZEngine/Core/VFS/VFSError.h @@ -6,18 +6,19 @@ namespace ZEngine::Core::VFS { enum class VFSError : uint32_t { - OK = 0, - NotFound = 1, - PermissionDenied = 2, - AlreadyExists = 3, - NotADirectory = 4, - NotAFile = 5, - InvalidPath = 6, - Unsupported = 7, - IOError = 8, - OutOfMemory = 9, - Corrupted = 10, - Cancelled = 11, + OK = 0, + NotFound = 1, + PermissionDenied = 2, + AlreadyExists = 3, + NotADirectory = 4, + NotAFile = 5, + InvalidPath = 6, + Unsupported = 7, + IOError = 8, + OutOfMemory = 9, + Corrupted = 10, + Cancelled = 11, + SizeLimitExceeded = 12, }; template diff --git a/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp b/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp new file mode 100644 index 00000000..d81afe6b --- /dev/null +++ b/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp @@ -0,0 +1,205 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Core::VFS +{ + namespace + { + // Retain only a 4 KB input chunk and the UUID, even for large materials. + class VFSReadBuffer : public std::streambuf + { + public: + VFSReadBuffer(IVFSFile* file, uint64_t size) : m_file(file), m_size(size) {} + VFSError Error = VFSError::OK; + + protected: + int_type underflow() override + { + if (m_offset == m_size) + return traits_type::eof(); + const size_t count = static_cast(std::min(sizeof(m_buffer), m_size - m_offset)); + auto read = m_file->Read({reinterpret_cast(m_buffer), count}, m_offset); + if (read.Failed() || read.Value() == 0 || read.Value() > count) + { + Error = read.Failed() ? read.Error() : VFSError::IOError; + return traits_type::eof(); + } + m_offset += read.Value(); + setg(m_buffer, m_buffer, m_buffer + read.Value()); + return traits_type::to_int_type(*gptr()); + } + + private: + IVFSFile* m_file; + uint64_t m_size; + uint64_t m_offset = 0; + char m_buffer[4096]; + }; + + struct MaterialUUIDReader : nlohmann::json_sax + { + uuids::uuid UUID{}; + int Depth = 0; + bool RootObject = false; + bool UUIDKey = false; + bool null() override + { + UUIDKey = false; + return true; + } + bool boolean(bool) override + { + return null(); + } + bool number_integer(number_integer_t) override + { + return null(); + } + bool number_unsigned(number_unsigned_t) override + { + return null(); + } + bool number_float(number_float_t, const string_t&) override + { + return null(); + } + bool string(string_t& value) override + { + if (UUIDKey) + { + auto parsed = uuids::uuid::from_string(value); + UUID = parsed.has_value() ? *parsed : uuids::uuid{}; + } + UUIDKey = false; + return true; + } + bool binary(binary_t&) override + { + return null(); + } + bool start_object(std::size_t) override + { + if (Depth == 0) + RootObject = true; + ++Depth; + UUIDKey = false; + return true; + } + bool key(string_t& value) override + { + UUIDKey = Depth == 1 && value == "uuid"; + if (UUIDKey) + UUID = {}; + return true; + } + bool end_object() override + { + --Depth; + UUIDKey = false; + return true; + } + bool start_array(std::size_t) override + { + ++Depth; + UUIDKey = false; + return true; + } + bool end_array() override + { + --Depth; + UUIDKey = false; + return true; + } + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception&) override + { + return false; + } + }; + } // namespace + + VFSResult ReadEmbeddedAssetUUID(IVFSContext& ctx, const VFSPath& path) + { + const bool mesh = path.Extension().Equals(".zemesh"); + if (!mesh && !path.Extension().Equals(".zematerial")) + return VFSResult::Fail(VFSError::Unsupported); + auto open = ctx.Open(path, VFSOpenFlags::Read); + if (open.Failed()) + return VFSResult::Fail(open.Error()); + auto* file = open.Value(); + auto result = [&]() -> VFSResult { + if (mesh) + { + uint8_t header[24]; + auto read = file->ReadAll({header, sizeof(header)}); + if (read.Failed()) + return VFSResult::Fail(read.Error()); + if (read.Value() != sizeof(header)) + return VFSResult::Fail(VFSError::Corrupted); + uint32_t magic, version; + uuids::uuid id; + std::memcpy(&magic, header, 4); + std::memcpy(&version, header + 4, 4); + std::memcpy(&id, header + 8, 16); + if (magic != ZEMESH_MAGIC || version != ASSET_FILE_VERSION || id.is_nil()) + return VFSResult::Fail(VFSError::Corrupted); + return VFSResult::Ok(id); + } + auto size = file->Size(); + if (size.Failed()) + return VFSResult::Fail(size.Error()); + if (size.Value() > MATERIAL_IDENTITY_MAX_FILE_SIZE) + return VFSResult::Fail(VFSError::SizeLimitExceeded); + VFSReadBuffer buffer(file, size.Value()); + std::istream input(&buffer); + MaterialUUIDReader reader; + const bool valid = nlohmann::json::sax_parse(input, &reader); + if (buffer.Error != VFSError::OK) + return VFSResult::Fail(buffer.Error); + if (!valid || !reader.RootObject || reader.UUID.is_nil()) + return VFSResult::Fail(VFSError::Corrupted); + return VFSResult::Ok(reader.UUID); + }(); + ctx.Close(file); + return result; + } + + VFSResult WriteFileAtomically(IVFSContext& ctx, const VFSPath& path, Containers::ArrayView data) + { + if (!path.IsValid() || path.IsRoot()) + return VFSResult::Fail(VFSError::InvalidPath); + + std::random_device random; + uuids::basic_uuid_random_generator generator(random); + const auto temporary = path.Parent().Append(("." + uuids::to_string(generator()) + ".tmp").c_str()); + if (temporary.Failed()) + return VFSResult::Fail(temporary.Error()); + const auto& tmp_path = temporary.Value(); + auto fail = [&](VFSError error) { + (void) ctx.Remove(tmp_path); + return VFSResult::Fail(error); + }; + auto open = ctx.Open(tmp_path, VFSOpenFlags::Write | VFSOpenFlags::Create | VFSOpenFlags::Truncate); + if (open.Failed()) + return fail(open.Error()); + auto* file = open.Value(); + auto write = file->Write(data, 0); + auto flush = file->Flush(); + auto close = file->Close(); + ctx.Close(file); + if (write.Failed()) + return fail(write.Error()); + if (write.Value() != data.size()) + return fail(VFSError::IOError); + if (flush.Failed()) + return fail(flush.Error()); + if (close.Failed()) + return fail(close.Error()); + auto rename = ctx.Rename(tmp_path, path); + return rename.Failed() ? fail(rename.Error()) : rename; + } +} // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/VFSFileIO.h b/ZEngine/ZEngine/Core/VFS/VFSFileIO.h new file mode 100644 index 00000000..f95aa83b --- /dev/null +++ b/ZEngine/ZEngine/Core/VFS/VFSFileIO.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +namespace ZEngine::Core::VFS +{ + // Material identity extraction uses a JSON parser that may temporarily retain one + // string token. Keep that bounded without imposing a limit on mesh assets. + inline constexpr uint64_t MATERIAL_IDENTITY_MAX_FILE_SIZE = 1024ULL * 1024ULL; + + // Read only the embedded identity; material JSON is streamed without an arena or DOM. + [[nodiscard]] VFSResult ReadEmbeddedAssetUUID(IVFSContext& ctx, const VFSPath& path); + + // Publish using a unique temporary sibling on the same mount/filesystem. + [[nodiscard]] VFSResult WriteFileAtomically(IVFSContext& ctx, const VFSPath& path, Containers::ArrayView data); +} // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp b/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp index f9adf35a..7d4ea083 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp @@ -4,10 +4,6 @@ #include #include #include -#include -#include -#include -#include #include namespace ZEngine::Core::VFS @@ -23,69 +19,6 @@ namespace ZEngine::Core::VFS return true; return false; } - - // .zemesh/.zematerial embed their own UUID at cook time. Peeking it here - // lets the caller pre-seed a .meta instead of GetOrCreate minting an - // unrelated random one (#755). Duplicated rather than depending on - // Importers from Core::VFS — ZEMESH_MAGIC/ASSET_FILE_VERSION are already - // shared, low-level constants (ZEngineDef.h). - static std::optional PeekEmbeddedUUID(IVFSContext& ctx, const VFSPath& path, Managers::AssetType type) - { - if (type != Managers::AssetType::MESH && type != Managers::AssetType::MATERIAL) - return std::nullopt; - - auto open_result = ctx.Open(path, VFSOpenFlags::Read); - if (open_result.Failed()) - return std::nullopt; - IVFSFile* file = open_result.Value(); - - std::optional result; - - if (type == Managers::AssetType::MESH) - { - // Layout matches AssetCodec::AssetMeshFileHeader: uint32 magic, uint32 version, 16-byte uuid. - uint8_t buf[24]; - auto read_result = file->Read({buf, sizeof(buf)}, 0); - if (read_result.Succeeded() && read_result.Value() == sizeof(buf)) - { - uint32_t magic = 0, version = 0; - std::memcpy(&magic, buf, sizeof(magic)); - std::memcpy(&version, buf + sizeof(magic), sizeof(version)); - if (magic == ZEMESH_MAGIC && version == ASSET_FILE_VERSION) - { - uuids::uuid id; - std::memcpy(&id, buf + sizeof(magic) + sizeof(version), sizeof(id)); - if (!id.is_nil()) - result = id; - } - } - } - else // MATERIAL — JSON, top-level "uuid" field (matches AssetCodec::SerializeMaterialAssetFile) - { - static constexpr size_t kReadCap = 16384; - auto size_result = file->Size(); - if (size_result.Succeeded() && size_result.Value() < kReadCap) - { - uint8_t buf[kReadCap]; - size_t size = (size_t) size_result.Value(); - auto read_result = file->ReadAll({buf, size}); - if (read_result.Succeeded()) - { - buf[size] = '\0'; - auto j = nlohmann::json::parse(reinterpret_cast(buf), nullptr, /*allow_exceptions=*/false); - if (!j.is_discarded() && j.contains("uuid") && j["uuid"].is_string()) - { - auto parsed = uuids::uuid::from_string(j["uuid"].get()); - if (parsed.has_value() && !parsed.value().is_nil()) - result = parsed.value(); - } - } - } - } - - ctx.Close(file); - return result; - } } // namespace void VFSScanner::Initialize(Core::Memory::ArenaAllocator* page_source) @@ -189,6 +122,7 @@ namespace ZEngine::Core::VFS m_metas_created.value.store(0, std::memory_order_relaxed); m_metas_updated.value.store(0, std::memory_order_relaxed); m_metas_up_to_date.value.store(0, std::memory_order_relaxed); + m_errors.value.store(0, std::memory_order_relaxed); m_pending_tasks.value.store(0, std::memory_order_relaxed); m_is_scanning.value.store(true, std::memory_order_release); m_scan_start = std::chrono::steady_clock::now(); @@ -216,7 +150,10 @@ namespace ZEngine::Core::VFS m_dir_semaphore.release(); if (result.Failed()) + { + ReportError(dir, "list directory", result.Error()); return; + } Containers::Array& entries = result.Value(); @@ -238,45 +175,42 @@ namespace ZEngine::Core::VFS if (IsAssetExtension(entries[i].Path)) { - ZEngine::Managers::AssetType type = ZEngine::Core::VFS::AssetRegistry::InferTypeFromExtension(entries[i].Path); + ZEngine::Managers::AssetType type = ZEngine::Core::VFS::AssetRegistry::InferTypeFromExtension(entries[i].Path); + + // Never write metadata using a fabricated hash after an I/O failure. + auto hash_result = MetaFileIO::ComputeHash(*ctx.Context, entries[i].Path); + if (hash_result.Failed()) + { + ReportError(entries[i].Path, "hash asset", hash_result.Error()); + continue; + } - // Pre-seed .meta from the embedded UUID before GetOrCreate mints an - // unrelated random one (#755). - if (MetaFileIO::Read(*ctx.Context, entries[i].Path).Failed()) + // GetOrCreate reads and, if necessary, seeds the embedded identity + // under its lock, avoiding redundant metadata reads and unchecked writes. + auto meta = MetaFileIO::GetOrCreate(*ctx.Context, entries[i].Path, "VFSScanner", hash_result.Value(), /*use_embedded_identity=*/true); + if (meta.Failed()) + { + ReportError(entries[i].Path, "resolve metadata", meta.Error()); + continue; + } + switch (meta.Value().Status) { - auto embedded = PeekEmbeddedUUID(*ctx.Context, entries[i].Path, type); - if (embedded.has_value()) - { - MetaFileData seed = {}; - seed.AssetUUID = *embedded; - Helpers::secure_strncpy(seed.ImporterName, sizeof(seed.ImporterName), "VFSScanner", sizeof(seed.ImporterName) - 1); - MetaFileIO::Write(*ctx.Context, entries[i].Path, seed); - } + case ImportStatus::New: + m_metas_created.value.fetch_add(1, std::memory_order_relaxed); + break; + case ImportStatus::Stale: + m_metas_updated.value.fetch_add(1, std::memory_order_relaxed); + break; + case ImportStatus::UpToDate: + m_metas_up_to_date.value.fetch_add(1, std::memory_order_relaxed); + break; + default: + break; } - auto hash_result = MetaFileIO::ComputeHash(*ctx.Context, entries[i].Path); - auto meta = MetaFileIO::GetOrCreate(*ctx.Context, entries[i].Path, "VFSScanner", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta.Succeeded()) + if (m_registry != nullptr) { - switch (meta.Value().Status) - { - case ImportStatus::New: - m_metas_created.value.fetch_add(1, std::memory_order_relaxed); - break; - case ImportStatus::Stale: - m_metas_updated.value.fetch_add(1, std::memory_order_relaxed); - break; - case ImportStatus::UpToDate: - m_metas_up_to_date.value.fetch_add(1, std::memory_order_relaxed); - break; - default: - break; - } - - if (m_registry != nullptr) - { - m_registry->OnScanFileDiscovered(*ctx.Context, entries[i].Path, type); - } + m_registry->OnScanFileDiscovered(*ctx.Context, entries[i].Path, type); } } } @@ -285,6 +219,12 @@ namespace ZEngine::Core::VFS ctx.Cache->SetListing(dir, std::move(entries)); } + void VFSScanner::ReportError(const VFSPath& path, const char* operation, VFSError error) + { + m_errors.value.fetch_add(1, std::memory_order_relaxed); + ZENGINE_LOG_VFS_WARN("VFS scan failed to {} for '{}' (VFS error {})", operation, path.CStr(), static_cast(error)) + } + void VFSScanner::OnTaskComplete(bool cancelled) { const int32_t remaining = m_pending_tasks.value.fetch_sub(1, std::memory_order_acq_rel) - 1; @@ -302,6 +242,7 @@ namespace ZEngine::Core::VFS stats.MetasCreated = m_metas_created.value.load(std::memory_order_relaxed); stats.MetasUpdated = m_metas_updated.value.load(std::memory_order_relaxed); stats.MetasUpToDate = m_metas_up_to_date.value.load(std::memory_order_relaxed); + stats.Errors = m_errors.value.load(std::memory_order_relaxed); m_complete_callback(m_complete_callback_ctx, stats); } } diff --git a/ZEngine/ZEngine/Core/VFS/VFSScanner.h b/ZEngine/ZEngine/Core/VFS/VFSScanner.h index d5e9f6e5..10e8d383 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSScanner.h +++ b/ZEngine/ZEngine/Core/VFS/VFSScanner.h @@ -29,6 +29,7 @@ namespace ZEngine::Core::VFS uint64_t MetasCreated = 0; // .meta generated for the first time uint64_t MetasUpdated = 0; // .meta existed but source SHA changed uint64_t MetasUpToDate = 0; // .meta existed and SHA matched + uint64_t Errors = 0; // failed directory/asset operations; affected sidecars are preserved }; struct VFSScanner @@ -74,6 +75,7 @@ namespace ZEngine::Core::VFS bool TryAcquireTask(uint32_t& out_slot); void ReleaseTask(uint32_t slot); void OnTaskComplete(bool cancelled); + void ReportError(const VFSPath& path, const char* operation, VFSError error); static void RunScanTask(void* context); int AcquireSlot(); @@ -90,6 +92,7 @@ namespace ZEngine::Core::VFS PaddedAtomic m_metas_created{}; PaddedAtomic m_metas_updated{}; PaddedAtomic m_metas_up_to_date{}; + PaddedAtomic m_errors{}; std::chrono::steady_clock::time_point m_scan_start{}; std::counting_semaphore m_dir_semaphore{MaxConcurrentDirLists}; diff --git a/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h b/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h index 21994ef3..9c313b3c 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h +++ b/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h @@ -1,8 +1,14 @@ #pragma once #include +#include namespace ZEngine::Core::VFS { + [[nodiscard]] inline uint64_t VFSWatchTimestampNowNanoseconds() + { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + } + enum class WatchEventKind : uint8_t { Created = 0, @@ -18,5 +24,7 @@ namespace ZEngine::Core::VFS char OldPath[MAX_FILE_PATH_COUNT] = {}; WatchEventKind Kind = WatchEventKind::Created; bool IsDirectory = false; + // Captured by the platform watcher before VFSFileWatcher debounces the event. + uint64_t ObservedAtNanoseconds = 0; }; } // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Engine.cpp b/ZEngine/ZEngine/Engine.cpp index 07c71204..5684881a 100644 --- a/ZEngine/ZEngine/Engine.cpp +++ b/ZEngine/ZEngine/Engine.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -119,11 +121,22 @@ namespace ZEngine ZENGINE_VALIDATE_ASSERT(Logging::Logger::IsInitialized(), "Engine::Initialize: Logger not initialized — Obelisk must call Logger::Initialize first") ZENGINE_VALIDATE_ASSERT(Helpers::ThreadPoolHelper::IsInitialized(), "Engine::Initialize: ThreadPool not initialized — Obelisk must call ThreadPoolHelper::Initialize first") - auto& arena = memory->BootstrapArena; + const Core::EngineAssetRootResolution resolved_asset_root = Core::ResolveEngineAssetRoot(); + const Core::EngineAssetRootResolution validated_asset_root = resolved_asset_root.Succeeded() ? Core::ValidateEngineAssetRoot(resolved_asset_root.Root) : resolved_asset_root; + if (!validated_asset_root.Succeeded()) + { + ZENGINE_CORE_CRITICAL("Engine startup aborted: {}", validated_asset_root.Diagnostic) + ZENGINE_EXIT_FAILURE() + } + + auto& arena = memory->BootstrapArena; - g_engine_ctx = ZPushStructCtor(&arena, EngineContext); + g_engine_ctx = ZPushStructCtor(&arena, EngineContext); + const std::string engine_assets_path = validated_asset_root.Root.string(); + g_engine_ctx->EngineAssetsNativeRoot = ZPushString(&arena, engine_assets_path.size() + 1); + ZENGINE_VALIDATE_ASSERT(Helpers::secure_memcpy(g_engine_ctx->EngineAssetsNativeRoot, engine_assets_path.size() + 1, engine_assets_path.c_str(), engine_assets_path.size() + 1) == Helpers::MEMORY_OP_SUCCESS, "Engine::Initialize: failed to persist engine asset root") - auto window = ZPushStructCtor(&arena, Windows::GameWindow); + auto window = ZPushStructCtor(&arena, Windows::GameWindow); window->SetCallbackFunction(std::bind(&Applications::GameApplication::ProcessEvent, app, std::placeholders::_1)); window->Initialize(&arena, *window_cfg_ptr); g_engine_ctx->Window = window; @@ -142,13 +155,12 @@ namespace ZEngine g_engine_ctx->VFS = vfs_ctx; { - const std::string engine_dir = std::filesystem::current_path().string() + "/ZodiacEngine"; - g_engine_ctx->EngineAssetsBackend.Initialize(engine_dir.c_str(), Core::VFS::VFSBackendCaps::Read | Core::VFS::VFSBackendCaps::Write | Core::VFS::VFSBackendCaps::List, &g_engine_ctx->VFSArena); + g_engine_ctx->EngineAssetsBackend.Initialize(g_engine_ctx->EngineAssetsNativeRoot, Core::VFS::VFSBackendCaps::Read | Core::VFS::VFSBackendCaps::Write | Core::VFS::VFSBackendCaps::List, &g_engine_ctx->VFSArena); auto mount_path = Core::VFS::VFSPath::Parse("/ZodiacEngine"); - if (mount_path.Succeeded()) + if (mount_path.Failed() || vfs_ctx->Mount(&g_engine_ctx->EngineAssetsBackend, mount_path.Value(), -1).Failed()) { - auto res = vfs_ctx->Mount(&g_engine_ctx->EngineAssetsBackend, mount_path.Value(), -1); - (void) res; + ZENGINE_CORE_CRITICAL("Engine startup aborted: failed to mount engine assets at /ZodiacEngine from {}", g_engine_ctx->EngineAssetsNativeRoot) + ZENGINE_EXIT_FAILURE() } const bool has_writable_workspace = app->WorkingSpacePath && app->WorkingSpacePath[0] != '\0' && app->VFSBackend && Core::VFS::HasCap(app->VFSBackend->Capabilities(), Core::VFS::VFSBackendCaps::Write); @@ -264,6 +276,11 @@ namespace ZEngine g_engine_ctx->RenderThread.join(); } + // Import and VFS work run on this pool. Join it while the scheduler and + // every engine-owned service are still valid, so an in-flight import cannot + // publish a completion into torn-down state. + Helpers::ThreadPoolHelper::Shutdown(); + Core::MainThreadScheduler::Shutdown(); // Step 3 — ECS shutdown: ActorManager before Scene (lifecycle order) @@ -356,7 +373,13 @@ namespace ZEngine static_cast(g_engine_ctx->VFS)->Tick(); if (window->IsMinimized()) + { + // Publish a zero-size snapshot so the render thread pauses too; + // retaining the last visible extent is not sufficient on Wayland. + g_engine_ctx->App->RenderPipeline->PublishFrameState({}); + frame_cap.WaitForFrameBudget(); continue; + } // Measure raw delta float raw_dt = frame_timer.End(); @@ -426,6 +449,8 @@ namespace ZEngine Applications::RenderFrameState state = {}; g_engine_ctx->App->PrepareScene(state); + state.FramebufferW = window->GetWidth(); + state.FramebufferH = window->GetHeight(); if (state.Scene) { // Scene mutation happens on this thread. Send an immutable @@ -474,7 +499,7 @@ namespace ZEngine while (true) { - if (g_engine_ctx->RequestTerminate.value.load(std::memory_order_acquire)) + if (g_engine_ctx->RequestTerminate.value.load(std::memory_order_acquire) || g_engine_ctx->CloseRequested.value.load(std::memory_order_acquire)) { break; } @@ -514,6 +539,7 @@ namespace ZEngine applied_resize_sequence = state.ResizeSequence; } + pipeline->Device->SwapchainPtr->UpdateFramebufferExtent(state.FramebufferW, state.FramebufferH); const bool frame_valid = pipeline->BeginFrame(); if (frame_valid && state.Scene) { @@ -521,6 +547,8 @@ namespace ZEngine pipeline->RenderScene(state.Camera, state.Scene, state.Sky, state.CelestialLight, state.SkyRevision, overlay); } pipeline->EndFrame(); + if (!frame_valid) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); // Update SmoothedDeltaTime with the render thread's smoothed frame time. // End() is called after EndFrame() so the vsync wait is included in the sample. diff --git a/ZEngine/ZEngine/Engine.h b/ZEngine/ZEngine/Engine.h index c63aa880..8ccf8c2b 100644 --- a/ZEngine/ZEngine/Engine.h +++ b/ZEngine/ZEngine/Engine.h @@ -33,6 +33,7 @@ namespace ZEngine // Mounted at /ZodiacEngine with priority -1 so the workspace backend // (priority 0) takes precedence for any overlapping paths. Core::VFS::VFSDiskBackend EngineAssetsBackend = {}; + char* EngineAssetsNativeRoot = nullptr; // Writable project-owned backend mounted specifically at /cache/pso. // It resolves to /.zodiacengine/cache/pso and is distinct diff --git a/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp b/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp index bf5123eb..be055f15 100644 --- a/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp +++ b/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp @@ -389,64 +389,6 @@ namespace ZEngine::Hardwares ZENGINE_CORE_ERROR("[!] Enqueued Command Buffer overflow detected") } - void CommandBufferManager::IncreaseBuffers() - { - // TotalPoolCount = Device->SwapchainImageCount * TotalThreadCount; - // TotalCommandBufferCount = TotalPoolCount * MaxBufferPerPool; - - // if (TotalCommandBufferCount > EnqueuedCommandBuffers.size()) - // { - // auto size = EnqueuedCommandBuffers.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // EnqueuedCommandBuffers.push(nullptr); - // } - // } - - // if (TotalPoolCount > CommandPools.size()) - // { - // auto size = CommandPools.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // CommandPools.push(ZPushStructCtorArgs(Device->Arena, Rendering::Pools::CommandPool, Device, Rendering::QueueType::GRAPHIC_QUEUE)); - // } - // } - - // if (TotalCommandBufferCount > CommandBuffers.size()) - // { - // auto size = CommandBuffers.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // int pool_index = GetPoolFromIndex(Rendering::QueueType::GRAPHIC_QUEUE, i); - // auto& pool = CommandPools[pool_index]; - // CommandBuffers.push(ZPushStructCtorArgs( - // Device->Arena, - // CommandBuffer, - // Device, - // pool->Handle, - // pool->QueueType, - // /*(i % MaxBufferPerPool) == 0 ? false : true */ false)); - // } - // } - - // if (Device->HasSeperateTransfertQueueFamily) - // { - // auto size = TransferCommandPools.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // TransferCommandPools.push(ZPushStructCtorArgs(Device->Arena, Rendering::Pools::CommandPool, Device, Rendering::QueueType::TRANSFER_QUEUE)); - // } - - // size = TransferCommandBuffers.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // int pool_index = GetPoolFromIndex(Rendering::QueueType::TRANSFER_QUEUE, i); - // auto& pool = TransferCommandPools[pool_index]; - // TransferCommandBuffers.push(ZPushStructCtorArgs(Device->Arena, CommandBuffer, Device, pool->Handle, pool->QueueType, true)); - // } - // } - } - bool CommandBufferManager::IsInitialized() const { return m_is_initialized; diff --git a/ZEngine/ZEngine/Hardwares/CommandBufferManager.h b/ZEngine/ZEngine/Hardwares/CommandBufferManager.h index 6116fe98..88cd5a75 100644 --- a/ZEngine/ZEngine/Hardwares/CommandBufferManager.h +++ b/ZEngine/ZEngine/Hardwares/CommandBufferManager.h @@ -42,7 +42,6 @@ namespace ZEngine::Hardwares /// @brief Returns the instant-submit command pool for a frame, thread, and queue role. Rendering::Pools::CommandPool* GetInstantCommandPool(Rendering::QueueType type, uint8_t frame_index, uint8_t thread_index); void ResetPool(uint8_t frame_index, uint8_t thread_index); - void IncreaseBuffers(); void EnqueueBuffer(CommandBufferPtr const buffer); void EndEnqueuedBuffers(); void ResetEnqueuedBufferIndex(); diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index a939b60a..011b841f 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -1,4 +1,5 @@ +#include #include #include #include @@ -6,6 +7,7 @@ #include #include #include +#include using namespace ZEngine::Core::Containers; using namespace ZEngine::Rendering; @@ -19,6 +21,7 @@ namespace ZEngine::Hardwares device->Arena->CreateSubArena(ZMega(3), &Arena, "VulkanDevice/Swapchain"); Device = device; + FramebufferExtent = {Device->CurrentWindow->GetWidth(), Device->CurrentWindow->GetHeight()}; BufferredFrameCount = buffered_frame_size; FrameContextPoolSize = BufferredFrameCount * FrameContextPoolSizeFactor; @@ -53,25 +56,42 @@ namespace ZEngine::Hardwares Create(); } + void DeviceSwapchain::UpdateFramebufferExtent(uint32_t width, uint32_t height) + { + if (FramebufferExtent.width == width && FramebufferExtent.height == height) + return; + + FramebufferExtent = {width, height}; + // Wayland may keep returning VK_SUCCESS after a resize, so do not rely + // exclusively on OUT_OF_DATE/SUBOPTIMAL to recreate the swapchain. + Recreation = RecreationState::Pending; + } + + VkExtent2D DeviceSwapchain::ResolveExtent(const VkSurfaceCapabilitiesKHR& capabilities, VkExtent2D framebuffer_extent) + { + if (framebuffer_extent.width == 0 || framebuffer_extent.height == 0) + return {}; + if (capabilities.currentExtent.width != std::numeric_limits::max()) + return capabilities.currentExtent; + + return { + std::clamp(framebuffer_extent.width, capabilities.minImageExtent.width, capabilities.maxImageExtent.width), + std::clamp(framebuffer_extent.height, capabilities.minImageExtent.height, capabilities.maxImageExtent.height), + }; + } + void DeviceSwapchain::Create() { VkSurfaceCapabilitiesKHR capabilities{}; vkGetPhysicalDeviceSurfaceCapabilitiesKHR(Device->PhysicalDevice, Device->Surface, &capabilities); - if (capabilities.currentExtent.width != std::numeric_limits::max()) - { - SwapchainImageWidth = capabilities.currentExtent.width; - SwapchainImageHeight = capabilities.currentExtent.height; - } - else - { - // Surface does not report a concrete extent (Wayland, headless) — ask the window. - SwapchainImageWidth = Device->CurrentWindow->GetWidth(); - SwapchainImageHeight = Device->CurrentWindow->GetHeight(); - } + const VkExtent2D extent = ResolveExtent(capabilities, FramebufferExtent); + SwapchainImageWidth = extent.width; + SwapchainImageHeight = extent.height; // {0,0} extent is a spec violation; destroy stale swapchain and retry next frame. if (SwapchainImageWidth == 0 || SwapchainImageHeight == 0) { + Recreation = RecreationState::Pending; if (SwapchainHandle != VK_NULL_HANDLE) { vkDestroySwapchainKHR(Device->LogicalDevice, SwapchainHandle, nullptr); @@ -185,6 +205,8 @@ namespace ZEngine::Hardwares if (previous_image_count != SwapchainImageCount) ++SwapchainImageCountChangeCount; + ZENGINE_CORE_INFO("Swapchain created: {}x{} (framebuffer={}x{})", SwapchainImageWidth, SwapchainImageHeight, FramebufferExtent.width, FramebufferExtent.height) + if (old_swapchain != VK_NULL_HANDLE) { ZENGINE_DESTROY_VULKAN_HANDLE(Device->LogicalDevice, vkDestroySwapchainKHR, old_swapchain, nullptr) @@ -224,6 +246,9 @@ namespace ZEngine::Hardwares void DeviceSwapchain::Dispose() { + DirectGraphicsTimeline = nullptr; + DirectGraphicsTimelineValue = 0; + // Destroy frame-context and swapchain-image Vulkan objects. // All destructors use Device->DeferFree — the second PendingFree.Drain() // at the end of VulkanDevice::Deinitialize() drains them before vkDestroyDevice. @@ -252,13 +277,48 @@ namespace ZEngine::Hardwares void DeviceSwapchain::AcquireNextImage(uint32_t frame_context_idx) { + // Do not reset fences or command pools while a minimized/zero-size + // window cannot acquire an image. Restore recreates through the normal path. + if (FramebufferExtent.width == 0 || FramebufferExtent.height == 0) + { + CurrentFrame = nullptr; + Recreation = RecreationState::Pending; + return; + } +#ifndef NDEBUG + auto trace_fence_wait = [this](cstring reason, const FrameContext& frame) { + if (!TraceSubmission) + return; + + uint64_t render_completed = 0; + vkGetSemaphoreCounterValue(Device->LogicalDevice, RenderTimeline->GetHandle(), &render_completed); + uint64_t direct_completed = 0; + if (DirectGraphicsTimeline) + vkGetSemaphoreCounterValue(Device->LogicalDevice, DirectGraphicsTimeline->GetHandle(), &direct_completed); + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire wait={} frame={} render_completed={} render_submitted={} direct_timeline={} direct_completed={} direct_submitted={}", reason, frame.Index, render_completed, RenderTimelineNextValue, static_cast(DirectGraphicsTimeline ? DirectGraphicsTimeline->GetHandle() : VK_NULL_HANDLE), direct_completed, DirectGraphicsTimelineValue) + }; +#endif if (Recreation != RecreationState::None) { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation begin state={}", static_cast(Recreation)) +#endif // Every submitted render command buffer is covered by its frame fence. for (uint32_t i = 0; i < ImageInFlights.size(); ++i) { if (ImageInFlights[i] != nullptr) + { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation image-fence wait begin image={}", i) +#endif ImageInFlights[i]->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation image-fence wait complete image={}", i) +#endif + } } // Defense-in-depth: ImageInFlights is sized SwapchainImageCount, @@ -271,11 +331,36 @@ namespace ZEngine::Hardwares { if (FrameContexts[i].Fence->GetState() == Rendering::Primitives::FenceState::Submitted) { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation frame-fence wait begin frame={}", i) +#endif FrameContexts[i].Fence->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation frame-fence wait complete frame={}", i) +#endif FrameContexts[i].Fence->Reset(); } } + // Frame fences do not cover independently submitted uploads. Drain every + // queue before clearing their frame waits or resetting texture timeline + // counters; this is especially important after an OUT_OF_DATE acquire, + // where Present() deliberately submits no fence-backed work. + if (!Device->IsDeviceLost.load(std::memory_order_acquire)) + { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation queue-wait-all begin") +#endif + Device->QueueWaitAll(); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation queue-wait-all complete") +#endif + } + for (int i = 0; i < FrameContextPoolSizeFactor; ++i) { FrameContexts[i + FrameContextOffset].Acquired->SetState(Primitives::SemaphoreState::Idle); @@ -319,47 +404,94 @@ namespace ZEngine::Hardwares Recreation = RecreationState::None; ZENGINE_CORE_WARN("Swapchain recreated: {}x{}", SwapchainImageWidth, SwapchainImageHeight) +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation complete") +#endif + if (OnSwapchainResized) OnSwapchainResized(SwapchainImageWidth, SwapchainImageHeight, OnSwapchainResizedCtx); } FrameContext& frame = FrameContexts[frame_context_idx + FrameContextOffset]; if (frame.Fence->GetState() == Rendering::Primitives::FenceState::Submitted) + { +#ifndef NDEBUG + trace_fence_wait("frame", frame); +#endif frame.Fence->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire frame-fence wait complete frame={}", frame.Index) +#endif + } + + // A reset fence must not remain associated with any swapchain image. + // Besides the normal completed submission, this clears a stale reference + // left by an aborted frame before it can become an unsignaled self-wait. + ClearImageInFlightReferences(frame.Fence, frame.Index); frame.Fence->Reset(); frame.Acquired->SetState(Primitives::SemaphoreState::Idle); - uint32_t image_idx = 0; - VkResult acquire_image_result = vkAcquireNextImageKHR(Device->LogicalDevice, SwapchainHandle, UINT64_MAX, frame.Acquired->GetHandle(), VK_NULL_HANDLE, &image_idx); - frame.Acquired->SetState(Primitives::SemaphoreState::Submitted); - Device->TickMemory(); - - if (acquire_image_result == VK_ERROR_OUT_OF_DATE_KHR) + uint32_t image_idx = 0; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire call frame={} semaphore={}", frame.Index, static_cast(frame.Acquired->GetHandle())) +#endif + VkResult acquire_image_result = vkAcquireNextImageKHR(Device->LogicalDevice, SwapchainHandle, ImageAcquireTimeoutNs, frame.Acquired->GetHandle(), VK_NULL_HANDLE, &image_idx); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire returned frame={} result={} image={}", frame.Index, static_cast(acquire_image_result), image_idx) +#endif + if (!ApplyAcquireResult(frame, acquire_image_result, image_idx)) { - // Semaphore not signalled (spec) — image_idx invalid, skip all GPU work. - frame.ImageIndex = std::numeric_limits::max(); - CurrentFrame = &frame; - Recreation = RecreationState::FrameAborted; + // Timeout/not-ready signal no semaphore and need no recreation. The + // next render-loop iteration can consume a resize/minimize/close request. + if (acquire_image_result != VK_TIMEOUT && acquire_image_result != VK_NOT_READY && acquire_image_result != VK_ERROR_OUT_OF_DATE_KHR && !Device->CheckDeviceLost(acquire_image_result, "AcquireNextImage")) + { + ZENGINE_CORE_ERROR("vkAcquireNextImageKHR did not yield a valid image: result={} image={} image_count={}", static_cast(acquire_image_result), image_idx, SwapchainImageCount) + Engine::RequestClose(); + } return; } + frame.Acquired->SetState(Primitives::SemaphoreState::Submitted); - if (Device->CheckDeviceLost(acquire_image_result, "AcquireNextImage")) + auto* const image_fence = ImageInFlights[image_idx]; + const bool current_frame_owns_image = image_fence == frame.Fence; + ZENGINE_VALIDATE_ASSERT(!current_frame_owns_image, "An acquired image cannot wait on the frame fence being reset") + if (!current_frame_owns_image && image_fence != nullptr && !image_fence->IsSignaled()) { - frame.ImageIndex = std::numeric_limits::max(); - CurrentFrame = &frame; - Recreation = RecreationState::FrameAborted; - return; +#ifndef NDEBUG + trace_fence_wait("image", frame); +#endif + image_fence->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire image-fence wait complete frame={} image={}", frame.Index, image_idx) +#endif } - if (ImageInFlights[image_idx] != nullptr && !ImageInFlights[image_idx]->IsSignaled()) - ImageInFlights[image_idx]->Wait(UINT64_MAX); - RenderCompletes[image_idx]->SetState(Rendering::Primitives::SemaphoreState::Idle); + // Tick only after a valid acquisition and the image fence wait. On a + // failed acquire (especially DEVICE_LOST), no further Vulkan work is safe. + Device->TickMemory(); + } + + bool DeviceSwapchain::ApplyAcquireResult(FrameContext& frame, VkResult result, uint32_t image_index) + { + CurrentFrame = &frame; + frame.ImageIndex = std::numeric_limits::max(); + if ((result == VK_SUCCESS || result == VK_SUBOPTIMAL_KHR) && image_index < SwapchainImageCount) + { + frame.ImageIndex = image_index; + if (result == VK_SUBOPTIMAL_KHR) + Recreation = RecreationState::Pending; + return true; + } - ImageInFlights[image_idx] = frame.Fence; - frame.ImageIndex = image_idx; - CurrentFrame = &frame; - // SUBOPTIMAL: image is valid; Present() schedules recreation after vkQueuePresentKHR. + if (result != VK_TIMEOUT && result != VK_NOT_READY) + Recreation = RecreationState::FrameAborted; + return false; } void DeviceSwapchain::CollectAsyncGPUOperations() @@ -383,9 +515,9 @@ namespace ZEngine::Hardwares RenderWorkSubmittedCallbacks.clear(); }; - if (Recreation == RecreationState::FrameAborted) + if (!IsFrameValid()) { - // OOD at acquire: semaphore not signalled, no GPU work submitted. + // Failed/zero-size acquire: no GPU work was recorded for this frame. IdleFrameCount.value.fetch_add(1, std::memory_order_acq_rel); Device->CommandBufferMgr->ResetEnqueuedBufferIndex(); discard_submission_callbacks(); @@ -448,7 +580,19 @@ namespace ZEngine::Hardwares if (render_complete->GetState() == Rendering::Primitives::SemaphoreState::Submitted) render_complete->SetState(Rendering::Primitives::SemaphoreState::Idle); if (CurrentFrame->Fence->GetState() == Rendering::Primitives::FenceState::Submitted) + { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-fence wait begin frame={}", CurrentFrame->Index) +#endif CurrentFrame->Fence->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-fence wait complete frame={}", CurrentFrame->Index) +#endif + ClearImageInFlightReferences(CurrentFrame->Fence, CurrentFrame->Index); + CurrentFrame->Fence->Reset(); + } QueueView queue = Device->GetQueue(Rendering::QueueType::GRAPHIC_QUEUE); @@ -504,7 +648,21 @@ namespace ZEngine::Hardwares }); } - uint64_t work_complete_value = ++RenderTimelineNextValue; + uint64_t work_complete_value = ++RenderTimelineNextValue; + +#ifndef NDEBUG + if (TraceSubmission) + { + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame={} image={} command_buffers={} waits={} signal_render_value={}", CurrentFrame->Index, CurrentFrame->ImageIndex, cmd_infos.size(), wait_sem_infos.size(), work_complete_value) + for (uint32_t wait_index = 0; wait_index < wait_sem_infos.size(); ++wait_index) + { + const VkSemaphoreSubmitInfo& wait = wait_sem_infos[wait_index]; + uint64_t completed = 0; + const VkResult result = wait.value == 0 ? VK_SUCCESS : vkGetSemaphoreCounterValue(Device->LogicalDevice, wait.semaphore, &completed); + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present wait={} semaphore={} value={} completed={} query_result={} stages={}", wait_index, static_cast(wait.semaphore), wait.value, completed, static_cast(result), static_cast(wait.stageMask)) + } + } +#endif VkSemaphoreSubmitInfo work_complete_signal = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, @@ -534,9 +692,25 @@ namespace ZEngine::Hardwares .pSignalSemaphoreInfos = work_signal_infos, }; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-heap flush begin frame={}", CurrentFrame->Index) +#endif Device->FrameHeaps[CurrentFrame->Index].Flush(&Device->GpuMem); - +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-heap flush complete frame={}", CurrentFrame->Index) +#endif + +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present submit begin frame={} render_value={} queue={}", CurrentFrame->Index, work_complete_value, static_cast(queue.Handle)) +#endif auto submit = vkQueueSubmit2(queue.Handle, 1, &submit_info_1, CurrentFrame->Fence->GetHandle()); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present submit returned frame={} result={}", CurrentFrame->Index, static_cast(submit)) +#endif if (Device->CheckDeviceLost(submit, "Present: render work submit")) { ZReleaseScratch(scratch); @@ -545,6 +719,18 @@ namespace ZEngine::Hardwares } ZENGINE_VALIDATE_ASSERT(submit == VK_SUCCESS, "Failed to submit queue") + // Associate an image only with a fence that Vulkan accepted for + // submission. Aborted frames therefore leave no unsignaled association. + CurrentFrame->Fence->SetState(Rendering::Primitives::FenceState::Submitted); + ImageInFlights[CurrentFrame->ImageIndex] = CurrentFrame->Fence; + +#ifndef NDEBUG + if (TraceSubmission) + { + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present accepted frame={} render_value={}", CurrentFrame->Index, work_complete_value) + } +#endif + // The graphics command buffers are now owned by Vulkan. Deliver callbacks // before presentation: a later WSI error cannot undo this submission. for (const RenderWorkSubmissionCallback& callback : RenderWorkSubmittedCallbacks) @@ -555,7 +741,6 @@ namespace ZEngine::Hardwares ZReleaseScratch(scratch); Device->CommandBufferMgr->ResetEnqueuedBufferIndex(); - CurrentFrame->Fence->SetState(Rendering::Primitives::FenceState::Submitted); render_complete->SetState(Rendering::Primitives::SemaphoreState::Submitted); @@ -571,7 +756,15 @@ namespace ZEngine::Hardwares .pSwapchains = swapchains, .pImageIndices = frames, }; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present WSI begin frame={} image={} queue={}", CurrentFrame->Index, CurrentFrame->ImageIndex, static_cast(queue.Handle)) +#endif VkResult present_result = vkQueuePresentKHR(queue.Handle, &present_info); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present WSI returned frame={} image={} result={}", CurrentFrame->Index, CurrentFrame->ImageIndex, static_cast(present_result)) +#endif IdleFrameCount.value.fetch_add(1, std::memory_order_acq_rel); @@ -600,4 +793,18 @@ namespace ZEngine::Hardwares if (fn) RenderWorkSubmittedCallbacks.push({.Function = fn, .Cancel = cancel_fn, .Context = context}); } + + void DeviceSwapchain::ClearImageInFlightReferences(Rendering::Primitives::Fence* fence, uint32_t frame_index) + { + for (uint32_t image_index = 0; image_index < ImageInFlights.size(); ++image_index) + { + if (ImageInFlights[image_index] != fence) + continue; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] cleared frame-fence reference frame={} image={}", frame_index, image_index) +#endif + ImageInFlights[image_index] = nullptr; + } + } } // namespace ZEngine::Hardwares diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h index 91595ec8..a5330723 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h @@ -13,8 +13,8 @@ namespace ZEngine::Hardwares struct VulkanDevice; // None: normal. Pending: recreate at start of next AcquireNextImage (set by - // SUBOPTIMAL/OOD at present or zero-size surface). FrameAborted: OOD at acquire — - // semaphore not signalled, no GPU work submitted, BeginFrame returns false. + // framebuffer resize, SUBOPTIMAL at acquire/present, OOD at present or zero-size + // surface). FrameAborted: failed acquire — no image, no GPU work submitted. enum class RecreationState : uint8_t { None = 0, @@ -57,6 +57,10 @@ namespace ZEngine::Hardwares struct DeviceSwapchain { + // Leave the render thread able to consume resize/minimize/close requests + // even when the compositor stops making presentation progress. + static constexpr uint64_t ImageAcquireTimeoutNs = 100'000'000; + Core::Memory::ArenaAllocator Arena = {}; VulkanDevice* Device = nullptr; RecreationState Recreation = RecreationState::None; @@ -69,6 +73,8 @@ namespace ZEngine::Hardwares uint32_t SwapchainImageWidth = std::numeric_limits::max(); uint32_t SwapchainImageHeight = std::numeric_limits::max(); + // Render-thread-owned copy of the main thread's physical framebuffer size. + VkExtent2D FramebufferExtent = {}; uint32_t FrameContextOffset = 0; uint32_t FrameContextPoolSize = 0; const uint32_t FrameContextPoolSizeFactor = 4; @@ -78,12 +84,20 @@ namespace ZEngine::Hardwares VkSwapchainKHR SwapchainHandle = VK_NULL_HANDLE; FrameContextPtr CurrentFrame = nullptr; Rendering::Primitives::Semaphore* RenderTimeline = nullptr; + // Latest graphics work submitted directly by RenderGraph rather than through + // Present(). A later upload must wait on this semaphore directly: relaying it + // through RenderTimeline preserves execution order but loses the draw access + // scope needed for a subsequent buffer write. + Rendering::Primitives::Semaphore* DirectGraphicsTimeline = nullptr; + uint64_t DirectGraphicsTimelineValue = 0; Rendering::Renderers::RenderPasses::Attachment* SwapchainAttachment = nullptr; Core::Containers::Array FrameContexts = {}; Core::Containers::Array SwapchainImages = {}; Core::Containers::Array SwapchainImageViews = {}; Core::Containers::Array SwapchainFramebuffers = {}; Core::Containers::Array SwapchainImageLayouts = {}; + // Fence from the accepted graphics submission that last rendered each + // swapchain image. Reset fences are never retained here. Core::Containers::Array ImageInFlights = {}; Core::Containers::Array RenderCompletes = {}; // Render-thread-owned snapshot of asynchronous GPU work relevant to the @@ -93,25 +107,33 @@ namespace ZEngine::Hardwares // vkQueueSubmit2 has accepted this frame's graphics command buffers. Core::Containers::Array RenderWorkSubmittedCallbacks = {}; - // Returns false when the frame was aborted (OUT_OF_DATE at acquire or + // Debug-only, opt-in render-graph diagnostic state. It is reset at the + // beginning of every frame and set only for a traced graph submission. + bool TraceSubmission = false; + + // Returns false when no image was acquired (timeout, failed acquire, or // zero-size surface). Callers must skip all rendering work for that frame. bool IsFrameValid() const { return Recreation != RecreationState::FrameAborted && CurrentFrame != nullptr && CurrentFrame->ImageIndex != std::numeric_limits::max(); } - void Initialize(VulkanDevice* const device, uint32_t buffered_frame_size); - void Create(); - void Clear(); - void Dispose(); + void Initialize(VulkanDevice* const device, uint32_t buffered_frame_size); + void UpdateFramebufferExtent(uint32_t width, uint32_t height); + static VkExtent2D ResolveExtent(const VkSurfaceCapabilitiesKHR& capabilities, VkExtent2D framebuffer_extent); + void Create(); + void Clear(); + void Dispose(); - void AcquireNextImage(uint32_t frame_context_idx); - void CollectAsyncGPUOperations(); + void AcquireNextImage(uint32_t frame_context_idx); + /// @brief Updates the frame's acquired-image state; only success/suboptimal have an image. + bool ApplyAcquireResult(FrameContext& frame, VkResult result, uint32_t image_index); + void CollectAsyncGPUOperations(); /// @brief Delivers `fn` after this frame's graphics work has been accepted by Vulkan. /// @details The caller owns `context` until delivery or cancellation. `cancel_fn` /// releases any CPU-side frame state when the graphics submission is rejected. - void EnqueueRenderWorkSubmittedCallback(RenderWorkSubmittedFn fn, void* context, RenderWorkCancelledFn cancel_fn = nullptr); - void Present(); + void EnqueueRenderWorkSubmittedCallback(RenderWorkSubmittedFn fn, void* context, RenderWorkCancelledFn cancel_fn = nullptr); + void Present(); #if !defined(NDEBUG) // Test-only: inject a recreation state without going through the Vulkan @@ -122,6 +144,9 @@ namespace ZEngine::Hardwares Recreation = state; } #endif + + private: + void ClearImageInFlightReferences(Rendering::Primitives::Fence* fence, uint32_t frame_index); }; ZDEFINE_PTR(DeviceSwapchain); } // namespace ZEngine::Hardwares diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp index e8f33d60..a74870fd 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp @@ -517,38 +517,31 @@ namespace ZEngine::Hardwares if (ComputeFamilyIndex == invalid_queue_family) ComputeFamilyIndex = GraphicFamilyIndex; - HasSeperateTransfertQueueFamily = GraphicFamilyIndex != TransferFamilyIndex; - HasSeparateComputeQueueFamily = GraphicFamilyIndex != ComputeFamilyIndex; - HasSeparateTransferQueue = HasSeperateTransfertQueueFamily; - HasSeparateComputeQueue = HasSeparateComputeQueueFamily; - GraphicQueueIndex = 0; - TransferQueueIndex = 0; - ComputeQueueIndex = 0; - - // A family can expose multiple distinct queue handles. Prefer a dedicated - // family, then consume additional graphics-family queues for compute and - // transfer work without introducing a queue-family ownership transfer. - uint32_t next_graphics_queue_index = 1; - const uint32_t graphics_queue_count = physical_device_queue_family_collection[GraphicFamilyIndex].queueCount; - if (!HasSeparateComputeQueue && ComputeFamilyIndex == GraphicFamilyIndex && next_graphics_queue_index < graphics_queue_count) - { - ComputeQueueIndex = next_graphics_queue_index++; - HasSeparateComputeQueue = true; - } - if (!HasSeparateTransferQueue && TransferFamilyIndex == GraphicFamilyIndex && next_graphics_queue_index < graphics_queue_count) - { - TransferQueueIndex = next_graphics_queue_index++; - HasSeparateTransferQueue = true; + GraphicQueueIndex = 0; + TransferQueueIndex = 0; + ComputeQueueIndex = 0; + + // Do not turn additional handles from the graphics/present family into + // asynchronous lanes. They have no dedicated capabilities, and treating them + // as separate queues complicates graph synchronization without proving that + // the hardware can overlap their work. A universal graphics/compute/transfer + // family therefore always uses one graphics queue. + // + // Compute and transfer can share one non-graphics family. Use two handles + // only when the family actually exposes two queues. With one handle, retain + // the compute lane and route transfer work through graphics instead of + // modelling one VkQueue as two independent scheduler lanes. + if (ComputeFamilyIndex == TransferFamilyIndex && ComputeFamilyIndex != GraphicFamilyIndex) + { + if (physical_device_queue_family_collection[ComputeFamilyIndex].queueCount > 1) + TransferQueueIndex = 1; + else + TransferFamilyIndex = GraphicFamilyIndex; } - // Some devices expose one non-graphics family for both compute and transfer. - // Request separate handles when that family has enough queues; otherwise both - // roles intentionally serialize on its single queue. - if (ComputeFamilyIndex == TransferFamilyIndex && ComputeFamilyIndex != GraphicFamilyIndex && physical_device_queue_family_collection[ComputeFamilyIndex].queueCount > 1) - { - ComputeQueueIndex = 0; - TransferQueueIndex = 1; - } + HasSeperateTransfertQueueFamily = GraphicFamilyIndex != TransferFamilyIndex; + HasSeparateTransferQueue = HasSeperateTransfertQueueFamily; + HasSeparateComputeQueue = GraphicFamilyIndex != ComputeFamilyIndex; QueueTimestampValidBits[static_cast(QueueType::GRAPHIC_QUEUE)] = physical_device_queue_family_collection[GraphicFamilyIndex].timestampValidBits; QueueTimestampValidBits[static_cast(QueueType::TRANSFER_QUEUE)] = physical_device_queue_family_collection[TransferFamilyIndex].timestampValidBits; @@ -716,6 +709,8 @@ namespace ZEngine::Hardwares m_queue_map.insert(Rendering::QueueType::COMPUTE_QUEUE, std::move(compute_queue)); } + ZENGINE_CORE_INFO("[GPU] Queue topology: graphics={}:{}; compute={}:{} (async={}); transfer={}:{} (async={})", GraphicFamilyIndex, GraphicQueueIndex, ComputeFamilyIndex, ComputeQueueIndex, HasSeparateComputeQueue, TransferFamilyIndex, TransferQueueIndex, HasSeparateTransferQueue) + /* Surface format selection */ uint32_t format_count = 0; Array surface_formats = {}; @@ -1266,9 +1261,28 @@ namespace ZEngine::Hardwares void VulkanDevice::QueueWaitAll() { - QueueWait(Rendering::QueueType::TRANSFER_QUEUE); - QueueWait(Rendering::QueueType::COMPUTE_QUEUE); - QueueWait(Rendering::QueueType::GRAPHIC_QUEUE); + constexpr QueueType queue_types[] = {QueueType::TRANSFER_QUEUE, QueueType::COMPUTE_QUEUE, QueueType::GRAPHIC_QUEUE}; + VkQueue waited_queues[3] = {}; + uint32_t waited_count = 0; + + for (QueueType type : queue_types) + { + const VkQueue queue = GetQueue(type).Handle; + bool already_waited = false; + for (uint32_t index = 0; index < waited_count; ++index) + { + if (waited_queues[index] == queue) + { + already_waited = true; + break; + } + } + if (already_waited) + continue; + + ZENGINE_VALIDATE_ASSERT(vkQueueWaitIdle(queue) == VK_SUCCESS, "Failed to wait on queue") + waited_queues[waited_count++] = queue; + } } void VulkanDevice::BeginDebugLabel(VkCommandBuffer command_buffer, cstring name) const diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.h b/ZEngine/ZEngine/Hardwares/VulkanDevice.h index b4aeda7c..3f22ff96 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.h +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.h @@ -1,6 +1,8 @@ #pragma once #include +struct RRMUploadBatchTestHelper; + namespace ZEngine::Rendering::Renderers::Pipelines { struct IPipeline; @@ -294,7 +296,6 @@ namespace ZEngine::Hardwares struct VulkanDevice { bool HasSeperateTransfertQueueFamily = false; - bool HasSeparateComputeQueueFamily = false; /// @brief True when transfer work has a distinct VkQueue handle. bool HasSeparateTransferQueue = false; /// @brief True when compute work has a distinct VkQueue handle. @@ -478,6 +479,7 @@ namespace ZEngine::Hardwares private: friend struct CommandBuffer; + friend struct ::RRMUploadBatchTestHelper; VulkanLayer m_layer = {}; Core::Containers::UnorderedHashMap m_queue_map = {}; diff --git a/ZEngine/ZEngine/Importers/AssetCodec.cpp b/ZEngine/ZEngine/Importers/AssetCodec.cpp index 1955c3ae..1d194a77 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.cpp +++ b/ZEngine/ZEngine/Importers/AssetCodec.cpp @@ -1,23 +1,29 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include #include #include #include #include +#include using namespace uuids; using namespace ZEngine::Helpers; using namespace ZEngine::Core::Containers; using namespace ZEngine::Core::Maths; +using ZEngine::Core::VFS::VFSError; using ZEngine::Core::VFS::VFSPath; +using ZEngine::Core::VFS::VFSResult; namespace ZEngine::Importers::AssetCodec { @@ -34,44 +40,146 @@ namespace ZEngine::Importers::AssetCodec } } // namespace - // Write data atomically via VFS: open .tmp, write, flush, close, rename to out_path. - static bool WriteVFS(Core::VFS::IVFSContext* vfs, const VFSPath& out_path, const std::string& data) + VFSResult MakeOutputPath(const char* directory, const char* filename) { - char tmp_buf[MAX_FILE_PATH_COUNT] = {}; - const char* raw = out_path.CStr(); - size_t raw_len = secure_strlen(raw); - size_t copy_len = raw_len < MAX_FILE_PATH_COUNT - 5 ? raw_len : MAX_FILE_PATH_COUNT - 5; - secure_memcpy(tmp_buf, MAX_FILE_PATH_COUNT, raw, copy_len); - const char tmp_suffix[] = ".tmp"; - secure_memcpy(tmp_buf + copy_len, MAX_FILE_PATH_COUNT - copy_len, tmp_suffix, sizeof(tmp_suffix)); - auto tmp_path = VFSPath::Parse(tmp_buf).Value(); + if (!filename || !filename[0]) + return VFSResult::Fail(VFSError::InvalidPath); + auto dir = VFSPath::Parse(directory); + if (dir.Failed()) + return VFSResult::Fail(dir.Error()); + auto path = dir.Value().Append(filename); + if (path.Failed()) + return path; + if (path.Value().Parent() != dir.Value()) + return VFSResult::Fail(VFSError::InvalidPath); + // Cooked assets also receive a .meta sidecar. + constexpr size_t metadata_suffix_size = sizeof(".meta") - 1; + if (path.Value().Length() + metadata_suffix_size >= MAX_FILE_PATH_COUNT) + return VFSResult::Fail(VFSError::InvalidPath); + // Account for the unique temporary sibling, even when the destination + // has a shorter name. Check without generating a UUID or touching VFS. + auto temporary = dir.Value().Append(".00000000-0000-0000-0000-000000000000.tmp"); + if (temporary.Failed()) + return temporary; + return path; + } - auto open_result = vfs->Open(tmp_path, Core::VFS::VFSOpenFlags::Write | Core::VFS::VFSOpenFlags::Create | Core::VFS::VFSOpenFlags::Truncate); - if (open_result.Failed()) - return false; + VFSResult ValidateImportConfiguration(const ImportConfiguration& config) + { + if (!config.VFS) + return VFSResult::Fail(VFSError::Unsupported); + auto mesh_path = MakeOutputPath(config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str()); + if (mesh_path.Failed()) + return mesh_path; + if (config.Options.ImportMaterials) + { + auto materials = MakeOutputPath(config.OutputMaterialPath.c_str(), config.AssetName.c_str()); + if (materials.Failed()) + return materials; + if (config.Options.ImportTextures) + { + auto textures = MakeOutputPath(config.OutputTextureFilesPath.c_str(), config.AssetName.c_str()); + if (textures.Failed()) + return textures; + } + } + return mesh_path; + } - Core::VFS::IVFSFile* file = open_result.Value(); - const auto* bytes = reinterpret_cast(data.data()); - auto w = file->Write({bytes, data.size()}, 0); - auto flush = file->Flush(); - file->Close(); - vfs->Close(file); + static VFSResult PrepareOutputPath(Core::VFS::IVFSContext* vfs, VFSResult path) + { + if (!vfs) + return VFSResult::Fail(VFSError::Unsupported); + if (path.Failed()) + return path; + const auto dir = path.Value().Parent(); + auto create = vfs->CreateDir(dir); + if (create.Failed() && create.Error() != VFSError::AlreadyExists) + return VFSResult::Fail(create.Error()); + return path; + } - if (w.Failed() || flush.Failed()) - return false; + void ReportPublishedArtifact(const ImportConfiguration& config, AssetFileType type, const VFSPath& path) + { + if (config.OnArtifactPublished && path.IsValid()) + config.OnArtifactPublished(config.ArtifactContext, type, path.CStr()); + } + + VFSResult CopyTextureFiles(ArrayView textures, const ImportConfiguration& config) + { + if (textures.size() == 0) + return VFSResult::Ok(); + if (!config.VFS) + return VFSResult::Fail(VFSError::Unsupported); + auto directory = MakeOutputPath(config.OutputTextureFilesPath.c_str(), config.AssetName.c_str()); + if (directory.Failed()) + return VFSResult::Fail(directory.Error()); + auto create = config.VFS->CreateDir(directory.Value()); + if (create.Failed() && create.Error() != VFSError::AlreadyExists) + return create; + for (size_t index = 0; index < textures.size(); ++index) + { + auto& texture = textures[index]; + if (texture.Path.empty()) + return VFSResult::Fail(VFSError::InvalidPath); + const std::filesystem::path source_path(texture.Path.c_str()); + const auto source = source_path.is_absolute() ? source_path : std::filesystem::path(config.InputBaseAssetFilePath.empty() ? "" : config.InputBaseAssetFilePath.c_str()) / source_path; + std::ifstream input(source, std::ios::binary | std::ios::ate); + if (!input.is_open()) + { + ZENGINE_CORE_ERROR("Texture not found: {}", source.string()) + return VFSResult::Fail(VFSError::NotFound); + } + const auto size = input.tellg(); + if (size <= 0) + return VFSResult::Fail(VFSError::IOError); + std::vector bytes(static_cast(size)); + input.seekg(0, std::ios::beg); + if (!input.read(reinterpret_cast(bytes.data()), static_cast(size))) + return VFSResult::Fail(VFSError::IOError); + // Source basenames are not unique (e.g. two folders with albedo.png). + const auto output_name = fmt::format("{}_{}{}", source.stem().string(), index, source.extension().string()); + auto destination = MakeOutputPath(directory.Value().CStr(), output_name.c_str()); + if (destination.Failed()) + return VFSResult::Fail(destination.Error()); + auto write = Core::VFS::WriteFileAtomically(*config.VFS, destination.Value(), {bytes.data(), bytes.size()}); + if (write.Failed()) + { + ZENGINE_CORE_ERROR("Failed to copy texture '{}' to '{}' (VFS error {})", source.string(), destination.Value().CStr(), static_cast(write.Error())) + return write; + } + ReportPublishedArtifact(config, AssetFileType::TEXTURES, destination.Value()); + texture.Path.clear(); + texture.Path.append(destination.Value().CStr()); + } + return VFSResult::Ok(); + } - return !vfs->Rename(tmp_path, out_path).Failed(); + // Write data atomically via VFS: open .tmp, write, flush, close, rename to out_path. + static VFSResult WriteVFS(Core::VFS::IVFSContext* vfs, const VFSPath& out_path, const std::string& data) + { + if (!vfs) + return VFSResult::Fail(VFSError::Unsupported); + const auto* bytes = reinterpret_cast(data.data()); + return Core::VFS::WriteFileAtomically(*vfs, out_path, {bytes, data.size()}); } - AssetImporterOutput SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config) + VFSResult RestoreAssetUUID(Core::VFS::IVFSContext& ctx, const VFSPath& path, uuids::uuid& id) { - AssetImporterOutput output = {}; - if (config.OutputAssetFile.empty()) - return output; + auto meta = Core::VFS::MetaFileIO::Read(ctx, path); + if (meta.Succeeded()) + id = meta.Value().AssetUUID; + else if (meta.Error() != VFSError::NotFound && meta.Error() != VFSError::Corrupted) + return VFSResult::Fail(meta.Error()); + return VFSResult::Ok(); + } - auto mesh_dir = VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - config.VFS->CreateDir(mesh_dir); + VFSResult SerializeMeshAssetFile(Core::Memory::ArenaAllocator* /*arena*/, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config) + { + auto path = PrepareOutputPath(config.VFS, MakeOutputPath(config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str())); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + const auto& mesh_path = path.Value(); std::ostringstream buf(std::ios::binary); WriteBinary(buf, ZEMESH_MAGIC); @@ -100,22 +208,77 @@ namespace ZEngine::Importers::AssetCodec WriteBinaryHashMap(buf, hierarchies.NodeMeshes); WriteBinaryHashMap(buf, hierarchies.NodeMaterials); - if (!WriteVFS(config.VFS, mesh_path, buf.str())) - return output; + auto write = WriteVFS(config.VFS, mesh_path, buf.str()); + if (write.Failed()) + return VFSResult::Fail(write.Error()); + + ReportPublishedArtifact(config, AssetFileType::MESH, mesh_path); + + return VFSResult::Ok({.Type = AssetFileType::MESH, .Path = mesh_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); + } + + std::string MaterialOutputFilename(const AssetMaterial& material, size_t index) + { + // Names are optional/non-unique in source formats. The source index keeps + // duplicate and unnamed materials distinct and stable across recooks. + return fmt::format("{}_{}.zematerial", material.Name.empty() ? "material" : material.Name.c_str(), index); + } + + VFSResult MaterialOutputPath(const AssetMaterial& material, const ImportConfiguration& config, size_t index) + { + auto directory = MakeOutputPath(config.OutputMaterialPath.c_str(), config.AssetName.c_str()); + if (directory.Failed()) + return directory; + return MakeOutputPath(directory.Value().CStr(), MaterialOutputFilename(material, index).c_str()); + } - output = {.Type = AssetFileType::MESH, .Path = mesh_path.CStr(), .RootPath = config.OutputWorkingSpacePath.c_str()}; - return output; + VFSResult SynchronizeTextureMetadata(Core::Memory::ArenaAllocator* arena, ArrayView textures, ArrayView materials, const ImportConfiguration& config, const char* importer) + { + if (!config.VFS) + return VFSResult::Fail(VFSError::Unsupported); + for (size_t t = 0; t < textures.size(); ++t) + { + auto& texture = textures[t]; + auto path = VFSPath::Parse(texture.Path.c_str()); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + auto hash = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, path.Value()); + if (hash.Failed()) + return VFSResult::Fail(hash.Error()); + auto meta = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, path.Value(), importer, hash.Value()); + if (meta.Failed()) + return VFSResult::Fail(meta.Error()); + + const auto original_uuid = texture.TextureUUID; + texture.TextureUUID = meta.Value().AssetUUID; + for (size_t m = 0; m < materials.size(); ++m) + { + auto& material = materials[m]; + auto sync = [&](uuids::uuid& id, String& destination) { + if (!original_uuid.is_nil() && id == original_uuid) + { + id = texture.TextureUUID; + destination.init(arena, texture.Path.c_str()); + } + }; + sync(material.AlbedoTexUUID, material.AlbedoTexPath); + sync(material.EmissiveTexUUID, material.EmissiveTexPath); + sync(material.NormalTexUUID, material.NormalTexPath); + sync(material.OpacityTexUUID, material.OpacityTexPath); + sync(material.SpecularTexUUID, material.SpecularTexPath); + } + } + return VFSResult::Ok(); } - AssetImporterOutput SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* /*arena*/, AssetMaterial& material, const ImportConfiguration& config) + VFSResult SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* /*arena*/, AssetMaterial& material, const ImportConfiguration& config, size_t material_index) { - AssetImporterOutput output = {}; - std::string asset_mat_filename = fmt::format("{}{}", material.Name.c_str(), ".zematerial"); - auto mat_dir = VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - config.VFS->CreateDir(mat_dir); - auto mat_path = mat_dir / asset_mat_filename.c_str(); + auto path = PrepareOutputPath(config.VFS, MaterialOutputPath(material, config, material_index)); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + const auto& mat_path = path.Value(); - auto tex_obj = [](const uuids::uuid& uuid, const Core::Containers::String& path) { + auto tex_obj = [](const uuids::uuid& uuid, const Core::Containers::String& path) { nlohmann::json t; t["uuid"] = uuids::to_string(uuid); t["path"] = path.empty() ? "" : path.c_str(); @@ -137,20 +300,24 @@ namespace ZEngine::Importers::AssetCodec j["specular_color"] = nlohmann::json::array({material.SpecularColor[0], material.SpecularColor[1], material.SpecularColor[2], material.SpecularColor[3]}); j["factors"] = nlohmann::json::array({material.Factors[0], material.Factors[1], material.Factors[2], material.Factors[3]}); - if (!WriteVFS(config.VFS, mat_path, j.dump(4))) - return output; + auto write = WriteVFS(config.VFS, mat_path, j.dump(4)); + if (write.Failed()) + return VFSResult::Fail(write.Error()); + + ReportPublishedArtifact(config, AssetFileType::MATERIAL, mat_path); - output = {.Type = AssetFileType::MATERIAL, .Path = mat_path.CStr(), .RootPath = config.OutputWorkingSpacePath.c_str()}; - return output; + return VFSResult::Ok({.Type = AssetFileType::MATERIAL, .Path = mat_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); } - AssetImporterOutput SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, ArrayView textures, const ImportConfiguration& config) + VFSResult SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* /*arena*/, ArrayView textures, const ImportConfiguration& config) { - AssetImporterOutput output = {}; - std::string asset_tex_filename = fmt::format("{}{}", config.AssetName.c_str(), ".zetextures"); - auto tex_dir = VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto tex_path = tex_dir / asset_tex_filename.c_str(); - config.VFS->CreateDir(tex_dir); + if (config.AssetName.empty()) + return VFSResult::Fail(VFSError::InvalidPath); + std::string asset_tex_filename = fmt::format("{}.zetextures", config.AssetName.c_str()); + auto path = PrepareOutputPath(config.VFS, MakeOutputPath(config.OutputAssetsPath.c_str(), asset_tex_filename.c_str())); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + const auto& tex_path = path.Value(); std::ostringstream buf(std::ios::binary); WriteBinary(buf, ZETEXTURES_MAGIC); @@ -163,11 +330,13 @@ namespace ZEngine::Importers::AssetCodec WriteBinaryString(buf, textures[i].Path); } - if (!WriteVFS(config.VFS, tex_path, buf.str())) - return output; + auto write = WriteVFS(config.VFS, tex_path, buf.str()); + if (write.Failed()) + return VFSResult::Fail(write.Error()); + + ReportPublishedArtifact(config, AssetFileType::TEXTURES, tex_path); - output = {.Type = AssetFileType::TEXTURES, .Path = tex_path.CStr(), .RootPath = config.OutputWorkingSpacePath.c_str()}; - return output; + return VFSResult::Ok({.Type = AssetFileType::TEXTURES, .Path = tex_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); } void DeserializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMesh& mesh, AssetNodeHierarchy& hierarchies) diff --git a/ZEngine/ZEngine/Importers/AssetCodec.h b/ZEngine/ZEngine/Importers/AssetCodec.h index c3930c80..991073b3 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.h +++ b/ZEngine/ZEngine/Importers/AssetCodec.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -52,10 +53,23 @@ namespace ZEngine::Importers::AssetCodec Core::Containers::String InputBaseAssetFilePath; Core::Containers::String OutputWorkingSpacePath; Core::Containers::String OutputTextureFilesPath; - Core::VFS::IVFSContext* VFS = nullptr; - ImportOptions Options = {}; + Core::VFS::IVFSContext* VFS = nullptr; + // Called synchronously after an artifact commits. This remains separate from + // the completion callback so a failed import can still report partial output. + void* ArtifactContext = nullptr; + void (*OnArtifactPublished)(void* context, AssetFileType type, const char* path) = nullptr; + ImportOptions Options = {}; }; + // Validate before parsing a source or accessing VFS metadata; returns the mesh destination. + [[nodiscard]] Core::VFS::VFSResult ValidateImportConfiguration(const ImportConfiguration& config); + [[nodiscard]] Core::VFS::VFSResult MakeOutputPath(const char* directory, const char* filename); + [[nodiscard]] Core::VFS::VFSResult CopyTextureFiles(Core::Containers::ArrayView textures, const ImportConfiguration& config); + void ReportPublishedArtifact(const ImportConfiguration& config, AssetFileType type, const Core::VFS::VFSPath& path); + std::string MaterialOutputFilename(const AssetMaterial& material, size_t index); + [[nodiscard]] Core::VFS::VFSResult MaterialOutputPath(const AssetMaterial& material, const ImportConfiguration& config, size_t index); + [[nodiscard]] Core::VFS::VFSResult SynchronizeTextureMetadata(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, Core::Containers::ArrayView materials, const ImportConfiguration& config, const char* importer); + struct AssetMeshFileHeader { uint32_t MagicNumber = 0xFFFFFF; @@ -91,30 +105,33 @@ namespace ZEngine::Importers::AssetCodec uint32_t ImporterVersion = ENVIRONMENT_MAP_IMPORTER_VERSION; }; - AssetImporterOutput SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config); + [[nodiscard]] Core::VFS::VFSResult SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config); + + // Preserve an existing identity; only missing/corrupt sidecars permit a new one. + [[nodiscard]] Core::VFS::VFSResult RestoreAssetUUID(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, uuids::uuid& id); - AssetImporterOutput SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, AssetMaterial& material, const ImportConfiguration& config); + [[nodiscard]] Core::VFS::VFSResult SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, AssetMaterial& material, const ImportConfiguration& config, size_t material_index); - AssetImporterOutput SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, const ImportConfiguration& config); + [[nodiscard]] Core::VFS::VFSResult SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, const ImportConfiguration& config); - [[nodiscard]] uint32_t GetEnvironmentMapFullMipCount(uint32_t face_size); - [[nodiscard]] bool IsEnvironmentMapFileHeaderValid(const EnvironmentMapFileHeader& header); - [[nodiscard]] bool DoesEnvironmentMapHeaderMatchSource(const EnvironmentMapFileHeader& header, uint64_t source_hash); + [[nodiscard]] uint32_t GetEnvironmentMapFullMipCount(uint32_t face_size); + [[nodiscard]] bool IsEnvironmentMapFileHeaderValid(const EnvironmentMapFileHeader& header); + [[nodiscard]] bool DoesEnvironmentMapHeaderMatchSource(const EnvironmentMapFileHeader& header, uint64_t source_hash); // VFS-based — writes through IVFSContext using atomic .tmp → rename protocol. // out_path: the VFS path to write (e.g. project://_cache/envmaps/.zenvmap) - Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap, const EnvironmentMapCookMetadata& metadata = {}); + Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap, const EnvironmentMapCookMetadata& metadata = {}); - void DeserializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMesh& mesh, AssetNodeHierarchy& hierarchies); + void DeserializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMesh& mesh, AssetNodeHierarchy& hierarchies); - void DeserializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMaterial& material); + void DeserializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMaterial& material); - void DeserializeTextureAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, Core::Containers::Array& textures); + void DeserializeTextureAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, Core::Containers::Array& textures); - bool ReadAssetMeshFileHeader(const char* asset_file, AssetMeshFileHeader& header); + bool ReadAssetMeshFileHeader(const char* asset_file, AssetMeshFileHeader& header); - bool DeserializeEnvironmentMapFile(const char* zenvmap_file, Rendering::Buffers::Bitmap& out_cubemap); + bool DeserializeEnvironmentMapFile(const char* zenvmap_file, Rendering::Buffers::Bitmap& out_cubemap); - bool ReadEnvironmentMapFileHeader(const char* zenvmap_file, EnvironmentMapFileHeader& out_header); + bool ReadEnvironmentMapFileHeader(const char* zenvmap_file, EnvironmentMapFileHeader& out_header); } // namespace ZEngine::Importers::AssetCodec diff --git a/ZEngine/ZEngine/Importers/AssetTypes.h b/ZEngine/ZEngine/Importers/AssetTypes.h index 486410de..d897840e 100644 --- a/ZEngine/ZEngine/Importers/AssetTypes.h +++ b/ZEngine/ZEngine/Importers/AssetTypes.h @@ -46,27 +46,40 @@ namespace ZEngine::Importers struct AssetMaterial { - Core::Containers::String Name = {}; - uuids::uuid MaterialUUID = {}; + Core::Containers::String Name = {}; + uuids::uuid MaterialUUID = {}; // Texture UUIDs — runtime lookup key into AssetManager - uuids::uuid AlbedoTexUUID = {}; - uuids::uuid EmissiveTexUUID = {}; - uuids::uuid NormalTexUUID = {}; - uuids::uuid OpacityTexUUID = {}; - uuids::uuid SpecularTexUUID = {}; + uuids::uuid AlbedoTexUUID = {}; + uuids::uuid EmissiveTexUUID = {}; + uuids::uuid NormalTexUUID = {}; + uuids::uuid OpacityTexUUID = {}; + uuids::uuid SpecularTexUUID = {}; // Texture paths — project-relative VFS path to the extracted image file // Stored in .zematerial so no separate .zetextures file is needed - Core::Containers::String AlbedoTexPath = {}; - Core::Containers::String EmissiveTexPath = {}; - Core::Containers::String NormalTexPath = {}; - Core::Containers::String OpacityTexPath = {}; - Core::Containers::String SpecularTexPath = {}; - float AmbientColor[4] = {0}; - float AlbedoColor[4] = {0}; - float EmissiveColor[4] = {0}; - float RoughnessColor[4] = {0}; - float SpecularColor[4] = {0}; - float Factors[4] = {0}; + Core::Containers::String AlbedoTexPath = {}; + Core::Containers::String EmissiveTexPath = {}; + Core::Containers::String NormalTexPath = {}; + Core::Containers::String OpacityTexPath = {}; + Core::Containers::String SpecularTexPath = {}; + float AmbientColor[4] = {0}; + float AlbedoColor[4] = {0}; + float EmissiveColor[4] = {0}; + float RoughnessColor[4] = {0}; + float SpecularColor[4] = {0}; + float Factors[4] = {0}; + + // Copy string storage as well as values when crossing an arena lifetime boundary. + [[nodiscard]] AssetMaterial Clone(Core::Memory::ArenaAllocator* arena) const + { + AssetMaterial copy = *this; + copy.Name.init(arena, Name.c_str()); + copy.AlbedoTexPath.init(arena, AlbedoTexPath.c_str()); + copy.EmissiveTexPath.init(arena, EmissiveTexPath.c_str()); + copy.NormalTexPath.init(arena, NormalTexPath.c_str()); + copy.OpacityTexPath.init(arena, OpacityTexPath.c_str()); + copy.SpecularTexPath.init(arena, SpecularTexPath.c_str()); + return copy; + } }; struct AssetTexture @@ -118,6 +131,8 @@ namespace ZEngine::Importers std::string RootPath = ""; }; + // Callbacks run synchronously during ImportFile; do not re-enter the same importer. + // The output view is borrowed for the duration of the callback; copy it before deferring work. using ImportCompleteCallback = void (*)(void* ctx, Core::Containers::ArrayView outputs); using ImportProgressCallback = void (*)(void* ctx, float progress); using ImportErrorCallback = void (*)(void* ctx, std::string_view message); diff --git a/ZEngine/ZEngine/Importers/AssimpImporter.cpp b/ZEngine/ZEngine/Importers/AssimpImporter.cpp index 0f7f6349..c0b8f609 100644 --- a/ZEngine/ZEngine/Importers/AssimpImporter.cpp +++ b/ZEngine/ZEngine/Importers/AssimpImporter.cpp @@ -1,16 +1,16 @@ #include -#include #include #include #include #include +#include #include #include #include #include +#include #include -#include -#include +#include using namespace ZEngine::Helpers; using ZEngine::Core::VFS::VFSPath; @@ -19,20 +19,11 @@ using namespace ZEngine::Core::Containers; using namespace ZEngine::Core::Maths; using namespace uuids; -namespace fs = std::filesystem; - namespace ZEngine::Importers { static constexpr uint32_t kDefaultFlags = aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_GenNormals | aiProcess_SortByPType; - AssimpImporter::AssimpImporter() : m_progress_handler{} - { - m_progress_handler.SetImporter(this); - } - - AssimpImporter::~AssimpImporter() {} - - void AssimpImporter::Initialize(Core::Memory::ArenaAllocator* arena) + void AssimpImporter::Initialize(Core::Memory::ArenaAllocator* arena) { arena->CreateSubArena(ZMega(128), &Arena, "ImportPipeline/AssimpImporter"); } @@ -46,10 +37,11 @@ namespace ZEngine::Importers Core::VFS::VFSResult AssimpImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { + std::lock_guard import_lock(m_import_mutex); // Resolve VFS path to native filesystem path for Assimp. // path is VFS-relative (e.g. /Assets/Lamp01.glb); Assimp needs the full native path. - char native[MAX_FILE_PATH_COUNT] = {}; - const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + char native[MAX_FILE_PATH_COUNT] = {}; + const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; if (working_space && working_space[0] != '\0') path.ResolveNative(working_space, native, sizeof(native)); else @@ -83,7 +75,6 @@ namespace ZEngine::Importers ExtractTextures(&scratch, scene, gen, materials, textures); CreateHierachy(&scratch, scene, gen, hierarchies, mesh, materials); - importer.SetProgressHandler(nullptr); importer.FreeScene(); // Ingest directly into AssetManager CPU buffers — no intermediate .zasset round-trip. @@ -103,6 +94,14 @@ namespace ZEngine::Importers void AssimpImporter::ImportFile(const char* filename, const AssetCodec::ImportConfiguration& cfg, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log) { + std::lock_guard import_lock(m_import_mutex); + auto mesh_path = AssetCodec::ValidateImportConfiguration(cfg); + if (mesh_path.Failed() || !filename || !filename[0]) + { + if (on_error) + on_error(context, fmt::format("Invalid import configuration (VFS error {})", static_cast(mesh_path.Failed() ? mesh_path.Error() : Core::VFS::VFSError::InvalidPath))); + return; + } // The caller's arena is sized only for a few short path strings (#760) — carve a // scratch sub-arena from this importer's own, generously-sized private Arena // instead, matching the pattern Import() already uses for hot-reload. @@ -118,10 +117,12 @@ namespace ZEngine::Importers config.AssetName.init(arena, cfg.AssetName.c_str()); config.OutputAssetFile.init(arena, cfg.OutputAssetFile.c_str()); config.InputBaseAssetFilePath.init(arena, cfg.InputBaseAssetFilePath.c_str()); - config.VFS = cfg.VFS; - config.Options = cfg.Options; + config.VFS = cfg.VFS; + config.ArtifactContext = cfg.ArtifactContext; + config.OnArtifactPublished = cfg.OnArtifactPublished; + config.Options = cfg.Options; - uint32_t flags = aiProcess_Triangulate | aiProcess_SortByPType; + uint32_t flags = aiProcess_Triangulate | aiProcess_SortByPType; if (config.Options.MergeVertices) flags |= aiProcess_JoinIdenticalVertices; if (config.Options.FlipUVs) @@ -132,9 +133,7 @@ namespace ZEngine::Importers flags |= aiProcess_GenSmoothNormals; Assimp::Importer importer{}; - importer.SetProgressHandler(&m_progress_handler); - - const aiScene* scene = importer.ReadFile(filename, flags); + const aiScene* scene = importer.ReadFile(filename, flags); if (!scene || (scene->mFlags & AI_SCENE_FLAGS_INCOMPLETE) || !scene->mRootNode) { @@ -162,11 +161,14 @@ namespace ZEngine::Importers // "this is an update to an existing mesh" — every re-cook would otherwise // mint a fresh random UUID and look like a brand new, unrelated asset. { - auto mesh_dir = Core::VFS::VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mesh_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - mesh.MeshUUID = meta_result.Value().AssetUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mesh_path.Value(), mesh.MeshUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read mesh metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } if (on_progress) @@ -197,51 +199,43 @@ namespace ZEngine::Importers // reference off this same materials array for each SubMesh. for (size_t m = 0; m < materials.size(); ++m) { - auto mat_dir = Core::VFS::VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - std::string mat_filename = fmt::format("{}{}", materials[m].Name.c_str(), ".zematerial"); - auto mat_path = mat_dir / mat_filename.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mat_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - materials[m].MaterialUUID = meta_result.Value().AssetUUID; + std::string mat_filename = AssetCodec::MaterialOutputFilename(materials[m], m); + auto mat_path = AssetCodec::MaterialOutputPath(materials[m], config, m); + if (mat_path.Failed()) + { + if (on_error) + on_error(context, fmt::format("Invalid material output '{}' (VFS error {})", mat_filename, static_cast(mat_path.Error()))); + Arena.Clear(); + return; + } + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mat_path.Value(), materials[m].MaterialUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read material metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } if (config.Options.ImportTextures) { ExtractTextures(arena, scene, gen, materials, textures); - CopyTextureFiles(arena, textures, config); - for (size_t m = 0; m < materials.size(); ++m) + auto copy = AssetCodec::CopyTextureFiles(ArrayView{textures}, config); + if (copy.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to copy texture files (VFS error {})", static_cast(copy.Error()))); + Arena.Clear(); + return; + } + auto metadata = AssetCodec::SynchronizeTextureMetadata(arena, ArrayView{textures}, ArrayView{materials}, config, "AssimpImporter"); + if (metadata.Failed()) { - // ExtractTextures assigned a throwaway random UUID (#755); now - // that the file is at its final path, replace it with a stable, - // content-hashed one, matched by the original UUID still held - // in the material fields. - auto sync_texture = [&](uuids::uuid& uuid_field, Core::Containers::String& path_out) { - for (size_t t = 0; t < textures.size(); ++t) - { - if (textures[t].TextureUUID == uuid_field && !textures[t].Path.empty()) - { - path_out.init(arena, textures[t].Path.c_str()); - - auto vfs_path = Core::VFS::VFSPath::Parse(textures[t].Path.c_str()); - if (vfs_path.Succeeded() && config.VFS) - { - auto hash_result = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, vfs_path.Value()); - auto meta_result = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, vfs_path.Value(), "AssimpImporter", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta_result.Succeeded()) - { - textures[t].TextureUUID = meta_result.Value().AssetUUID; - uuid_field = meta_result.Value().AssetUUID; - } - } - return; - } - } - }; - sync_texture(materials[m].AlbedoTexUUID, materials[m].AlbedoTexPath); - sync_texture(materials[m].EmissiveTexUUID, materials[m].EmissiveTexPath); - sync_texture(materials[m].NormalTexUUID, materials[m].NormalTexPath); - sync_texture(materials[m].OpacityTexUUID, materials[m].OpacityTexPath); - sync_texture(materials[m].SpecularTexUUID, materials[m].SpecularTexPath); + if (on_error) + on_error(context, fmt::format("Failed to persist texture metadata (VFS error {})", static_cast(metadata.Error()))); + Arena.Clear(); + return; } } } @@ -249,21 +243,45 @@ namespace ZEngine::Importers if (on_progress) on_progress(context, 0.7f); - Array outputs = {}; - outputs.init(arena, 100); - outputs.push(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchies, config)); + std::vector outputs; + outputs.reserve(1 + materials.size()); + auto append_output = [&](Core::VFS::VFSResult result, const char* directory, const std::string& filename) { + if (result.Failed()) + { + const auto message = fmt::format("Failed to write '{}/{}' (VFS error {})", directory, filename, static_cast(result.Error())); + ZENGINE_CORE_ERROR("[AssimpImporter] {}", message) + if (on_error) + on_error(context, message); + return false; + } + outputs.push_back(std::move(result.Value())); + return true; + }; + bool serialized = true; if (config.Options.ImportMaterials) for (size_t i = 0; i < materials.size(); ++i) - outputs.push(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config)); + if (!append_output(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config, i), config.OutputMaterialPath.c_str(), fmt::format("{}/{}", config.AssetName.c_str(), AssetCodec::MaterialOutputFilename(materials[i], i)))) + { + serialized = false; + break; + } - if (on_progress) - on_progress(context, 1.0f); - if (on_complete) - on_complete(context, ArrayView{outputs}); + // The mesh is the last published artifact: failed dependencies leave + // the previous mesh untouched, without a multi-file transaction layer. + if (serialized) + serialized = append_output(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchies, config), config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str()); + if (serialized) + { + std::rotate(outputs.begin(), outputs.end() - 1, outputs.end()); + if (on_progress) + on_progress(context, 1.0f); + if (on_complete) + on_complete(context, {outputs.data(), outputs.size()}); + } } - importer.SetProgressHandler(nullptr); importer.FreeScene(); + Arena.Clear(); } void AssimpImporter::ExtractMeshes(Core::Memory::ArenaAllocator* arena, const aiScene* scene, uuids::uuid_random_generator& generator, AssetMesh& mesh) @@ -304,7 +322,7 @@ namespace ZEngine::Importers for (int v = 0; v < ai_mesh->mNumVertices; ++v) { const aiVector3D position = ai_mesh->mVertices[v]; - const aiVector3D normal = ai_mesh->mNormals[v]; + const aiVector3D normal = ai_mesh->HasNormals() ? ai_mesh->mNormals[v] : aiVector3D{0, 1, 0}; const aiVector3D texture = ai_mesh->HasTextureCoords(0) ? ai_mesh->mTextureCoords[0][v] : aiVector3D{}; mesh.Vertices.push(position.x); @@ -568,103 +586,22 @@ namespace ZEngine::Importers n.init(arena, mesh_name.C_Str() ? mesh_name.C_Str() : ""); hierarchy.NodeMeshes[sub_node_id] = mesh; - hierarchy.NodeMaterials[sub_node_id] = material_id; hierarchy.GlobalTransforms[sub_node_id] = Identity(); hierarchy.LocalTransforms[sub_node_id] = Identity(); - auto& asset_mat = materials[material_id]; auto& sub_mesh = asset_mesh.SubMeshes[mesh]; - sub_mesh.MaterialUUID = asset_mat.MaterialUUID; - } - - for (uint32_t child = 0; child < node->mNumChildren; ++child) - { - TraverseNode(arena, ai_scene, node->mChildren[child], hierarchy, asset_mesh, materials, node_id, (depth_level + 1)); - } - } - - void AssimpImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) - { - /* - * Normalize file naming - */ - char dst_dir_buf[MAX_FILE_PATH_COUNT] = {}; - (VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()).ResolveNative(config.OutputWorkingSpacePath.c_str(), dst_dir_buf, sizeof(dst_dir_buf)); - std::string dst_dir = dst_dir_buf; - - auto CreateBaseDirectoryFn = [](std::string_view filename) -> void { - auto base_dir = fs::absolute(filename).parent_path(); - - std::error_code err = {}; - if (!fs::exists(base_dir)) - { - fs::create_directories(base_dir, err); - } - }; - - auto scratch = ZGetScratch(arena); - Array src_tex_files = {}; - Array dst_tex_files = {}; - src_tex_files.init(scratch.Arena, textures.size()); - dst_tex_files.init(scratch.Arena, textures.size()); - - for (auto& tex : textures) - { - if (tex.Path.empty()) - { - continue; - } - - fs::path tex_path(tex.Path.c_str()); - auto src_file = (tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path).string(); - auto dst_file = (fs::path(dst_dir) / tex_path.filename()).string(); - - CreateBaseDirectoryFn(dst_file); - - auto& sf = src_tex_files.push_use({}); - auto& df = dst_tex_files.push_use({}); - - sf.init(scratch.Arena, src_file.c_str()); - df.init(scratch.Arena, dst_file.c_str()); - } - /* - * Texture files processing - * (1) Ensuring Scene sub-dir is created - * (2) Copying files to destination - */ - - ZENGINE_VALIDATE_ASSERT(src_tex_files.size() == dst_tex_files.size(), "source files count can't be diff of destination files count") - for (int i = 0; i < src_tex_files.size(); ++i) - { - auto src = fs::absolute(src_tex_files[i].c_str()); - auto dst = fs::absolute(dst_tex_files[i].c_str()); - - std::ifstream in(src.c_str(), std::ios::binary); - std::ofstream out(dst.c_str(), std::ios::binary); - - if (!in.is_open() || !out.is_open()) + if (material_id < materials.size()) { - in.close(); - out.close(); - continue; + hierarchy.NodeMaterials[sub_node_id] = material_id; + sub_mesh.MaterialUUID = materials[material_id].MaterialUUID; } - - out << in.rdbuf(); - - in.close(); - out.close(); + else + sub_mesh.MaterialUUID = {}; } - ZReleaseScratch(scratch); - - /* - * Update texture path - */ - for (auto& tex : textures) + for (uint32_t child = 0; child < node->mNumChildren; ++child) { - auto new_path = std::string((VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str() / tex.Path.c_str()).CStr()); - tex.Path.clear(); - tex.Path.append(new_path.c_str()); + TraverseNode(arena, ai_scene, node->mChildren[child], hierarchy, asset_mesh, materials, node_id, (depth_level + 1)); } } @@ -681,13 +618,4 @@ namespace ZEngine::Importers return mm; } - void AssimpProgressHandler::SetImporter(AssimpImporter* const importer) - { - m_importer = importer; - } - - bool AssimpProgressHandler::Update(float /*percentage*/) - { - return true; - } } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/AssimpImporter.h b/ZEngine/ZEngine/Importers/AssimpImporter.h index fe968973..7ddc515b 100644 --- a/ZEngine/ZEngine/Importers/AssimpImporter.h +++ b/ZEngine/ZEngine/Importers/AssimpImporter.h @@ -3,30 +3,15 @@ #include #include #include -#include #include #include +#include namespace ZEngine::Importers { - class AssimpImporter; - struct AssimpProgressHandler; - - struct AssimpProgressHandler : public Assimp::ProgressHandler - { - void SetImporter(AssimpImporter* const importer); - bool Update(float percentage) override; - - private: - AssimpImporter* m_importer{nullptr}; - }; - class AssimpImporter : public IAssetImporter { public: - AssimpImporter(); - virtual ~AssimpImporter(); - void Initialize(Core::Memory::ArenaAllocator* arena); Core::Memory::ArenaAllocator Arena = {}; @@ -39,12 +24,8 @@ namespace ZEngine::Importers // file-picker import path (produces cooked .zasset artifacts). void ImportFile(const char* filename, const AssetCodec::ImportConfiguration& config, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log); - void CopyTextureFiles(Core::Memory::ArenaAllocator*, Core::Containers::Array&, const AssetCodec::ImportConfiguration&); - private: - AssimpProgressHandler m_progress_handler; - - friend struct AssimpProgressHandler; + std::mutex m_import_mutex; void ExtractMeshes(Core::Memory::ArenaAllocator*, const aiScene*, uuids::uuid_random_generator&, AssetMesh&); void ExtractMaterials(Core::Memory::ArenaAllocator*, const aiScene*, uuids::uuid_random_generator&, Core::Containers::Array&, AssetNodeHierarchy&); diff --git a/ZEngine/ZEngine/Importers/FbxImporter.cpp b/ZEngine/ZEngine/Importers/FbxImporter.cpp index 183c0417..6032ae91 100644 --- a/ZEngine/ZEngine/Importers/FbxImporter.cpp +++ b/ZEngine/ZEngine/Importers/FbxImporter.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,10 +6,10 @@ #include #include #include -#include -#include +#include #include #include +#include using ZEngine::Core::VFS::VFSPath; using namespace ZEngine::Core::Containers; @@ -18,8 +17,6 @@ using namespace ZEngine::Core::Maths; using namespace ZEngine::Importers; using namespace uuids; -namespace fs = std::filesystem; - namespace ZEngine::Importers { // Vertex deduplication key — (position, normal, uv) index triple. @@ -54,8 +51,9 @@ namespace ZEngine::Importers Core::VFS::VFSResult FbxImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { - char native[MAX_FILE_PATH_COUNT] = {}; - const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + std::lock_guard import_lock(m_import_mutex); + char native[MAX_FILE_PATH_COUNT] = {}; + const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; if (working_space && working_space[0] != '\0') path.ResolveNative(working_space, native, sizeof(native)); else @@ -215,6 +213,14 @@ namespace ZEngine::Importers void FbxImporter::ImportFile(const char* filename, const AssetCodec::ImportConfiguration& cfg, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log) { + std::lock_guard import_lock(m_import_mutex); + auto mesh_path = AssetCodec::ValidateImportConfiguration(cfg); + if (mesh_path.Failed() || !filename || !filename[0]) + { + if (on_error) + on_error(context, fmt::format("Invalid import configuration (VFS error {})", static_cast(mesh_path.Failed() ? mesh_path.Error() : Core::VFS::VFSError::InvalidPath))); + return; + } // The caller's arena is sized only for a few short path strings (#760) — route // everything through this importer's own, generously-sized private Arena instead. Arena.Clear(); @@ -229,6 +235,8 @@ namespace ZEngine::Importers config.OutputAssetFile.init(arena, cfg.OutputAssetFile.c_str()); config.InputBaseAssetFilePath.init(arena, cfg.InputBaseAssetFilePath.c_str()); config.VFS = cfg.VFS; + config.ArtifactContext = cfg.ArtifactContext; + config.OnArtifactPublished = cfg.OnArtifactPublished; config.Options = cfg.Options; ufbx_load_opts opts = {}; @@ -273,11 +281,15 @@ namespace ZEngine::Importers // update to an existing mesh" — every re-cook would otherwise mint a fresh // random UUID and look like a brand new, unrelated asset. { - auto mesh_dir = VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mesh_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - mesh.MeshUUID = meta_result.Value().AssetUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mesh_path.Value(), mesh.MeshUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read mesh metadata (VFS error {})", static_cast(identity.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } } mesh.Vertices.init(scratch.Arena, (size_t) total_tris * 3 * 8); @@ -432,56 +444,55 @@ namespace ZEngine::Importers { for (size_t m = 0; m < materials.size(); ++m) { - auto mat_dir = VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - std::string mat_filename = fmt::format("{}{}", materials[m].Name.c_str(), ".zematerial"); - auto mat_path = mat_dir / mat_filename.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mat_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) + std::string mat_filename = AssetCodec::MaterialOutputFilename(materials[m], m); + auto mat_path = AssetCodec::MaterialOutputPath(materials[m], config, m); + if (mat_path.Failed()) + { + if (on_error) + on_error(context, fmt::format("Invalid material output '{}' (VFS error {})", mat_filename, static_cast(mat_path.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } + const auto old_uuid = materials[m].MaterialUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mat_path.Value(), materials[m].MaterialUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read material metadata (VFS error {})", static_cast(identity.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } + if (old_uuid != materials[m].MaterialUUID) { - uuids::uuid old_uuid = materials[m].MaterialUUID; - uuids::uuid new_uuid = meta_result.Value().AssetUUID; - materials[m].MaterialUUID = new_uuid; for (size_t s = 0; s < mesh.SubMeshes.size(); ++s) if (mesh.SubMeshes[s].MaterialUUID == old_uuid) - mesh.SubMeshes[s].MaterialUUID = new_uuid; + mesh.SubMeshes[s].MaterialUUID = materials[m].MaterialUUID; } } } if (config.Options.ImportMaterials && config.Options.ImportTextures) { - CopyTextureFiles(arena, textures, config); + auto copy = AssetCodec::CopyTextureFiles(ArrayView{textures}, config); + if (copy.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to copy texture files (VFS error {})", static_cast(copy.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } - // Fbx never synced texture paths into materials (pre-existing gap), and - // push_tex assigned throwaway random UUIDs (#755) — fix both here, - // matched by the original UUID before it's replaced. - for (size_t m = 0; m < materials.size(); ++m) + auto metadata = AssetCodec::SynchronizeTextureMetadata(arena, ArrayView{textures}, ArrayView{materials}, config, "FbxImporter"); + if (metadata.Failed()) { - auto sync_texture = [&](uuids::uuid& uuid_field, Core::Containers::String& path_out) { - for (size_t t = 0; t < textures.size(); ++t) - { - if (textures[t].TextureUUID == uuid_field && !textures[t].Path.empty()) - { - path_out.init(arena, textures[t].Path.c_str()); - - auto vfs_path = VFSPath::Parse(textures[t].Path.c_str()); - if (vfs_path.Succeeded() && config.VFS) - { - auto hash_result = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, vfs_path.Value()); - auto meta_result = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, vfs_path.Value(), "FbxImporter", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta_result.Succeeded()) - { - textures[t].TextureUUID = meta_result.Value().AssetUUID; - uuid_field = meta_result.Value().AssetUUID; - } - } - return; - } - } - }; - sync_texture(materials[m].AlbedoTexUUID, materials[m].AlbedoTexPath); - sync_texture(materials[m].NormalTexUUID, materials[m].NormalTexPath); - sync_texture(materials[m].SpecularTexUUID, materials[m].SpecularTexPath); + if (on_error) + on_error(context, fmt::format("Failed to persist texture metadata (VFS error {})", static_cast(metadata.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; } } @@ -503,12 +514,35 @@ namespace ZEngine::Importers hier.LocalTransforms.push(Identity()); hier.GlobalTransforms.push(Identity()); - Array outputs = {}; - outputs.init(scratch.Arena, 16); - outputs.push(AssetCodec::SerializeMeshAssetFile(scratch.Arena, mesh, hier, config)); + std::vector outputs; + outputs.reserve(1 + materials.size()); + auto append_output = [&](Core::VFS::VFSResult result, const char* directory, const std::string& filename) { + if (result.Failed()) + { + const auto message = fmt::format("Failed to write '{}/{}' (VFS error {})", directory, filename, static_cast(result.Error())); + ZENGINE_CORE_ERROR("[FbxImporter] {}", message) + if (on_error) + on_error(context, message); + return false; + } + outputs.push_back(std::move(result.Value())); + return true; + }; if (config.Options.ImportMaterials) for (size_t i = 0; i < materials.size(); ++i) - outputs.push(AssetCodec::SerializeMaterialAssetFile(scratch.Arena, materials[i], config)); + if (!append_output(AssetCodec::SerializeMaterialAssetFile(scratch.Arena, materials[i], config, i), config.OutputMaterialPath.c_str(), fmt::format("{}/{}", config.AssetName.c_str(), AssetCodec::MaterialOutputFilename(materials[i], i)))) + { + ZReleaseScratch(scratch); + return; + } + + // Dependencies first; publish the mesh last while keeping callback order. + if (!append_output(AssetCodec::SerializeMeshAssetFile(scratch.Arena, mesh, hier, config), config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str())) + { + ZReleaseScratch(scratch); + return; + } + std::rotate(outputs.begin(), outputs.end() - 1, outputs.end()); auto* mgr = Managers::AssetManager::Instance(); if (mgr) @@ -524,42 +558,9 @@ namespace ZEngine::Importers if (on_progress) on_progress(context, 1.0f); if (on_complete) - on_complete(context, ArrayView{outputs}); + on_complete(context, {outputs.data(), outputs.size()}); ZReleaseScratch(scratch); } - void FbxImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) - { - if (textures.empty()) - return; - - char dst_dir_buf[MAX_FILE_PATH_COUNT] = {}; - (VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()).ResolveNative(config.OutputWorkingSpacePath.c_str(), dst_dir_buf, sizeof(dst_dir_buf)); - if (config.VFS) - config.VFS->CreateDir(VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()); - - for (auto& tex : textures) - { - if (tex.Path.empty()) - continue; - - fs::path tex_path(tex.Path.c_str()); - fs::path src = tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path; - fs::path dst = fs::path(dst_dir_buf) / tex_path.filename(); - - std::ifstream in(src, std::ios::binary); - if (!in.is_open()) - { - ZENGINE_CORE_WARN("[FbxImporter] Texture not found: {}", src.string()) - continue; - } - std::ofstream out(dst, std::ios::binary); - out << in.rdbuf(); - - auto new_path = std::string((VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str() / tex_path.filename().string().c_str()).CStr()); - tex.Path.clear(); - tex.Path.append(new_path.c_str()); - } - } } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/FbxImporter.h b/ZEngine/ZEngine/Importers/FbxImporter.h index e548ccdd..25adde8a 100644 --- a/ZEngine/ZEngine/Importers/FbxImporter.h +++ b/ZEngine/ZEngine/Importers/FbxImporter.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace ZEngine::Importers { @@ -18,6 +19,6 @@ namespace ZEngine::Importers void ImportFile(const char* filename, const AssetCodec::ImportConfiguration& config, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log); private: - void CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config); + std::mutex m_import_mutex; }; } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/GltfImporter.cpp b/ZEngine/ZEngine/Importers/GltfImporter.cpp index c95c797e..f2318fd1 100644 --- a/ZEngine/ZEngine/Importers/GltfImporter.cpp +++ b/ZEngine/ZEngine/Importers/GltfImporter.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -16,8 +15,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -78,7 +79,224 @@ namespace ZEngine::Importers node.transform); } - static void ExtractMeshes(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, AssetMesh& out) + static bool HasTriangleGeometry(const fastgltf::Primitive& primitive) + { + return primitive.type == fastgltf::PrimitiveType::Triangles && primitive.findAttribute("POSITION") != primitive.attributes.end(); + } + + static ArrayView BufferViewBytes(const fastgltf::Asset& asset, size_t index) + { + if (index >= asset.bufferViews.size()) + return {}; + const auto& view = asset.bufferViews[index]; + if (view.bufferIndex >= asset.buffers.size()) + return {}; + const auto& buffer = asset.buffers[view.bufferIndex]; + ArrayView bytes; + if (const auto* array = std::get_if(&buffer.data)) + bytes = {reinterpret_cast(array->bytes.data()), array->bytes.size()}; + else if (const auto* byte_view = std::get_if(&buffer.data)) + bytes = {reinterpret_cast(byte_view->bytes.data()), byte_view->bytes.size()}; + const size_t available = std::min(bytes.size(), buffer.byteLength); + if (!bytes.data() || view.byteOffset > available || view.byteLength > available - view.byteOffset) + return {}; + return {bytes.data() + view.byteOffset, view.byteLength}; + } + + static bool HasAccessorRange(const fastgltf::Asset& asset, size_t view_index, size_t offset, size_t count, size_t element_size, size_t stride) + { + const auto bytes = BufferViewBytes(asset, view_index); + if (!count || !element_size || stride < element_size || offset > bytes.size() || element_size > bytes.size() - offset) + return false; + // Division avoids overflow in offset + (count - 1) * stride + size. + return count - 1 <= (bytes.size() - offset - element_size) / stride; + } + + static bool HasValidAccessor(const fastgltf::Asset& asset, size_t index, fastgltf::AccessorType type) + { + if (index >= asset.accessors.size()) + return false; + const auto& accessor = asset.accessors[index]; + const auto element_size = fastgltf::getElementByteSize(accessor.type, accessor.componentType); + if (accessor.type != type || !accessor.count || !element_size || accessor.count > std::numeric_limits::max()) + return false; + if (accessor.bufferViewIndex) + { + const auto view = *accessor.bufferViewIndex; + if (view >= asset.bufferViews.size() || !HasAccessorRange(asset, view, accessor.byteOffset, accessor.count, element_size, asset.bufferViews[view].byteStride.value_or(element_size))) + return false; + } + else if (accessor.byteOffset != 0) + return false; + if (accessor.sparse) + { + const auto& sparse = *accessor.sparse; + if (sparse.count > accessor.count || (sparse.indexComponentType != fastgltf::ComponentType::UnsignedByte && sparse.indexComponentType != fastgltf::ComponentType::UnsignedShort && sparse.indexComponentType != fastgltf::ComponentType::UnsignedInt)) + return false; + const auto index_size = fastgltf::getComponentByteSize(sparse.indexComponentType); + if (!HasAccessorRange(asset, sparse.indicesBufferView, sparse.indicesByteOffset, sparse.count, index_size, index_size) || !HasAccessorRange(asset, sparse.valuesBufferView, sparse.valuesByteOffset, sparse.count, element_size, element_size)) + return false; + } + return true; + } + + static bool HasValidGltfData(const fastgltf::Asset& asset) + { + size_t vertex_count = 0, index_count = 0; + for (const auto& mesh : asset.meshes) + for (const auto& primitive : mesh.primitives) + { + if (!HasTriangleGeometry(primitive)) + continue; + const auto position = primitive.findAttribute("POSITION")->accessorIndex; + if (!HasValidAccessor(asset, position, fastgltf::AccessorType::Vec3)) + return false; + const auto count = asset.accessors[position].count; + for (const auto& attribute : primitive.attributes) + { + const auto type = attribute.name == "NORMAL" ? fastgltf::AccessorType::Vec3 : fastgltf::AccessorType::Vec2; + if (attribute.name != "NORMAL" && attribute.name != "TEXCOORD_0") + continue; + if (!HasValidAccessor(asset, attribute.accessorIndex, type) || asset.accessors[attribute.accessorIndex].count != count) + return false; + } + size_t indices = count; + if (primitive.indicesAccessor) + { + if (!HasValidAccessor(asset, *primitive.indicesAccessor, fastgltf::AccessorType::Scalar)) + return false; + const auto& accessor = asset.accessors[*primitive.indicesAccessor]; + if (accessor.componentType != fastgltf::ComponentType::UnsignedByte && accessor.componentType != fastgltf::ComponentType::UnsignedShort && accessor.componentType != fastgltf::ComponentType::UnsignedInt) + return false; + indices = accessor.count; + } + if (indices % 3 != 0 || count > std::numeric_limits::max() / 8 - vertex_count || indices > std::numeric_limits::max() - index_count) + return false; + vertex_count += count; + index_count += indices; + } + for (const auto& texture : asset.textures) + if (texture.imageIndex) + { + if (*texture.imageIndex >= asset.images.size()) + return false; + if (const auto* view = std::get_if(&asset.images[*texture.imageIndex].data); view && BufferViewBytes(asset, view->bufferViewIndex).size() == 0) + return false; + } + return true; + } + + static bool HasValidNodeGraph(const fastgltf::Asset& asset) + { + if (asset.defaultScene && *asset.defaultScene >= asset.scenes.size()) + return false; + std::vector parents(asset.nodes.size()); + for (const auto& node : asset.nodes) + { + if (node.meshIndex && *node.meshIndex >= asset.meshes.size()) + return false; + for (auto child : node.children) + if (child >= asset.nodes.size() || ++parents[child] > 1) + return false; // glTF nodes form trees, not shared-child DAGs. + } + for (const auto& scene : asset.scenes) + { + std::vector roots(asset.nodes.size()); + for (auto root : scene.nodeIndices) + { + if (root >= asset.nodes.size() || parents[root] != 0 || roots[root]) + return false; + roots[root] = true; + } + } + // Topological traversal detects cycles, including disconnected nodes, + // without recursing on untrusted scene depth. + std::vector nodes; + for (size_t i = 0; i < parents.size(); ++i) + if (parents[i] == 0) + nodes.push_back(i); + for (size_t i = 0; i < nodes.size(); ++i) + for (auto child : asset.nodes[nodes[i]].children) + if (--parents[child] == 0) + nodes.push_back(child); + return nodes.size() == asset.nodes.size(); + } + + static bool FitsScratch(const fastgltf::Asset& asset, size_t available, bool import_materials, bool import_textures, size_t& hierarchy_capacity) + { + // Upper bound every arena allocation, including hash-table rounding and + // copied strings. Keep the existing bounded scratch policy, but fail + // before custom containers can dereference a failed allocation. + auto consume = [&](size_t count, size_t width) { + if (count > available / width) + return false; + available -= count * width; + return true; + }; + using NodeMap = UnorderedHashMap; + if (!consume(1, 1024 + 3 * 16 * sizeof(NodeMap::Entry))) + return false; // minimum map capacities and allocation alignment + for (const auto& mesh : asset.meshes) + for (const auto& primitive : mesh.primitives) + if (HasTriangleGeometry(primitive)) + { + const auto vertices = asset.accessors[primitive.findAttribute("POSITION")->accessorIndex].count; + const auto indices = primitive.indicesAccessor ? asset.accessors[*primitive.indicesAccessor].count : vertices; + if (!consume(vertices, 8 * sizeof(float)) || !consume(indices, sizeof(uint32_t)) || !consume(1, sizeof(AssetSubMesh))) + return false; + } + if (import_materials) + { + if (!consume(asset.materials.size(), sizeof(AssetMaterial) + sizeof(String) + (import_textures ? 5 * MAX_FILE_PATH_COUNT : 0))) + return false; + for (const auto& material : asset.materials) + if (!consume(2, std::max(material.name.size(), size_t(8)) + 1)) + return false; + if (import_textures) + { + if (!consume(asset.textures.size(), sizeof(AssetTexture) + MAX_FILE_PATH_COUNT + 512)) + return false; + for (const auto& texture : asset.textures) + if (texture.imageIndex && !consume(1, asset.images[*texture.imageIndex].name.size() + 1)) + return false; + } + } + const size_t node_bytes = sizeof(Helpers::NodeHierarchy) + 2 * sizeof(Mat4f) + sizeof(String) + 3 * 4 * sizeof(NodeMap::Entry); + hierarchy_capacity = 0; + std::vector pending; + auto count_scene = [&](const fastgltf::Scene& scene) { + pending.assign(scene.nodeIndices.begin(), scene.nodeIndices.end()); + while (!pending.empty()) + { + const auto& node = asset.nodes[pending.back()]; + pending.pop_back(); + if (!consume(1, node_bytes) || !consume(1, std::max(node.name.size(), size_t(6)) + 1)) + return false; + ++hierarchy_capacity; + if (node.meshIndex) + { + const auto& mesh = asset.meshes[*node.meshIndex]; + for (const auto& primitive : mesh.primitives) + if (HasTriangleGeometry(primitive)) + { + if (!consume(1, node_bytes) || !consume(1, std::max(mesh.name.size(), size_t(6)) + 1)) + return false; + ++hierarchy_capacity; + } + } + pending.insert(pending.end(), node.children.begin(), node.children.end()); + } + return true; + }; + if (asset.defaultScene) + return count_scene(asset.scenes[*asset.defaultScene]); + for (const auto& scene : asset.scenes) + if (!count_scene(scene)) + return false; + return true; + } + + static bool ExtractMeshes(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, AssetMesh& out) { uint32_t total_verts = 0; uint32_t total_indices = 0; @@ -88,7 +306,7 @@ namespace ZEngine::Importers { for (const auto& prim : mesh.primitives) { - if (prim.type != fastgltf::PrimitiveType::Triangles) + if (!HasTriangleGeometry(prim)) continue; auto it = prim.findAttribute("POSITION"); if (it != prim.attributes.end()) @@ -113,7 +331,7 @@ namespace ZEngine::Importers { for (const auto& prim : mesh.primitives) { - if (prim.type != fastgltf::PrimitiveType::Triangles) + if (!HasTriangleGeometry(prim)) continue; auto pos_it = prim.findAttribute("POSITION"); @@ -165,7 +383,13 @@ namespace ZEngine::Importers { const auto& idx_acc = asset.accessors[prim.indicesAccessor.value()]; ic = (uint32_t) idx_acc.count; - fastgltf::iterateAccessorWithIndex(asset, idx_acc, [&](uint32_t idx, std::size_t) { out.Indices.push(idx); }); + bool valid_indices = true; + fastgltf::iterateAccessorWithIndex(asset, idx_acc, [&](uint32_t idx, std::size_t) { + valid_indices = valid_indices && idx < vc; + out.Indices.push(idx); + }); + if (!valid_indices) + return false; } else { @@ -189,6 +413,7 @@ namespace ZEngine::Importers index_offset += ic; } } + return true; } static void ExtractMaterials(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, uuid_random_generator& gen, Array& out) @@ -286,87 +511,90 @@ namespace ZEngine::Importers } } - static void TraverseNode(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, std::size_t node_index, AssetNodeHierarchy& hier, const Array& mats, int parent_id, int depth) + static void TraverseNodes(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, const fastgltf::Scene& scene, AssetNodeHierarchy& hier, const Array& mats, const std::vector& mesh_offsets) { - const fastgltf::Node& node = asset.nodes[node_index]; - int id = AddNode(hier, parent_id, depth); - hier.NodeNames[id] = (uint32_t) hier.Names.size(); - auto& name = hier.Names.push_use({}); - name.init(arena, node.name.empty() ? "" : node.name.c_str()); - - hier.LocalTransforms[id] = NodeLocalTransform(node); - hier.GlobalTransforms[id] = Identity(); - - if (node.meshIndex.has_value()) + struct PendingNode { - std::size_t mesh_idx = node.meshIndex.value(); - const auto& mesh = asset.meshes[mesh_idx]; - for (std::size_t p = 0; p < mesh.primitives.size(); ++p) + size_t Index; + int Parent; + int Depth; + }; + std::vector pending; + for (auto root = scene.nodeIndices.rbegin(); root != scene.nodeIndices.rend(); ++root) + pending.push_back({*root, -1, 0}); + while (!pending.empty()) + { + const auto [node_index, parent_id, depth] = pending.back(); + pending.pop_back(); + const fastgltf::Node& node = asset.nodes[node_index]; + int id = AddNode(hier, parent_id, depth); + hier.NodeNames[id] = (uint32_t) hier.Names.size(); + auto& name = hier.Names.push_use({}); + name.init(arena, node.name.empty() ? "" : node.name.c_str()); + + hier.LocalTransforms[id] = NodeLocalTransform(node); + hier.GlobalTransforms[id] = Identity(); + + if (node.meshIndex.has_value()) { - int sub_id = AddNode(hier, id, depth + 1); - hier.NodeNames[sub_id] = (uint32_t) hier.Names.size(); - auto& sub_name = hier.Names.push_use({}); - sub_name.init(arena, mesh.name.empty() ? "" : mesh.name.c_str()); - - hier.NodeMeshes[sub_id] = (uint32_t) mesh_idx; - hier.LocalTransforms[sub_id] = Identity(); - hier.GlobalTransforms[sub_id] = Identity(); - - if (mesh.primitives[p].materialIndex.has_value()) + std::size_t mesh_idx = node.meshIndex.value(); + const auto& mesh = asset.meshes[mesh_idx]; + uint32_t submesh_index = mesh_offsets[mesh_idx]; + for (std::size_t p = 0; p < mesh.primitives.size(); ++p) { - uint32_t mat_idx = (uint32_t) mesh.primitives[p].materialIndex.value(); - hier.NodeMaterials[sub_id] = mat_idx; - if (mat_idx < mats.size()) + if (!HasTriangleGeometry(mesh.primitives[p])) + continue; + int sub_id = AddNode(hier, id, depth + 1); + hier.NodeNames[sub_id] = (uint32_t) hier.Names.size(); + auto& sub_name = hier.Names.push_use({}); + sub_name.init(arena, mesh.name.empty() ? "" : mesh.name.c_str()); + + hier.NodeMeshes[sub_id] = submesh_index++; + hier.LocalTransforms[sub_id] = Identity(); + hier.GlobalTransforms[sub_id] = Identity(); + + if (mesh.primitives[p].materialIndex.has_value()) { - AssetMesh dummy{}; - // wire material UUID into the corresponding sub-mesh - // (done outside traverse in BuildHierarchy below) - (void) dummy; + uint32_t mat_idx = (uint32_t) mesh.primitives[p].materialIndex.value(); + if (mat_idx < mats.size()) + hier.NodeMaterials[sub_id] = mat_idx; } } } - } - for (std::size_t child : node.children) - TraverseNode(arena, asset, child, hier, mats, id, depth + 1); + for (auto child = node.children.rbegin(); child != node.children.rend(); ++child) + pending.push_back({*child, id, depth + 1}); + } } - static void BuildHierarchy(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, uuid_random_generator& gen, AssetNodeHierarchy& hier, AssetMesh& mesh, const Array& mats) + static void BuildHierarchy(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, uuid_random_generator& gen, AssetNodeHierarchy& hier, AssetMesh& mesh, const Array& mats, size_t capacity) { hier.NodeHierarchyUUID = gen(); hier.MeshUUID = mesh.MeshUUID; - uint32_t cap = 3000; - hier.Hierarchies.init(arena, cap); - hier.LocalTransforms.init(arena, cap); - hier.GlobalTransforms.init(arena, cap); - hier.Names.init(arena, cap); - hier.NodeNames.init(arena, cap); - hier.NodeMeshes.init(arena, cap); - hier.NodeMaterials.init(arena, cap); + hier.Hierarchies.init(arena, capacity); + hier.LocalTransforms.init(arena, capacity); + hier.GlobalTransforms.init(arena, capacity); + hier.Names.init(arena, capacity); + hier.NodeNames.init(arena, 2 * capacity); + hier.NodeMeshes.init(arena, 2 * capacity); + hier.NodeMaterials.init(arena, 2 * capacity); hier.MaterialNames.init(arena, (uint32_t) mats.size(), (uint32_t) mats.size()); for (uint32_t m = 0; m < (uint32_t) mats.size(); ++m) hier.MaterialNames[m].init(arena, mats[m].Name.c_str()); - if (asset.defaultScene.has_value()) - { - for (std::size_t n : asset.scenes[asset.defaultScene.value()].nodeIndices) - TraverseNode(arena, asset, n, hier, mats, -1, 0); - } - else - { - for (const auto& scene : asset.scenes) - for (std::size_t n : scene.nodeIndices) - TraverseNode(arena, asset, n, hier, mats, -1, 0); - } - - // Wire material UUIDs into sub-meshes - uint32_t sub_idx = 0; + // Use the same primitive filtering as extraction. These offsets also map + // hierarchy nodes to flattened submeshes across multi-primitive meshes. + std::vector mesh_offsets(asset.meshes.size()); + uint32_t sub_idx = 0; for (std::size_t mi = 0; mi < asset.meshes.size(); ++mi) { const auto& fmesh = asset.meshes[mi]; + mesh_offsets[mi] = sub_idx; for (std::size_t p = 0; p < fmesh.primitives.size(); ++p) { + if (!HasTriangleGeometry(fmesh.primitives[p])) + continue; if (sub_idx >= mesh.SubMeshes.size()) break; if (fmesh.primitives[p].materialIndex.has_value()) @@ -378,6 +606,15 @@ namespace ZEngine::Importers ++sub_idx; } } + if (asset.defaultScene.has_value()) + { + TraverseNodes(arena, asset, asset.scenes[*asset.defaultScene], hier, mats, mesh_offsets); + } + else + { + for (const auto& scene : asset.scenes) + TraverseNodes(arena, asset, scene, hier, mats, mesh_offsets); + } } void GltfImporter::Initialize(Core::Memory::ArenaAllocator* arena) @@ -394,9 +631,10 @@ namespace ZEngine::Importers Core::VFS::VFSResult GltfImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { + std::lock_guard import_lock(m_import_mutex); // Resolve VFS path to native filesystem path - char native[MAX_FILE_PATH_COUNT] = {}; - const char* ws = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + char native[MAX_FILE_PATH_COUNT] = {}; + const char* ws = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; if (ws && ws[0] != '\0') path.ResolveNative(ws, native, sizeof(native)); else @@ -411,7 +649,7 @@ namespace ZEngine::Importers return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } - auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages | fastgltf::Options::GenerateMeshIndices); + auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages); if (result.error() != fastgltf::Error::None) { @@ -419,10 +657,21 @@ namespace ZEngine::Importers return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } - fastgltf::Asset& asset = result.get(); + fastgltf::Asset& asset = result.get(); + if (!HasValidGltfData(asset) || !HasValidNodeGraph(asset)) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::Corrupted); + size_t hierarchy_capacity = 0; + if (!FitsScratch(asset, ZMega(32), true, true, hierarchy_capacity)) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::OutOfMemory); Core::Memory::ArenaAllocator scratch{}; Arena.CreateSubArena(ZMega(32), &scratch, "ImportPipeline/GltfImporter/RuntimeScratch"); + if (!scratch.Allocate(ZMega(32))) + { + Arena.Clear(); + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::OutOfMemory); + } + scratch.Clear(); std::random_device rd; std::mt19937 generator(rd()); @@ -433,12 +682,16 @@ namespace ZEngine::Importers Array materials = {}; Array textures = {}; - ExtractMeshes(&scratch, asset, mesh); + if (!ExtractMeshes(&scratch, asset, mesh)) + { + Arena.Clear(); + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::Corrupted); + } mesh.MeshUUID = meta.AssetUUID; ExtractMaterials(&scratch, asset, gen, materials); ExtractTextures(&scratch, asset, gen, textures, materials); - BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials); + BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials, hierarchy_capacity); auto* mgr = Managers::AssetManager::Instance(); if (mgr) @@ -467,34 +720,33 @@ namespace ZEngine::Importers void GltfImporter::ImportFile(const char* filename, const AssetCodec::ImportConfiguration& cfg, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log) { + std::lock_guard import_lock(m_import_mutex); + auto mesh_path = AssetCodec::ValidateImportConfiguration(cfg); + if (mesh_path.Failed() || !filename || !filename[0]) + { + if (on_error) + on_error(context, fmt::format("Invalid import configuration (VFS error {})", static_cast(mesh_path.Failed() ? mesh_path.Error() : Core::VFS::VFSError::InvalidPath))); + return; + } // The caller's arena is sized only for a few short path strings (#760) — carve a // scratch sub-arena from this importer's own, generously-sized private Arena - // instead, matching the pattern Import() already uses for hot-reload. Declared - // once here (rather than down at the old ExtractMeshes call site) so the config - // copy below and the final serialized outputs share the same backing memory - // instead of each getting an independent, aliasing CreateSubArena carve-out. + // instead, matching the pattern Import() already uses for hot-reload. + // Intermediate geometry uses this scratch arena; output paths own their + // strings independently and remain valid throughout the completion callback. Core::Memory::ArenaAllocator scratch{}; Arena.CreateSubArena(ZMega(32), &scratch, "ImportPipeline/GltfImporter/EditorScratch"); - arena = &scratch; - - // Build arena-allocated config copy (same pattern as AssimpImporter::ImportFile) - AssetCodec::ImportConfiguration config = {}; - config.OutputWorkingSpacePath.init(arena, cfg.OutputWorkingSpacePath.c_str()); - config.OutputTextureFilesPath.init(arena, cfg.OutputTextureFilesPath.c_str()); - config.OutputAssetsPath.init(arena, cfg.OutputAssetsPath.c_str()); - config.OutputMaterialPath.init(arena, cfg.OutputMaterialPath.c_str()); - config.AssetName.init(arena, cfg.AssetName.c_str()); - config.OutputAssetFile.init(arena, cfg.OutputAssetFile.c_str()); - config.InputBaseAssetFilePath.init(arena, cfg.InputBaseAssetFilePath.c_str()); - config.VFS = cfg.VFS; - config.Options = cfg.Options; - - auto fs_path = std::filesystem::path(filename); - auto buf = fastgltf::GltfDataBuffer::FromPath(fs_path); + arena = &scratch; + + // Config remains borrowed throughout this synchronous call; no scratch copy. + const auto& config = cfg; + + auto fs_path = std::filesystem::path(filename); + auto buf = fastgltf::GltfDataBuffer::FromPath(fs_path); if (buf.error() != fastgltf::Error::None) { if (on_error) on_error(context, fastgltf::getErrorMessage(buf.error())); + Arena.Clear(); return; } @@ -502,19 +754,37 @@ namespace ZEngine::Importers on_progress(context, 0.1f); fastgltf::Parser parser; - auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages | fastgltf::Options::GenerateMeshIndices); + auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages); if (result.error() != fastgltf::Error::None) { if (on_error) on_error(context, fastgltf::getErrorMessage(result.error())); + Arena.Clear(); return; } if (on_progress) on_progress(context, 0.3f); - fastgltf::Asset& asset = result.get(); + fastgltf::Asset& asset = result.get(); + if (!HasValidGltfData(asset) || !HasValidNodeGraph(asset)) + { + if (on_error) + on_error(context, "Invalid glTF buffer range, accessor, triangle count, or node graph"); + Arena.Clear(); + return; + } + + size_t hierarchy_capacity = 0; + if (!FitsScratch(asset, scratch.m_total_size, config.Options.ImportMaterials, config.Options.ImportMaterials && config.Options.ImportTextures, hierarchy_capacity) || !scratch.Allocate(scratch.m_total_size)) + { + if (on_error) + on_error(context, "glTF exceeds the importer scratch memory budget or memory could not be committed"); + Arena.Clear(); + return; + } + scratch.Clear(); std::random_device rd; std::mt19937 gen_mt(rd()); @@ -525,7 +795,13 @@ namespace ZEngine::Importers Array materials = {}; Array textures = {}; - ExtractMeshes(&scratch, asset, mesh); + if (!ExtractMeshes(&scratch, asset, mesh)) + { + if (on_error) + on_error(context, "Invalid glTF triangle vertex index"); + Arena.Clear(); + return; + } mesh.MeshUUID = gen(); // Stabilize the mesh UUID (#762): re-importing the same destination path must @@ -533,11 +809,14 @@ namespace ZEngine::Importers // update to an existing mesh" — every re-cook would otherwise mint a fresh // random UUID and look like a brand new, unrelated asset. { - auto mesh_dir = Core::VFS::VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mesh_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - mesh.MeshUUID = meta_result.Value().AssetUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mesh_path.Value(), mesh.MeshUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read mesh metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } // Apply per-vertex transform options @@ -575,26 +854,12 @@ namespace ZEngine::Importers // Optimize each submesh: vertex cache, overdraw, vertex fetch. for (uint32_t si = 0; si < mesh.SubMeshes.size(); ++si) { - auto& sub = mesh.SubMeshes[si]; - uint32_t* sub_idx = mesh.Indices.data() + sub.IndexOffset; - - // Cross-primitive index contamination or a malformed GLTF can leave an index - // outside this submesh's own vertex range. Rebasing such an index underflows - // (uint32_t subtraction), which meshopt::buildTriangleAdjacency asserts on. - // Skip optimization for this submesh rather than crash — indices are left - // untouched (still correct against the unmodified vertex buffer). - bool in_range = true; - for (uint32_t j = 0; j < sub.IndexCount && in_range; ++j) - if (sub_idx[j] < sub.VertexOffset || sub_idx[j] >= sub.VertexOffset + sub.VertexCount) - in_range = false; - if (!in_range) - continue; - - for (uint32_t j = 0; j < sub.IndexCount; ++j) - sub_idx[j] -= sub.VertexOffset; + auto& sub = mesh.SubMeshes[si]; + uint32_t* sub_idx = mesh.Indices.data() + sub.IndexOffset; + + // glTF indices are already local to the primitive's vertex slice. + // Extraction validated both the range and triangle count. Importers::OptimizeMeshSubmesh(mesh.Vertices.data() + sub.VertexOffset * 8, sub.VertexCount, sub_idx, sub.IndexCount); - for (uint32_t j = 0; j < sub.IndexCount; ++j) - sub_idx[j] += sub.VertexOffset; } if (config.Options.ImportMaterials) @@ -608,36 +873,41 @@ namespace ZEngine::Importers // which reads MaterialUUID by reference off this same materials array. for (size_t m = 0; m < materials.size(); ++m) { - auto mat_dir = Core::VFS::VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - std::string mat_filename = fmt::format("{}{}", materials[m].Name.c_str(), ".zematerial"); - auto mat_path = mat_dir / mat_filename.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mat_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - materials[m].MaterialUUID = meta_result.Value().AssetUUID; + std::string mat_filename = AssetCodec::MaterialOutputFilename(materials[m], m); + auto mat_path = AssetCodec::MaterialOutputPath(materials[m], config, m); + if (mat_path.Failed()) + { + if (on_error) + on_error(context, fmt::format("Invalid material output '{}' (VFS error {})", mat_filename, static_cast(mat_path.Error()))); + Arena.Clear(); + return; + } + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mat_path.Value(), materials[m].MaterialUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read material metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } if (config.Options.ImportTextures) ExtractTextures(&scratch, asset, gen, textures, materials); } - BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials); + BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials, hierarchy_capacity); // Extract texture image bytes to disk and record project-relative paths if (config.Options.ImportTextures && config.Options.ImportMaterials) { - char dest_dir_buf[MAX_FILE_PATH_COUNT] = {}; - (ZEngine::Core::VFS::VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()).ResolveNative(config.OutputWorkingSpacePath.c_str(), dest_dir_buf, sizeof(dest_dir_buf)); - config.VFS->CreateDir(ZEngine::Core::VFS::VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()); - - // Pre-resolve base pointers for every buffer — for a GLB all images share - // buffer 0 (the binary chunk). Resolving once avoids repeated get_if per texture. - std::vector buf_ptrs(asset.buffers.size(), nullptr); - for (size_t i = 0; i < asset.buffers.size(); ++i) + auto texture_dir = ZEngine::Core::VFS::VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str(); + auto create_dir = config.VFS->CreateDir(texture_dir); + if (create_dir.Failed() && create_dir.Error() != ZEngine::Core::VFS::VFSError::AlreadyExists) { - const auto& bd = asset.buffers[i].data; - if (const auto* arr = std::get_if(&bd)) - buf_ptrs[i] = reinterpret_cast(arr->bytes.data()); - else if (const auto* bvd = std::get_if(&bd)) - buf_ptrs[i] = reinterpret_cast(bvd->bytes.data()); + if (on_error) + on_error(context, "Unable to create the texture output directory"); + Arena.Clear(); + return; } for (size_t tex_idx = 0; tex_idx < textures.size() && tex_idx < asset.textures.size(); ++tex_idx) @@ -669,15 +939,11 @@ namespace ZEngine::Importers } else if (const auto* bv_src = std::get_if(&img.data)) { - const auto& bv = asset.bufferViews[bv_src->bufferViewIndex]; - const uint8_t* base = bv.bufferIndex < buf_ptrs.size() ? buf_ptrs[bv.bufferIndex] : nullptr; - if (base) - { - bytes = base + bv.byteOffset; - nbytes = bv.byteLength; - if (bv_src->mimeType == fastgltf::MimeType::JPEG) - ext = ".jpg"; - } + const auto view = BufferViewBytes(asset, bv_src->bufferViewIndex); + bytes = view.data(); + nbytes = view.size(); + if (bv_src->mimeType == fastgltf::MimeType::JPEG) + ext = ".jpg"; } else if (const auto* uri_src = std::get_if(&img.data)) { @@ -696,9 +962,11 @@ namespace ZEngine::Importers if (file_sz > 0) { uri_file_buf.resize(static_cast(file_sz)); - fread(uri_file_buf.data(), 1, uri_file_buf.size(), src_file); - bytes = uri_file_buf.data(); - nbytes = uri_file_buf.size(); + if (fread(uri_file_buf.data(), 1, uri_file_buf.size(), src_file) == uri_file_buf.size()) + { + bytes = uri_file_buf.data(); + nbytes = uri_file_buf.size(); + } // Derive extension from URI path if (path_sv.size() >= 4) { @@ -743,8 +1011,10 @@ namespace ZEngine::Importers if (!bytes || nbytes == 0) { - ZENGINE_LOG_ASSET_WARN("GltfImporter: tex {} image data not accessible (unsupported source variant)", tex_idx) - continue; + if (on_error) + on_error(context, fmt::format("Unable to read texture {} image data", tex_idx)); + Arena.Clear(); + return; } // Build output filename — prefer image name, then URI filename, then tex_N @@ -764,57 +1034,36 @@ namespace ZEngine::Importers } else { - snprintf(stem_buf, sizeof(stem_buf), "tex_%zu", tex_idx); + Helpers::secure_strcpy(stem_buf, sizeof(stem_buf), "image"); } - char out_path_buf[MAX_FILE_PATH_COUNT] = {}; - snprintf(out_path_buf, sizeof(out_path_buf), "%s/%s%s", dest_dir_buf, stem_buf, ext); - - // fwrite is faster than std::ofstream for plain binary blobs - if (FILE* f = fopen(out_path_buf, "wb")) + // Image names are optional/non-unique. The source image index + // preserves identity while textures sharing an image share a file. + const auto output_name = fmt::format("{}_{}{}", stem_buf, *fgltf_tex.imageIndex, ext); + auto output_path = AssetCodec::MakeOutputPath(texture_dir.CStr(), output_name.c_str()); + auto write = output_path.Succeeded() ? Core::VFS::WriteFileAtomically(*config.VFS, output_path.Value(), {bytes, nbytes}) : Core::VFS::VFSResult::Fail(output_path.Error()); + if (write.Succeeded()) { - fwrite(bytes, 1, nbytes, f); - fclose(f); - ZENGINE_LOG_ASSET_INFO("GltfImporter: extracted texture '{}' ({} bytes)", out_path_buf, nbytes) - - char rel_buf[MAX_FILE_PATH_COUNT] = {}; - snprintf(rel_buf, sizeof(rel_buf), "%s/%s/%s%s", config.OutputTextureFilesPath.c_str(), config.AssetName.c_str(), stem_buf, ext); - textures[tex_idx].Path.init(&scratch, rel_buf); + ZENGINE_LOG_ASSET_INFO("GltfImporter: extracted texture '{}' ({} bytes)", output_path.Value().CStr(), nbytes) + AssetCodec::ReportPublishedArtifact(config, AssetFileType::TEXTURES, output_path.Value()); + textures[tex_idx].Path.init(&scratch, output_path.Value().CStr()); + } + else + { + if (on_error) + on_error(context, fmt::format("Failed to write '{}/{}' (VFS error {})", texture_dir.CStr(), output_name, static_cast(write.Error()))); + Arena.Clear(); + return; } } - // Propagate tex.Path → material.*TexPath by UUID match, replacing - // ExtractTextures' throwaway random UUID (#755) with a stable, - // content-hashed one. - for (size_t m = 0; m < materials.size(); ++m) + auto metadata = AssetCodec::SynchronizeTextureMetadata(arena, ArrayView{textures}, ArrayView{materials}, config, "GltfImporter"); + if (metadata.Failed()) { - auto sync_texture = [&](uuids::uuid& uuid_field, Core::Containers::String& path_out) { - for (size_t t = 0; t < textures.size(); ++t) - { - if (textures[t].TextureUUID == uuid_field && !textures[t].Path.empty()) - { - path_out.init(&scratch, textures[t].Path.c_str()); - - auto vfs_path = Core::VFS::VFSPath::Parse(textures[t].Path.c_str()); - if (vfs_path.Succeeded() && config.VFS) - { - auto hash_result = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, vfs_path.Value()); - auto meta_result = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, vfs_path.Value(), "GltfImporter", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta_result.Succeeded()) - { - textures[t].TextureUUID = meta_result.Value().AssetUUID; - uuid_field = meta_result.Value().AssetUUID; - } - } - return; - } - } - }; - sync_texture(materials[m].AlbedoTexUUID, materials[m].AlbedoTexPath); - sync_texture(materials[m].EmissiveTexUUID, materials[m].EmissiveTexPath); - sync_texture(materials[m].NormalTexUUID, materials[m].NormalTexPath); - sync_texture(materials[m].OpacityTexUUID, materials[m].OpacityTexPath); - sync_texture(materials[m].SpecularTexUUID, materials[m].SpecularTexPath); + if (on_error) + on_error(context, fmt::format("Failed to persist texture metadata (VFS error {})", static_cast(metadata.Error()))); + Arena.Clear(); + return; } } @@ -822,12 +1071,37 @@ namespace ZEngine::Importers on_progress(context, 0.7f); // Serialize to disk — .zemesh + .zematerial (no .zetextures: paths are inline in material) - Array outputs = {}; - outputs.init(arena, 16); - outputs.push(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchy, config)); + // AssetImporterOutput owns std::strings and must use constructed storage. + std::vector outputs; + outputs.reserve(1 + materials.size()); + auto append_output = [&](Core::VFS::VFSResult result, const char* directory, const std::string& filename) { + if (result.Failed()) + { + const auto message = fmt::format("Failed to write '{}/{}' (VFS error {})", directory, filename, static_cast(result.Error())); + ZENGINE_CORE_ERROR("[GltfImporter] {}", message) + if (on_error) + on_error(context, message); + return false; + } + outputs.push_back(std::move(result.Value())); + return true; + }; if (config.Options.ImportMaterials) for (size_t i = 0; i < materials.size(); ++i) - outputs.push(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config)); + if (!append_output(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config, i), config.OutputMaterialPath.c_str(), fmt::format("{}/{}", config.AssetName.c_str(), AssetCodec::MaterialOutputFilename(materials[i], i)))) + { + Arena.Clear(); + return; + } + + // Publish the mesh only after all referenced dependencies were written. + // Preserve the mesh-first completion callback ordering. + if (!append_output(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchy, config), config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str())) + { + Arena.Clear(); + return; + } + std::rotate(outputs.begin(), outputs.end() - 1, outputs.end()); auto* mgr = Managers::AssetManager::Instance(); if (mgr) @@ -844,7 +1118,7 @@ namespace ZEngine::Importers on_progress(context, 1.0f); if (on_complete) - on_complete(context, ArrayView{outputs}); + on_complete(context, {outputs.data(), outputs.size()}); Arena.Clear(); } diff --git a/ZEngine/ZEngine/Importers/GltfImporter.h b/ZEngine/ZEngine/Importers/GltfImporter.h index 885b0e30..a4386b23 100644 --- a/ZEngine/ZEngine/Importers/GltfImporter.h +++ b/ZEngine/ZEngine/Importers/GltfImporter.h @@ -3,13 +3,14 @@ #include #include #include +#include #include namespace ZEngine::Importers { // Handles GLB and GLTF import via fastgltf. - // Stateless across concurrent imports — each Import() call carves a scratch - // sub-arena from the importer's own Arena for intermediate geometry data. + // Shared scratch is serialized across runtime and editor imports to preserve + // the bounded arena budget without allowing one import to rewind another. class GltfImporter : public IAssetImporter { public: @@ -25,5 +26,8 @@ namespace ZEngine::Importers void ImportFile(const char* filename, const AssetCodec::ImportConfiguration& config, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log); Core::Memory::ArenaAllocator Arena = {}; + + private: + std::mutex m_import_mutex; }; } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/IAssetImporter.h b/ZEngine/ZEngine/Importers/IAssetImporter.h index bca04030..394ca577 100644 --- a/ZEngine/ZEngine/Importers/IAssetImporter.h +++ b/ZEngine/ZEngine/Importers/IAssetImporter.h @@ -10,7 +10,8 @@ namespace ZEngine::Importers int AddNode(AssetNodeHierarchy& hierarchy, int parent, int depth); // Lightweight per-format importer interface used by ImportCoordinator. - // Implementations must be stateless — a single instance handles concurrent imports. + // A single instance handles concurrent calls. Implementations with shared scratch + // storage must serialize its use; different importer instances remain independent. // UUID must NOT be generated inside Import(); read it from meta.AssetUUID which was // assigned by MetaFileIO at scan time and is stable across reimports. struct IAssetImporter diff --git a/ZEngine/ZEngine/Importers/TextureImporter.cpp b/ZEngine/ZEngine/Importers/TextureImporter.cpp index 1b0f21bc..d52b576a 100644 --- a/ZEngine/ZEngine/Importers/TextureImporter.cpp +++ b/ZEngine/ZEngine/Importers/TextureImporter.cpp @@ -22,6 +22,7 @@ namespace ZEngine::Importers Core::VFS::VFSResult TextureImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { + std::lock_guard import_lock(m_import_mutex); (void) ctx; // path is workspace-relative — ToNative alone only swaps separators and would diff --git a/ZEngine/ZEngine/Importers/TextureImporter.h b/ZEngine/ZEngine/Importers/TextureImporter.h index c856801a..f6060745 100644 --- a/ZEngine/ZEngine/Importers/TextureImporter.h +++ b/ZEngine/ZEngine/Importers/TextureImporter.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace ZEngine::Importers { @@ -19,5 +20,8 @@ namespace ZEngine::Importers Core::VFS::VFSResult Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) override; Core::Memory::ArenaAllocator Arena = {}; + + private: + std::mutex m_import_mutex; }; } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Logging/Logger.cpp b/ZEngine/ZEngine/Logging/Logger.cpp index c41ec1f5..d86ea3ee 100644 --- a/ZEngine/ZEngine/Logging/Logger.cpp +++ b/ZEngine/ZEngine/Logging/Logger.cpp @@ -79,7 +79,7 @@ namespace ZEngine::Logging } spdlog::init_thread_pool(8192, 2); - spdlog::flush_every(std::chrono::duration_cast(configuration.PeriodicFlush)); + spdlog::flush_every(configuration.PeriodicFlush); spdlog::flush_on(spdlog::level::critical); s_sink = std::make_shared((log_directory_path / configuration.LogFilename).string(), 1024 * 1024, 5, false); diff --git a/ZEngine/ZEngine/Managers/AssetManager.cpp b/ZEngine/ZEngine/Managers/AssetManager.cpp index b0c176a8..11d1a266 100644 --- a/ZEngine/ZEngine/Managers/AssetManager.cpp +++ b/ZEngine/ZEngine/Managers/AssetManager.cpp @@ -304,7 +304,7 @@ namespace ZEngine::Managers return; auto slot = static_cast(s_Instance->Materials.size()); - s_Instance->Materials.push(mat); + s_Instance->Materials.push(mat.Clone(s_Instance->Arena)); RegisterAsset(AssetType::MATERIAL, mat.MaterialUUID, slot); s_Instance->UUIDToMaterialSlot.insert(mat.MaterialUUID, slot); diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index 103900f0..ffd9c9cb 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -2,6 +2,7 @@ // stbi_load pixel buffers stay on the slab rather than the system heap. // Falls back to malloc/free/realloc on the main thread (slab = nullptr). #include +#include #include #include #define STBI_MALLOC(sz) (ZEngine::Helpers::GetWorkerSlab() ? ZEngine::Helpers::GetWorkerSlab()->Alloc(sz) : std::malloc(sz)) @@ -56,7 +57,7 @@ namespace ZEngine::Rendering uint16_t FloatToHalf(float value) { uint32_t bits = 0; - std::memcpy(&bits, &value, sizeof(bits)); + ZENGINE_VALIDATE_ASSERT(secure_memcpy(&bits, sizeof(bits), &value, sizeof(value)) == MEMORY_OP_SUCCESS, "FloatToHalf: failed to copy float bits") const uint16_t sign = static_cast((bits >> 16) & 0x8000u); const int32_t exponent = static_cast((bits >> 23) & 0xFFu) - 127 + 15; @@ -264,6 +265,7 @@ namespace ZEngine::Rendering while (m_pending_texture_decodes.value.load(std::memory_order_acquire) > 0) std::this_thread::yield(); + DiscardQueuedTextureDecodes(); m_device->QueueWaitAll(); ShutdownTextureTimelines(); DiscardTextureDeferrals(); @@ -315,8 +317,14 @@ namespace ZEngine::Rendering void RenderResourceManager::BeginFrame(uint32_t frame_index) { + // Initialization may leave a builtin-geometry batch open for slot 0. + // A resize or skipped acquire can make the first rendered slot different. + // Submit its copies before changing slots or retiring their staging data. + if (m_batch_mode && m_batch_frame_index != frame_index) + EndBatchUpload(); m_active_frame_index = static_cast(frame_index); RetireBatchStagings(); + DispatchQueuedTextureDecodes(); m_streaming_mgr.Tick(frame_index); if (m_streaming_mgr.IsCompactionRequested()) RunCompaction(); @@ -553,8 +561,8 @@ namespace ZEngine::Rendering ZENGINE_VALIDATE_ASSERT(m_builtin_idx_cursor + idx_bytes <= BUILTIN_IDX_CAPACITY, "RRM::RegisterBuiltinGeometry: builtin index buffer out of space") // Called pre-render-thread (single-threaded init) — the batch this opens stays - // open until the first real frame's RRM::EndFrame, where any mesh uploads from - // that frame join it too. Builtin data goes into the pinned builtin buffers, not + // open until the first real frame joins it or submits it before switching + // frame slots. Builtin data goes into the pinned builtin buffers, not // the streaming global buffers, so a ResetGeometryBuffers never corrupts it. EnsureBatchOpen(static_cast(m_active_frame_index)); AppendToGlobalBuffer(m_builtin_vertex_buf, vtx_data, vtx_bytes, m_builtin_vtx_cursor, m_active_frame_index); @@ -653,6 +661,9 @@ namespace ZEngine::Rendering for (uint32_t i = 0; i < m_batch_frames.size(); ++i) { + // LastSignal covers the previous submission, not newly recorded copies. + if (m_batch_mode && i == m_batch_frame_index) + continue; BatchFrameState& frame = m_batch_frames[i]; if (frame.StagingCount == 0) continue; @@ -667,6 +678,8 @@ namespace ZEngine::Rendering void RenderResourceManager::EnsureBatchOpen(uint8_t frame_index) { + if (m_batch_mode && m_batch_frame_index != frame_index) + EndBatchUpload(); if (!m_batch_mode) BeginBatchUpload(frame_index); } @@ -687,11 +700,8 @@ namespace ZEngine::Rendering if (frame.LastSignal != 0) m_batch_timeline->Wait(frame.LastSignal, UINT64_MAX); - // RetireBatchStagings (called earlier in BeginFrame) handles staging cleanup via - // the RenderTimeline gate — it always runs before this point. If stagings remain - // here it means RetireBatchStagings hasn't confirmed GPU completion yet (render - // timeline hasn't reached SafeRetireAfterRenderValue). Leave them for the next poll - // rather than freeing while the command buffer may still be tracked as in-use. + // RetireBatchStagings polls the batch timeline each frame. Any staging buffers + // still owned by this frame index are released only after its prior batch signal. m_batch_cmd->ResetState(); vkResetCommandBuffer(m_batch_cmd->GetHandle(), 0); m_batch_cmd->Begin(); @@ -702,17 +712,55 @@ namespace ZEngine::Rendering { m_batch_cmd->End(); - // Submission is deferred to SubmitAsyncUploads (AppRenderPipeline::EndFrame) instead - // of submitted-and-blocked-on here, so a mesh drop never stalls the render thread. + // Submit without blocking so a mesh drop never stalls the render thread. Present() + // consumes the resulting timeline operation in the graphics submission below. // Signals m_batch_timeline — a dedicated semaphore with exactly one writer (this // function) — rather than DeviceSwapchain::RenderTimeline, which Present() also // drives independently. uint64_t signal_value = ++m_batch_next_value; m_batch_frames[m_batch_frame_index].LastSignal = signal_value; - VkPipelineStageFlags2 wait_flag = VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; - m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, wait_flag, signal_value, UINT64_MAX, nullptr); - m_device->EnqueueAsyncGPUOperation({wait_flag, signal_value, m_batch_timeline}); + // The global geometry buffers are bound as whole-buffer storage descriptors. + // A fresh copy therefore needs to wait for every prior graphics read before + // writing, even when its byte range is newly allocated: validation cannot infer + // the shader's per-draw subrange, and separate graphics queues may overlap. + // + // RenderTimeline covers the previous frame's final Present() submission. Earlier + // graphics batches are submitted directly by RenderGraph, however. Relaying those + // batches through RenderTimeline preserves execution order but not their shader + // access scope, so the copy must wait on their direct timeline as well. + auto* const swapchain = m_device->SwapchainPtr; + constexpr auto copy_wait_stage = VK_PIPELINE_STAGE_2_TRANSFER_BIT; + // A batch can contain generic UpdateBuffer copies as well as mesh data. Its + // consumer queue is therefore not known here: it can be graphics or a + // dedicated compute queue. ALL_COMMANDS is valid for either queue and keeps + // every possible consumer behind the copy. + constexpr auto consumer_wait_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; + + VkSemaphoreSubmitInfo copy_waits[2] = {}; + uint32_t copy_wait_count = 0; + if (swapchain->RenderTimelineNextValue != 0) + { + copy_waits[copy_wait_count++] = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = swapchain->RenderTimeline->GetHandle(), + .value = swapchain->RenderTimelineNextValue, + .stageMask = copy_wait_stage, + }; + } + + if (swapchain->DirectGraphicsTimeline && swapchain->DirectGraphicsTimelineValue != 0) + { + copy_waits[copy_wait_count++] = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = swapchain->DirectGraphicsTimeline->GetHandle(), + .value = swapchain->DirectGraphicsTimelineValue, + .stageMask = copy_wait_stage, + }; + } + + m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, signal_value, copy_waits, copy_wait_count); + m_device->EnqueueAsyncGPUOperation({consumer_wait_stage, signal_value, m_batch_timeline}); // Left in m_batch_frames[m_batch_frame_index] for RetireBatchStagings (or the next // BeginBatchUpload for this same frame index) to free once m_batch_timeline proves @@ -1582,6 +1630,8 @@ namespace ZEngine::Rendering if (!ProcessTextureDeferral(frame_index, deferral)) m_tex_deferral_retry.push(deferral); } + + DispatchQueuedTextureDecodes(); } void RenderResourceManager::SubmitAsyncUploads() @@ -1687,6 +1737,14 @@ namespace ZEngine::Rendering { m_async_uploads.Clear(); m_device->AsyncGPUOperations.clear(); + + // Geometry batches are submitted immediately, unlike the cancellable + // texture jobs above. Keep their semaphore dependency across recreation: + // an OUT_OF_DATE acquire has no Present() submission to relay it to the + // next frame's graph. + if (m_batch_timeline && m_batch_next_value != 0) + m_device->EnqueueAsyncGPUOperation({VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, m_batch_next_value, m_batch_timeline}); + m_streaming_upload_tickets.clear(); } @@ -1893,38 +1951,63 @@ namespace ZEngine::Rendering if (!tex_handle.Valid()) return {}; - uint8_t decode_slab_index = UINT8_MAX; - if (!TryAcquireTextureDecodeSlab(&decode_slab_index)) - { - if (track_decode) - PublishTextureDecodeCompletion(tex_handle, false); - if (!existing.Valid()) - m_device->DestroyTexture(tex_handle); - ZENGINE_CORE_WARN("[RRM] Texture decode capacity ({}) reached — rejecting {}", MAX_CONCURRENT_TEXTURE_DECODES, filename) - return existing; - } - auto* task = static_cast(m_texture_task_slab.Alloc(sizeof(TextureDecodeTask))); ZConstruct(task, TextureDecodeTask); task->Owner = this; task->Specification = spec; task->Texture = tex_handle; - task->DecodeSlabIndex = decode_slab_index; task->IsEnvironmentMap = is_environment_map; task->TrackCompletion = track_decode; Helpers::secure_strcpy(task->Filename, sizeof(task->Filename), filename); - m_pending_texture_decodes.value.fetch_add(1, std::memory_order_release); - if (!Helpers::ThreadPoolHelper::Submit(task, &RenderResourceManager::RunTextureDecodeTask)) + if (!m_queued_texture_decodes.push(task)) { if (task->TrackCompletion) PublishTextureDecodeCompletion(tex_handle, false); + if (!existing.Valid()) + m_device->DestroyTexture(tex_handle); + m_texture_task_slab.Free(task); + ZENGINE_CORE_ERROR("[RRM] Texture decode queue full — rejecting {}", filename) + return existing; + } + + return tex_handle; + } + + void RenderResourceManager::DispatchQueuedTextureDecodes() + { + for (;;) + { + uint8_t decode_slab_index = UINT8_MAX; + if (!TryAcquireTextureDecodeSlab(&decode_slab_index)) + return; + + TextureDecodeTask* task = nullptr; + if (!m_queued_texture_decodes.pop(task)) + { + ReleaseTextureDecodeSlab(decode_slab_index); + return; + } + + task->DecodeSlabIndex = decode_slab_index; + m_pending_texture_decodes.value.fetch_add(1, std::memory_order_release); + if (Helpers::ThreadPoolHelper::Submit(task, &RenderResourceManager::RunTextureDecodeTask)) + continue; + + if (task->TrackCompletion) + PublishTextureDecodeCompletion(task->Texture, false); ReleaseTextureDecodeSlab(task->DecodeSlabIndex); CompleteTextureDecodeTask(task); ZENGINE_CORE_ERROR("[RRM] Texture decode rejected because the thread pool is shutting down") + return; } + } - return tex_handle; + void RenderResourceManager::DiscardQueuedTextureDecodes() + { + TextureDecodeTask* task = nullptr; + while (m_queued_texture_decodes.pop(task)) + m_texture_task_slab.Free(task); } void RenderResourceManager::RunTextureDecodeTask(void* context) @@ -2094,11 +2177,20 @@ namespace ZEngine::Rendering Rendering::Textures::TextureHandle RenderResourceManager::GetOrCreateFallbackTexture() { - static constexpr const char* kFallbackPath = "ZodiacEngine/Settings/FallbackTexture.png"; + static constexpr const char* kFallbackRelativePath = "Settings/FallbackTexture.png"; - if (!std::filesystem::exists(kFallbackPath)) + const auto* engine_context = Engine::GetContext(); + if (!engine_context || !engine_context->EngineAssetsNativeRoot) { - // 4×4 (255, 20, 147, 255) fallback color for missing textures + ZENGINE_CORE_ERROR("Fallback texture cannot be loaded because the engine asset root is unavailable") + return {}; + } + + const std::string fallback_path = (std::filesystem::path(engine_context->EngineAssetsNativeRoot) / kFallbackRelativePath).string(); + if (!std::filesystem::exists(fallback_path)) + { + // Keep GetOrCreateFallbackTexture's recovery behavior for development + // packages that were produced without the bundled fallback image. static constexpr int W = 4, H = 4; static constexpr uint8_t R = 255, G = 20, B = 147, A = 255; uint8_t pixels[W * H * 4]; @@ -2109,10 +2201,14 @@ namespace ZEngine::Rendering pixels[i * 4 + 2] = B; pixels[i * 4 + 3] = A; } - stbi_write_png(kFallbackPath, W, H, 4, pixels, W * 4); + if (!stbi_write_png(fallback_path.c_str(), W, H, 4, pixels, W * 4)) + { + ZENGINE_CORE_ERROR("Failed to create fallback texture: {}", fallback_path) + return {}; + } } - auto result = SubmitTextureFile(kFallbackPath); + auto result = SubmitTextureFile(fallback_path.c_str()); if (result.Valid()) { auto texture = m_device->GlobalTextures.Access(result); diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.h b/ZEngine/ZEngine/Rendering/RenderResourceManager.h index 18ac14b0..dfae9603 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.h +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.h @@ -24,6 +24,7 @@ // Forward declaration for the test-only helper that accesses RRM private members. struct RRMTestHelper; struct RRMDecodeTestHelper; +struct RRMUploadBatchTestHelper; namespace ZEngine::Hardwares { @@ -101,8 +102,9 @@ namespace ZEngine::Rendering /// @param frame_index Current swapchain frame index (0 .. FRAMES_IN_FLIGHT-1). void BeginFrame(uint32_t frame_index); - /// @brief Closes this frame's deferred upload batch, if anything joined it. - /// @details Called by AppRenderPipeline::EndFrame, before SubmitAsyncUploads. + /// @brief Closes the current deferred upload batch, if anything joined it. + /// @details Called by RenderGraph after its declaration phase and by + /// AppRenderPipeline::EndFrame as an idempotent fallback. void EndFrame(); /// @brief Ingest a texture file, uploading (existing invalid) or reloading it in @@ -195,8 +197,7 @@ namespace ZEngine::Rendering /// @param frame_index Render frame index to upload against. void CompleteDeferrals(uint8_t frame_index); - /// @brief Submit all pending async upload jobs (texture uploads and mesh batch - /// uploads) to their resolved GPU queues. + /// @brief Submit all pending asynchronous texture uploads to their resolved GPU queues. /// @details Called from AppRenderPipeline::EndFrame. Processes m_async_uploads. void SubmitAsyncUploads(); @@ -220,8 +221,9 @@ namespace ZEngine::Rendering /// @param thread_index Thread index within the per-frame pool. void RetireTextureSlots(uint8_t frame_index, uint8_t thread_index); - /// @brief Cancel all queued async upload jobs without submitting them. - /// @details Called on swapchain resize or recreate to discard stale uploads. + /// @brief Cancel queued texture upload jobs without submitting them. + /// @details Called on swapchain recreation. Already-submitted geometry batches + /// remain published through their timeline semaphore. void ClearAsyncUploads(); /// @brief Reset all texture timeline semaphore counters after a swapchain recreate. @@ -464,6 +466,7 @@ namespace ZEngine::Rendering static constexpr uint32_t MAX_UUID_MAP = 4096; static constexpr uint32_t MAX_PENDING = 1024; static constexpr uint32_t MAX_TEXTURE_DEFERRALS = 8192; + static constexpr uint32_t MAX_TEXTURE_TASKS = 4096; // Per-frame-index batch state: the m_batch_timeline value last signalled for that // frame index's batch, and the staging buffers still owned by it. Retired @@ -473,7 +476,6 @@ namespace ZEngine::Rendering struct BatchFrameState { uint64_t LastSignal = 0; - uint64_t SafeRetireAfterRenderValue = 0; // RenderTimeline value after which stagings are safe to free Core::Memory::BufferView StagingBuffers[MAX_PENDING * 2] = {}; uint32_t StagingCount = 0; }; @@ -524,6 +526,8 @@ namespace ZEngine::Rendering Core::Containers::MPSCQueue Completions = {}; }; + using TextureDecodeTaskQueue = Core::Containers::MPSCQueue; + /// @brief Append one mesh asset's vertex/index data to the global buffers. /// @details Shared by DoUploadMesh (new slot) and FlushPendingSwaps (reuse slot). /// Returns a zero-VtxCount MeshSlot on failure. @@ -558,22 +562,26 @@ namespace ZEngine::Rendering void BeginBatchUpload(uint8_t frame_index); void EndBatchUpload(); - /// @brief Open the batch if not already open this frame. Idempotent — every join - /// point (mesh uploads, hot-reload swaps, UpdateBuffer staging, builtin - /// registration) calls this before recording so only the first one opens it. + /// @brief Reuse this slot's open batch, or submit the previous slot's batch + /// before opening a new one. Every upload joins through this method. void EnsureBatchOpen(uint8_t frame_index); /// @brief Free frame-index batch stagings once m_batch_timeline reaches LastSignal. /// Called once per frame from BeginFrame. void RetireBatchStagings(); + /// @brief Start queued texture decodes while a bounded decode slab is available. + /// @details Render-thread only. Queued requests preserve their texture handles; + /// they wait here rather than being replaced with the fallback texture. + void DispatchQueuedTextureDecodes(); + void DiscardQueuedTextureDecodes(); + void ResetGeometryBuffersInternal(); /// @brief Re-pack all Resident mesh regions from offset 0, eliminating fragmentation holes. /// @details Resets the pool and re-uploads every Resident mesh's CPU asset data into /// fresh batch regions using the existing AppendMeshData path. Slot regions are - /// updated in place before RenderScene runs. The batch is submitted by - /// EndFrame/SubmitAsyncUploads; Present()'s m_batch_timeline wait ensures the - /// render submission sees the new data before drawing. + /// updated in place before RenderScene runs. EndFrame submits the batch, and + /// Present() waits on m_batch_timeline before drawing the new data. void RunCompaction(); void InitUploadPool(); void InitGlobalBuffers(); @@ -590,6 +598,7 @@ namespace ZEngine::Rendering friend class GeometryStreamingManager; friend struct ::RRMTestHelper; // test-only — grants slot state access to streaming manager tests friend struct ::RRMDecodeTestHelper; + friend struct ::RRMUploadBatchTestHelper; Hardwares::VulkanDevice* m_device = nullptr; Core::VFS::AssetRegistry* m_registry = nullptr; @@ -597,8 +606,7 @@ namespace ZEngine::Rendering // Dedicated command buffer manager for geometry/font-atlas uploads, separate from // Device->CommandBufferMgr. Regular pool slot 0 serves the two remaining // synchronous callers (UploadFontAtlas, UpdateBuffer's ring path); the instant - // pool serves BeginBatchUpload/EndBatchUpload, whose submission is deferred to - // SubmitAsyncUploads instead of blocked on. + // pool serves BeginBatchUpload/EndBatchUpload, submitted without a CPU wait. Hardwares::CommandBufferManagerPtr m_upload_cmd_mgr = {}; // The only two remaining synchronous, fence-blocking uploads (UploadFontAtlas and // UpdateBuffer's ring path) are both render-thread-only and always fully block @@ -642,6 +650,11 @@ namespace ZEngine::Rendering PaddedAtomic m_pending_texture_decodes = {}; PaddedAtomic m_accept_texture_decodes = {}; + // SubmitTextureFile can be called by importer threads. The render thread drains + // this queue whenever a decode slab becomes available, preserving backpressure + // without dropping texture requests from large imports. + TextureDecodeTaskQueue m_queued_texture_decodes = {}; + TextureDecodeTracker m_texture_decode_tracker = {}; Rendering::Textures::TextureHandle m_fallback_cubemap = {}; EnvironmentLightingResources m_fallback_environment_lighting = {}; diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp index 8ab7b17d..adaaaedd 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include using namespace ZEngine::Core::Containers; using namespace ZEngine::Helpers; @@ -22,6 +24,19 @@ namespace ZEngine::Rendering::Renderers namespace { +#ifndef NDEBUG + bool IsAtmosphereSubmissionTraceEnabled() + { + const char* const value = std::getenv("ZENGINE_TRACE_ATMOSPHERE_SUBMISSIONS"); + return value && value[0] == '1' && value[1] == '\0'; + } +#endif + + bool IsAtmosphereViewPass(cstring name) + { + return name && (std::strcmp(name, "Sky View LUT Pass") == 0 || std::strcmp(name, "Aerial Perspective Pass") == 0 || std::strcmp(name, "Sky Composite Pass") == 0); + } + // Register() callbacks may decline a pass after making declarations. Keep a // snapshot of every declaration-owned mutation so a declined callback is // indistinguishable from one that was never registered. @@ -485,7 +500,7 @@ namespace ZEngine::Rendering::Renderers } } // namespace - void BuildQueueBatches(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, Array& out_batches) + static void BuildQueueBatchesForTrace(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, bool isolate_atmosphere_view_passes, Array& out_batches) { out_batches.clear(); @@ -500,7 +515,9 @@ namespace ZEngine::Rendering::Renderers if ((pass.Queue == Rendering::QueueType::TRANSFER_QUEUE && !has_separate_transfer_queue) || (pass.Queue == Rendering::QueueType::COMPUTE_QUEUE && !has_separate_compute_queue)) pass.Queue = Rendering::QueueType::GRAPHIC_QUEUE; - if (!out_batches.empty() && out_batches.back().Queue == pass.Queue && out_batches.back().FirstPassOrder + out_batches.back().PassCount == order_index) + const bool isolate_current_pass = isolate_atmosphere_view_passes && IsAtmosphereViewPass(pass.Name); + const bool isolate_previous_pass = !out_batches.empty() && isolate_atmosphere_view_passes && IsAtmosphereViewPass(passes[order[out_batches.back().FirstPassOrder + out_batches.back().PassCount - 1]].Name); + if (!isolate_current_pass && !isolate_previous_pass && !out_batches.empty() && out_batches.back().Queue == pass.Queue && out_batches.back().FirstPassOrder + out_batches.back().PassCount == order_index) { ++out_batches.back().PassCount; continue; @@ -513,6 +530,11 @@ namespace ZEngine::Rendering::Renderers } } + void BuildQueueBatches(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, Array& out_batches) + { + BuildQueueBatchesForTrace(passes, order, has_separate_transfer_queue, has_separate_compute_queue, false, out_batches); + } + void BuildQueueDependencies(Core::Memory::ArenaAllocator* scratch_arena, ArrayView passes, ArrayView order, ArrayView batches, ArrayView pass_dependencies, Array& out_dependencies) { out_dependencies.clear(); @@ -1273,6 +1295,11 @@ namespace ZEngine::Rendering::Renderers SceneData = data; RenderWidth = Device && Device->SwapchainPtr ? Device->SwapchainPtr->SwapchainImageWidth : 0; RenderHeight = Device && Device->SwapchainPtr ? Device->SwapchainPtr->SwapchainImageHeight : 0; + if (Device && Device->SwapchainPtr) + { + Device->SwapchainPtr->DirectGraphicsTimeline = nullptr; + Device->SwapchainPtr->DirectGraphicsTimelineValue = 0; + } // RenderGraph has a bounded number of virtual passes/resources per frame. // Persistent pipelines, framebuffers, and transient images stay in the @@ -1606,6 +1633,33 @@ namespace ZEngine::Rendering::Renderers { Register({.Scene = SceneData, .FrameIndex = static_cast(Device->SwapchainPtr->CurrentFrame->Index), .RenderWidth = RenderWidth, .RenderHeight = RenderHeight}); Compile(); + +#ifndef NDEBUG + { + bool has_atmosphere_view_passes = false; + for (const RGPass& pass : Passes) + { + if (pass.IsActive() && IsAtmosphereViewPass(pass.Name)) + { + has_atmosphere_view_passes = true; + break; + } + } + Device->SwapchainPtr->TraceSubmission = IsAtmosphereSubmissionTraceEnabled() && has_atmosphere_view_passes; + } +#endif + + // Register() can append deferred copies (notably the UI upload), and scene + // uploads were already recorded before Execute(). Close that single batch now: + // immediate graph batches below must wait for it just like the Present-owned + // final batch. Closing later would let direct graphics read new mesh data before + // the copy submission is visible. + if (Device->RRM) + { + static_cast(Device->RRM)->EndFrame(); + Device->SwapchainPtr->CollectAsyncGPUOperations(); + } + if (!m_compile_valid) { CancelUnsubmittedReadbacks(); @@ -2221,6 +2275,14 @@ namespace ZEngine::Rendering::Renderers if (retain_for_overlay) { +#ifndef NDEBUG + if (Device->SwapchainPtr->TraceSubmission) + { + const RGPass& first_pass = Passes[SortedPassIndices[batch.FirstPassOrder]]; + const RGPass& last_pass = Passes[SortedPassIndices[batch.FirstPassOrder + batch.PassCount - 1]]; + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph final batch={} queue={} passes={} first_pass='{}' last_pass='{}'", batch_index, QueueName(batch.Queue), batch.PassCount, first_pass.Name, last_pass.Name) + } +#endif for (uint32_t order_index = batch.FirstPassOrder; order_index < batch.FirstPassOrder + batch.PassCount && order_index < SortedPassIndices.size(); ++order_index) { const RGPass& pass = Passes[SortedPassIndices[order_index]]; @@ -2261,14 +2323,40 @@ namespace ZEngine::Rendering::Renderers ZENGINE_VALIDATE_ASSERT(timeline_index < QueueTimelineCount, "Invalid render graph queue type") auto* const timeline = QueueTimelines[timeline_index]; const uint64_t signal_value = ++QueueTimelineValues[timeline_index]; +#ifndef NDEBUG + if (Device->SwapchainPtr->TraceSubmission) + { + const RGPass& first_pass = Passes[SortedPassIndices[batch.FirstPassOrder]]; + const RGPass& last_pass = Passes[SortedPassIndices[batch.FirstPassOrder + batch.PassCount - 1]]; + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} queue={} passes={} first_pass='{}' last_pass='{}' waits={} signal_timeline={} signal_value={}", batch_index, QueueName(batch.Queue), batch.PassCount, first_pass.Name, last_pass.Name, wait_infos.size(), static_cast(timeline), signal_value) + for (const VkSemaphoreSubmitInfo& wait : wait_infos) + { + uint64_t completed = 0; + const VkResult result = vkGetSemaphoreCounterValue(Device->LogicalDevice, wait.semaphore, &completed); + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} waits timeline={} value={} completed={} query_result={} stages={}", batch_index, static_cast(wait.semaphore), wait.value, completed, static_cast(result), static_cast(wait.stageMask)) + } + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph submit begin batch={} queue={} signal_value={}", batch_index, QueueName(batch.Queue), signal_value) + } +#endif if (!Device->QueueSubmit(target, timeline, signal_value, wait_infos.data(), static_cast(wait_infos.size()))) { graph_submission_failed = true; break; } +#ifndef NDEBUG + if (Device->SwapchainPtr->TraceSubmission) + { + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} accepted", batch_index) + } +#endif batch_timelines[batch_index] = timeline; batch_signal_values[batch_index] = signal_value; + if (batch.Queue == Rendering::QueueType::GRAPHIC_QUEUE) + { + Device->SwapchainPtr->DirectGraphicsTimeline = timeline; + Device->SwapchainPtr->DirectGraphicsTimelineValue = signal_value; + } AcknowledgeStreamingAcquires(batch.FirstPassOrder, batch.PassCount); SubmitReadbacks(batch.FirstPassOrder, batch.PassCount, timeline, signal_value); // RenderGraph and DeviceSwapchain run on the render thread. Appending @@ -2377,6 +2465,13 @@ namespace ZEngine::Rendering::Renderers void RenderGraph::Dispose() { + constexpr uint32_t graphics_timeline_index = static_cast(Rendering::QueueType::GRAPHIC_QUEUE); + if (Device && Device->SwapchainPtr && Device->SwapchainPtr->DirectGraphicsTimeline == QueueTimelines[graphics_timeline_index]) + { + Device->SwapchainPtr->DirectGraphicsTimeline = nullptr; + Device->SwapchainPtr->DirectGraphicsTimelineValue = 0; + } + ReadbackRing.Dispose(); DisposeQueryPools(); DisposeTimestampFrames(); @@ -4187,7 +4282,11 @@ namespace ZEngine::Rendering::Renderers if (!pass.QueryWrites.empty()) pass.RequestedQueue = Rendering::QueueType::GRAPHIC_QUEUE; } - BuildQueueBatches(Passes, SortedPassIndices, Device->HasSeparateTransferQueue, Device->HasSeparateComputeQueue, QueueBatches); + bool isolate_atmosphere_view_passes = false; +#ifndef NDEBUG + isolate_atmosphere_view_passes = IsAtmosphereSubmissionTraceEnabled(); +#endif + BuildQueueBatchesForTrace(Passes, SortedPassIndices, Device->HasSeparateTransferQueue, Device->HasSeparateComputeQueue, isolate_atmosphere_view_passes, QueueBatches); auto scratch = ZGetScratch(Device->Arena); BuildQueueDependencies(scratch.Arena, Passes, SortedPassIndices, QueueBatches, PassDependencies, QueueDependencies); diff --git a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp index 47a91082..965b1a7f 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include using namespace ZEngine::Core::Memory; @@ -14,6 +16,29 @@ using namespace ZEngine::Rendering::Specifications; namespace ZEngine::Rendering::Renderers { + static_assert(offsetof(ZUIDrawPushConstant, FbScale) == 16); + static_assert(offsetof(ZUIDrawPushConstant, TexIdx) == 24); + static_assert(sizeof(ZUIDrawPushConstant) == 28); + + VkRect2D ZUIRenderPayload::GetScissorRect(const UI::ZUIDrawListCmd& cmd, VkExtent2D render_extent) const + { + if (!(Scale[0] > 0.f) || !(Scale[1] > 0.f) || !(cmd.ClipW > 0.f) || !(cmd.ClipH > 0.f)) + return {}; + + const float scale_x = 0.5f * (float) render_extent.width * Scale[0]; + const float scale_y = 0.5f * (float) render_extent.height * Scale[1]; + // Round outward and clamp both endpoints before converting to unsigned + // extents. This also handles partially off-screen and empty clip regions. + const float x0 = std::clamp(floorf(cmd.ClipX * scale_x), 0.f, (float) render_extent.width); + const float y0 = std::clamp(floorf(cmd.ClipY * scale_y), 0.f, (float) render_extent.height); + const float x1 = std::clamp(ceilf((cmd.ClipX + cmd.ClipW) * scale_x), x0, (float) render_extent.width); + const float y1 = std::clamp(ceilf((cmd.ClipY + cmd.ClipH) * scale_y), y0, (float) render_extent.height); + return { + { (int32_t) x0, (int32_t) y0}, + {(uint32_t) (x1 - x0), (uint32_t) (y1 - y0)} + }; + } + // Initialize / Deinitialize void ZUIPass::Initialize(Hardwares::VulkanDevicePtr device) @@ -142,14 +167,13 @@ namespace ZEngine::Rendering::Renderers const uint32_t kMaxIdx = 131072; const uint32_t max_boxes = ctx->MaxBoxesPerFrame; - float fb_w = ctx->ScreenW > 0 ? (float) ctx->ScreenW : (float) Device->SwapchainPtr->SwapchainImageWidth; - float fb_h = ctx->ScreenH > 0 ? (float) ctx->ScreenH : (float) Device->SwapchainPtr->SwapchainImageHeight; + const float screen_w = ctx->ScreenW > 0 ? (float) ctx->ScreenW : 1.f; + const float screen_h = ctx->ScreenH > 0 ? (float) ctx->ScreenH : 1.f; - out->Scale[0] = 2.f / fb_w; - out->Scale[1] = 2.f / fb_h; + out->Scale[0] = 2.f / screen_w; + out->Scale[1] = 2.f / screen_h; out->Translate[0] = -1.f; out->Translate[1] = -1.f; - out->FramebufferScale = ctx->UIScale > 0.f ? ctx->UIScale : 1.f; uint32_t atlas_idx = ctx->Atlas ? static_cast(ctx->Atlas->Handle.Index) : 0; float wu = ctx->Atlas ? ctx->Atlas->WhiteU : 0.f; @@ -157,7 +181,7 @@ namespace ZEngine::Rendering::Renderers // Init draw list into payload_arena ZUIDrawListInit(&ctx->DrawList, payload_arena, kMaxVtx, kMaxIdx, wu, wv, atlas_idx); - ZUIDrawListPushClipRect(&ctx->DrawList, 0.f, 0.f, fb_w, fb_h, false); + ZUIDrawListPushClipRect(&ctx->DrawList, 0.f, 0.f, screen_w, screen_h, false); // Helpers @@ -797,13 +821,20 @@ namespace ZEngine::Rendering::Renderers const uint32_t frame_index = device->SwapchainPtr->CurrentFrame->Index; const uint32_t fi = frame_index % FRAMES_IN_FLIGHT; { - auto* gp = static_cast(pass); - command_buffer->SetViewport(gp->GetRenderAreaWidth(), gp->GetRenderAreaHeight()); + auto* gp = static_cast(pass); + const VkExtent2D render_extent = {gp->GetRenderAreaWidth(), gp->GetRenderAreaHeight()}; + command_buffer->SetViewport(render_extent.width, render_extent.height); command_buffer->BindPipeline(gp->Pipeline); command_buffer->BindVertexBuffer(VtxBHandles[fi]); command_buffer->BindIndexBuffer(IdxBHandles[fi], VK_INDEX_TYPE_UINT16); - float fs = payload.FramebufferScale; + ZUIDrawPushConstant pc = {}; + pc.Scale[0] = payload.Scale[0]; + pc.Scale[1] = payload.Scale[1]; + pc.Translate[0] = payload.Translate[0]; + pc.Translate[1] = payload.Translate[1]; + pc.FbScale[0] = 0.5f * (float) render_extent.width * payload.Scale[0]; + pc.FbScale[1] = 0.5f * (float) render_extent.height * payload.Scale[1]; for (uint32_t i = 0; i < payload.CmdCount; ++i) { @@ -818,16 +849,11 @@ namespace ZEngine::Rendering::Renderers continue; } - // Logical → physical pixel scissor - command_buffer->SetScissor((uint32_t) (cmd.ClipW * fs), (uint32_t) (cmd.ClipH * fs), (int32_t) (cmd.ClipX * fs), (int32_t) (cmd.ClipY * fs)); - - ZUIDrawPushConstant pc = {}; - pc.Scale[0] = payload.Scale[0]; - pc.Scale[1] = payload.Scale[1]; - pc.Translate[0] = payload.Translate[0]; - pc.Translate[1] = payload.Translate[1]; - pc.TexIdx = cmd.TexIdx; - pc.FbScale = payload.FramebufferScale; + const VkRect2D scissor = payload.GetScissorRect(cmd, render_extent); + if (scissor.extent.width == 0 || scissor.extent.height == 0) + continue; + command_buffer->SetScissor(scissor.extent.width, scissor.extent.height, scissor.offset.x, scissor.offset.y); + pc.TexIdx = cmd.TexIdx; command_buffer->PushConstants(VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(ZUIDrawPushConstant), &pc); command_buffer->BindDescriptorSets(frame_index); diff --git a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h index 6ea0f396..a4beeadb 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h +++ b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h @@ -15,18 +15,19 @@ namespace ZEngine::Rendering::Renderers /// @brief Frame-local draw data consumed by ZUIPass. struct ZUIRenderPayload { - UI::ZUIDrawVtx* Vtx = nullptr; - uint32_t VtxCount = 0; - uint16_t* Idx = nullptr; - uint32_t IdxCount = 0; - UI::ZUIDrawListCmd* Cmds = nullptr; - uint32_t CmdCount = 0; - /// @brief Physical-to-logical scale used for scissor rectangles. - float FramebufferScale = 1.f; - /// @brief NDC scale: 2 / framebuffer extent. - float Scale[2] = {}; + UI::ZUIDrawVtx* Vtx = nullptr; + uint32_t VtxCount = 0; + uint16_t* Idx = nullptr; + uint32_t IdxCount = 0; + UI::ZUIDrawListCmd* Cmds = nullptr; + uint32_t CmdCount = 0; + /// @brief NDC scale: 2 / logical UI extent. + float Scale[2] = {}; /// @brief NDC origin offset. - float Translate[2] = {}; + float Translate[2] = {}; + + /// @brief Projects a logical clip rectangle into the actual render target. + VkRect2D GetScissorRect(const UI::ZUIDrawListCmd& cmd, VkExtent2D render_extent) const; }; /// @brief Push-constant layout shared with zui_draw.vert. @@ -34,9 +35,9 @@ namespace ZEngine::Rendering::Renderers { float Scale[2] = {}; float Translate[2] = {}; + /// @brief Logical-to-physical scale consumed by uFbScale. + float FbScale[2] = {1.f, 1.f}; uint32_t TexIdx = 0; - /// @brief Physical-to-logical scale consumed by uFbScale. - float FbScale = 1.f; }; /// @brief Final graphics pass that uploads and draws the ZUI draw list. diff --git a/ZEngine/ZEngine/UI/ZUIContext.h b/ZEngine/ZEngine/UI/ZUIContext.h index 4e113e57..5efe8965 100644 --- a/ZEngine/ZEngine/UI/ZUIContext.h +++ b/ZEngine/ZEngine/UI/ZUIContext.h @@ -383,13 +383,17 @@ namespace ZEngine::UI // Vector draw list — populated by PreparePayload each frame (FrameArena-backed) ZUIDrawList DrawList = {}; - // current swapchain dimensions — set by AppRenderPipeline::BeginOverlayFrame each frame + // Logical UI dimensions — set by AppRenderPipeline::BeginOverlayFrame each frame. uint32_t ScreenW = 1280; uint32_t ScreenH = 720; - // display content scale (glfwGetWindowContentScale); 1.0=standard, 2.0=Retina. - // Widgets multiply logical pixel sizes by this to stay readable at any DPI. + // Physical framebuffer pixels per ZUI coordinate. This incorporates the GLFW + // content scale on platforms where the framebuffer/window ratio remains 1:1. float UIScale = 1.f; - // guard against per-frame ContentScale log spam — log only once + // Converts GLFW cursor coordinates into ZUI coordinates. GLFW reports + // positions in window coordinates, which may be larger than ZUI's + // DPI-independent coordinate space. + float InputScale[2] = {1.f, 1.f}; + // Log dimensions initially and on changes, never on every frame. bool UIScaleLogged = false; // drag-and-drop — source is set by ZUIBeginDragSource while a box is held+moving; diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index 638e4f49..fa77510c 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -21,6 +22,10 @@ extern "C" void ZEngineSetDockIcon(const char* path); #include #include +#if defined(__linux__) +#include +#endif + #ifdef _WIN32 #include @@ -110,7 +115,13 @@ namespace ZEngine::Windows m_property.Title = cfg.Title.c_str(); m_property.VSync = cfg.EnableVsync; - int glfw_init = glfwInit(); +#if defined(__APPLE__) + // GLFW otherwise tries to dlopen the release Vulkan loader name, while + // Debug builds link the debug loader directly. + glfwInitVulkanLoader(vkGetInstanceProcAddr); +#endif + + int glfw_init = glfwInit(); if (glfw_init == GLFW_FALSE) { ZENGINE_CORE_CRITICAL("Unable to initialize glfw..") @@ -133,27 +144,36 @@ namespace ZEngine::Windows } { - auto icon_path = (std::filesystem::current_path() / "ZodiacEngine/Settings/Icons/AppIconBadge.png").string(); - int w = 0, h = 0, channels = 0; - unsigned char* pixels = stbi_load(icon_path.c_str(), &w, &h, &channels, STBI_rgb_alpha); - if (pixels) + const auto* engine_context = Engine::GetContext(); + const auto icon_path = engine_context && engine_context->EngineAssetsNativeRoot ? (std::filesystem::path(engine_context->EngineAssetsNativeRoot) / "Settings/Icons/AppIconBadge.png").string() : std::string{}; +#if !defined(__APPLE__) +#if defined(__linux__) + // GLFW reports this as an error on Wayland, and this application's GLFW + // error callback is intentionally fatal. Wayland has no window-icon API; + // the compositor selects the application icon instead. + if (glfwGetPlatform() != GLFW_PLATFORM_WAYLAND) +#endif { - GLFWimage icon{w, h, pixels}; - glfwSetWindowIcon(m_native_window, 1, &icon); - stbi_image_free(pixels); + int w = 0, h = 0, channels = 0; + unsigned char* pixels = stbi_load(icon_path.c_str(), &w, &h, &channels, STBI_rgb_alpha); + if (pixels) + { + GLFWimage icon{w, h, pixels}; + glfwSetWindowIcon(m_native_window, 1, &icon); + stbi_image_free(pixels); + } } -#if defined(__APPLE__) +#else ZEngineSetDockIcon(icon_path.c_str()); #endif } - int window_width = 0, window_height = 0; - glfwGetWindowSize(m_native_window, &window_width, &window_height); - if ((window_width > 0) && (window_height > 0) && (m_property.Width != window_width) && (m_property.Height != window_height)) - { - m_property.SetWidth(window_width); - m_property.SetHeight(window_height); - } + // GetWidth/GetHeight describe physical framebuffer pixels, including before + // the first resize callback (Retina and Wayland can already be scaled). + int framebuffer_width = 0, framebuffer_height = 0; + glfwGetFramebufferSize(m_native_window, &framebuffer_width, &framebuffer_height); + m_property.SetWidth(static_cast(framebuffer_width)); + m_property.SetHeight(static_cast(framebuffer_height)); uint32_t count = 0; const char** extensions_layer_names = glfwGetRequiredInstanceExtensions(&count); @@ -437,66 +457,49 @@ namespace ZEngine::Windows #elif defined(__linux__) { - // Zenity and kdialog block until the picker closes. Run them off - // the UI thread, then resume this coroutine on the main thread. std::vector extensions; extensions.reserve(type_filters.size()); for (std::string_view filter : type_filters) extensions.emplace_back(filter); - std::string default_dir_copy(default_dir); - std::string message_copy(message); - std::future result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy)] { - std::string filter; - for (const std::string& extension : extensions) - { - if (!filter.empty()) - filter += ' '; - filter += '*'; - filter += extension; - } - - const std::string start_dir = default_dir.empty() ? "." : default_dir; - const std::string title = message.empty() ? "Select a file" : message; + std::string default_dir_copy(default_dir); + std::string message_copy(message); + const int platform = glfwGetPlatform(); - std::string cmd; - if (system("which zenity > /dev/null 2>&1") == 0) - { - cmd = "zenity --file-selection --title='" + title + "'"; - if (!default_dir.empty()) - cmd += " --filename='" + start_dir + "/'"; - if (!filter.empty()) - cmd += " --file-filter='" + filter + "'"; - } - else if (system("which kdialog > /dev/null 2>&1") == 0) + std::unique_ptr wayland_parent; + std::string portal_parent; + unsigned long x11_parent_window = 0; + if (platform == GLFW_PLATFORM_WAYLAND) + { + // The exported object remains alive in this coroutine frame until + // the portal response arrives, keeping the dialog parent valid. + wayland_parent = Platform::CreateWaylandPortalParent(m_native_window); + if (!wayland_parent) + ZENGINE_CORE_WARN("[FileDialog] Wayland compositor does not support xdg-foreign; opening the portal without a transient parent") + else + portal_parent = wayland_parent->Handle(); + } + else if (platform == GLFW_PLATFORM_X11) + { + x11_parent_window = glfwGetX11Window(m_native_window); + if (x11_parent_window != 0) { - cmd = "kdialog --getopenfilename " + start_dir + " '"; - for (const std::string& extension : extensions) - cmd += '*' + extension + ' '; - if (!cmd.empty() && cmd.back() == ' ') - cmd.pop_back(); - cmd += "'"; + char parent_buffer[32] = {}; + std::snprintf(parent_buffer, sizeof(parent_buffer), "x11:0x%lx", x11_parent_window); + portal_parent = parent_buffer; } + } - std::string selected_path; - if (cmd.empty()) - return selected_path; - - FILE* pipe = popen(cmd.c_str(), "r"); - if (!pipe) - return selected_path; - - char buf[4096] = {}; - if (fgets(buf, sizeof(buf), pipe)) - { - selected_path = buf; - if (!selected_path.empty() && selected_path.back() == '\n') - selected_path.pop_back(); - } - pclose(pipe); - return selected_path; - }); - path = co_await result; + std::future portal_result = std::async(std::launch::async, [extensions, default_dir = default_dir_copy, message = message_copy, portal_parent] { return Platform::OpenPortalFileDialog(portal_parent, extensions, default_dir, message); }); + auto result = co_await portal_result; + if (result.Outcome != Platform::PortalFileDialogStatus::Failed) + path = std::move(result.Path); + else + { + ZENGINE_CORE_WARN("[FileDialog] Desktop portal failed; trying the Linux fallback picker") + std::future fallback_result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy), x11_parent_window, use_x11 = platform == GLFW_PLATFORM_X11] { return Platform::OpenLinuxFallbackFileDialog(x11_parent_window, use_x11, extensions, default_dir, message); }); + path = co_await fallback_result; + } } #endif diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp new file mode 100644 index 00000000..69c9849f --- /dev/null +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp @@ -0,0 +1,334 @@ +#include +#include + +#if defined(__linux__) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xdg-foreign-unstable-v2-client-protocol.h" + +namespace ZEngine::Windows::Platform +{ + namespace + { + constexpr char kPortalService[] = "org.freedesktop.portal.Desktop"; + constexpr char kPortalObjectPath[] = "/org/freedesktop/portal/desktop"; + constexpr char kPortalFileChooser[] = "org.freedesktop.portal.FileChooser"; + constexpr char kPortalRequest[] = "org.freedesktop.portal.Request"; + constexpr char kPortalResponse[] = "Response"; + constexpr char kSupportedFilesLabel[] = "Supported files"; + + struct PortalRequest + { + GMainLoop* Loop = nullptr; + std::string Path; + std::string SelectedPath; + uint32_t Response = 2; + }; + + std::atomic_uint64_t s_portal_token = 0; + + void LogError(const char* operation, GError* error) + { + ZENGINE_CORE_ERROR("[FileDialog] {}: {}", operation, error ? error->message : "unknown error") + if (error) + g_error_free(error); + } + + GVariant* BuildPortalOptions(std::span extensions, std::string_view default_directory) + { + GVariantBuilder options; + g_variant_builder_init(&options, G_VARIANT_TYPE("a{sv}")); + + const std::string handle_token = "zengine" + std::to_string(s_portal_token.fetch_add(1, std::memory_order_relaxed)); + g_variant_builder_add(&options, "{sv}", "handle_token", g_variant_new_string(handle_token.c_str())); + g_variant_builder_add(&options, "{sv}", "modal", g_variant_new_boolean(TRUE)); + + if (!extensions.empty()) + { + GVariantBuilder patterns; + g_variant_builder_init(&patterns, G_VARIANT_TYPE("a(us)")); + for (const std::string& extension : extensions) + { + const std::string pattern = "*" + extension; + g_variant_builder_add(&patterns, "(us)", 0u, pattern.c_str()); + } + + GVariantBuilder filters; + g_variant_builder_init(&filters, G_VARIANT_TYPE("a(sa(us))")); + g_variant_builder_add(&filters, "(s@a(us))", kSupportedFilesLabel, g_variant_builder_end(&patterns)); + g_variant_builder_add(&options, "{sv}", "filters", g_variant_builder_end(&filters)); + } + + if (!default_directory.empty()) + { + GVariantBuilder folder; + g_variant_builder_init(&folder, G_VARIANT_TYPE("ay")); + for (const unsigned char character : default_directory) + g_variant_builder_add(&folder, "y", character); + g_variant_builder_add(&folder, "y", 0u); + g_variant_builder_add(&options, "{sv}", "current_folder", g_variant_builder_end(&folder)); + } + + return g_variant_builder_end(&options); + } + + void OnPortalResponse(GDBusConnection*, const gchar*, const gchar* object_path, const gchar*, const gchar*, GVariant* parameters, gpointer user_data) + { + auto* request = static_cast(user_data); + if (!object_path || request->Path != object_path) + return; + + guint32 response = 1; + GVariant* results = nullptr; + g_variant_get(parameters, "(u@a{sv})", &response, &results); + request->Response = response; + + if (response == 0) + { + GVariant* uris = g_variant_lookup_value(results, "uris", G_VARIANT_TYPE("as")); + if (uris) + { + gsize count = 0; + gchar** values = g_variant_dup_strv(uris, &count); + if (count > 0) + { + gchar* native_path = g_filename_from_uri(values[0], nullptr, nullptr); + if (native_path) + { + request->SelectedPath = native_path; + g_free(native_path); + } + } + g_strfreev(values); + g_variant_unref(uris); + } + } + + g_variant_unref(results); + g_main_loop_quit(request->Loop); + } + + std::string ShellQuote(std::string_view value) + { + std::string quoted("'"); + for (char character : value) + { + if (character == '\'') + quoted += "'\"'\"'"; + else + quoted += character; + } + quoted += '\''; + return quoted; + } + } // namespace + + WaylandPortalParent::~WaylandPortalParent() + { + if (m_exported) + zxdg_exported_v2_destroy(m_exported); + if (m_exporter) + zxdg_exporter_v2_destroy(m_exporter); + if (m_registry) + wl_registry_destroy(m_registry); + if (m_event_queue) + wl_event_queue_destroy(m_event_queue); + } + + const std::string& WaylandPortalParent::Handle() const + { + return m_handle; + } + + bool WaylandPortalParent::Initialize(GLFWwindow* window) + { + if (!window || glfwGetPlatform() != GLFW_PLATFORM_WAYLAND) + return false; + + wl_display* display = glfwGetWaylandDisplay(); + wl_surface* surface = glfwGetWaylandWindow(window); + if (!display || !surface) + return false; + + m_event_queue = wl_display_create_queue(display); + if (!m_event_queue) + return false; + + static const wl_registry_listener registry_listener = { + .global = &WaylandPortalParent::OnRegistryGlobal, + .global_remove = nullptr, + }; + static const zxdg_exported_v2_listener exported_listener = { + .handle = &WaylandPortalParent::OnExportedHandle, + }; + + m_registry = wl_display_get_registry(display); + if (!m_registry) + return false; + wl_proxy_set_queue(reinterpret_cast(m_registry), m_event_queue); + wl_registry_add_listener(m_registry, ®istry_listener, this); + + if (wl_display_roundtrip_queue(display, m_event_queue) < 0 || !m_exporter) + return false; + + m_exported = zxdg_exporter_v2_export_toplevel(m_exporter, surface); + if (!m_exported) + return false; + wl_proxy_set_queue(reinterpret_cast(m_exported), m_event_queue); + zxdg_exported_v2_add_listener(m_exported, &exported_listener, this); + + return wl_display_roundtrip_queue(display, m_event_queue) >= 0 && !m_handle.empty(); + } + + void WaylandPortalParent::OnRegistryGlobal(void* context, wl_registry* registry, uint32_t name, const char* interface, uint32_t version) + { + auto* parent = static_cast(context); + if (parent->m_exporter || std::strcmp(interface, zxdg_exporter_v2_interface.name) != 0) + return; + + parent->m_exporter = static_cast(wl_registry_bind(registry, name, &zxdg_exporter_v2_interface, std::min(version, 1u))); + if (parent->m_exporter) + wl_proxy_set_queue(reinterpret_cast(parent->m_exporter), parent->m_event_queue); + } + + void WaylandPortalParent::OnExportedHandle(void* context, zxdg_exported_v2*, const char* handle) + { + if (handle) + static_cast(context)->m_handle = "wayland:" + std::string(handle); + } + + std::unique_ptr CreateWaylandPortalParent(GLFWwindow* window) + { + auto parent = std::unique_ptr(new WaylandPortalParent{}); + if (!parent->Initialize(window)) + return {}; + return parent; + } + + PortalFileDialogResult OpenPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title) + { + GMainContext* context = g_main_context_new(); + g_main_context_push_thread_default(context); + + GError* error = nullptr; + GDBusConnection* connection = g_bus_get_sync(G_BUS_TYPE_SESSION, nullptr, &error); + if (!connection) + { + LogError("Unable to connect to the desktop portal", error); + g_main_context_pop_thread_default(context); + g_main_context_unref(context); + return {}; + } + + PortalRequest request; + request.Loop = g_main_loop_new(context, FALSE); + + const guint subscription = g_dbus_connection_signal_subscribe(connection, kPortalService, kPortalRequest, kPortalResponse, nullptr, nullptr, G_DBUS_SIGNAL_FLAGS_NONE, &OnPortalResponse, &request, nullptr); + + const std::string parent(parent_handle); + const std::string dialog_title(title.empty() ? "Select a file" : title); + GVariant* reply = g_dbus_connection_call_sync(connection, kPortalService, kPortalObjectPath, kPortalFileChooser, "OpenFile", g_variant_new("(ss@a{sv})", parent.c_str(), dialog_title.c_str(), BuildPortalOptions(extensions, default_directory)), G_VARIANT_TYPE("(o)"), G_DBUS_CALL_FLAGS_NONE, -1, nullptr, &error); + + if (!reply) + { + LogError("Unable to open the desktop portal file picker", error); + } + else + { + const gchar* request_path = nullptr; + g_variant_get(reply, "(&o)", &request_path); + if (request_path) + request.Path = request_path; + g_variant_unref(reply); + if (!request.Path.empty()) + g_main_loop_run(request.Loop); + else + ZENGINE_CORE_ERROR("[FileDialog] Desktop portal returned an invalid request path") + } + + g_dbus_connection_signal_unsubscribe(connection, subscription); + g_main_loop_unref(request.Loop); + g_object_unref(connection); + g_main_context_pop_thread_default(context); + g_main_context_unref(context); + if (static_cast(request.Response) == PortalFileDialogStatus::Selected && !request.SelectedPath.empty()) + return {PortalFileDialogStatus::Selected, std::move(request.SelectedPath)}; + if (static_cast(request.Response) == PortalFileDialogStatus::Cancelled) + return {PortalFileDialogStatus::Cancelled, {}}; + + ZENGINE_CORE_ERROR("[FileDialog] Desktop portal returned an error response ({})", request.Response) + return {}; + } + + std::string OpenLinuxFallbackFileDialog(unsigned long x11_parent_window, bool use_x11, std::span extensions, std::string_view default_directory, std::string_view title) + { + std::string filter; + for (const std::string& extension : extensions) + { + if (!filter.empty()) + filter += ' '; + filter += '*'; + filter += extension; + } + + const std::string start_dir = default_directory.empty() ? "." : std::string(default_directory); + const std::string dialog_title(title.empty() ? "Select a file" : title); + + std::string command; + if (system("command -v zenity >/dev/null 2>&1") == 0) + { + command = use_x11 ? "GDK_BACKEND=x11 " : ""; + command += "zenity --file-selection --modal --title=" + ShellQuote(dialog_title); + if (use_x11 && x11_parent_window != 0) + command += " --attach=" + std::to_string(x11_parent_window); + if (!default_directory.empty()) + command += " --filename=" + ShellQuote(start_dir + "/"); + if (!filter.empty()) + command += " --file-filter=" + ShellQuote(filter); + } + else if (system("command -v kdialog >/dev/null 2>&1") == 0) + { + command = "kdialog"; + if (use_x11 && x11_parent_window != 0) + command += " --attach " + std::to_string(x11_parent_window); + command += " --getopenfilename " + ShellQuote(start_dir); + if (!filter.empty()) + command += " " + ShellQuote(filter); + } + else + { + ZENGINE_CORE_ERROR("[FileDialog] No Linux picker is available. Install xdg-desktop-portal, zenity, or kdialog") + return {}; + } + + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) + { + ZENGINE_CORE_ERROR("[FileDialog] Unable to launch the fallback picker") + return {}; + } + + char buffer[4096] = {}; + std::string selected_path; + if (fgets(buffer, sizeof(buffer), pipe)) + { + selected_path = buffer; + if (!selected_path.empty() && selected_path.back() == '\n') + selected_path.pop_back(); + } + pclose(pipe); + return selected_path; + } +} // namespace ZEngine::Windows::Platform + +#endif diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h new file mode 100644 index 00000000..15d6bda4 --- /dev/null +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h @@ -0,0 +1,70 @@ +#pragma once + +#if defined(__linux__) + +#include +#include +#include +#include +#include + +struct GLFWwindow; +struct wl_event_queue; +struct wl_registry; +struct zxdg_exported_v2; +struct zxdg_exporter_v2; + +namespace ZEngine::Windows::Platform +{ + enum class PortalFileDialogStatus : uint32_t + { + Selected = 0, + Cancelled = 1, + Failed = 2, + }; + + struct PortalFileDialogResult + { + PortalFileDialogStatus Outcome = PortalFileDialogStatus::Failed; + std::string Path; + }; + + // Owns the xdg-foreign export for as long as the portal dialog is open. + // Wayland invalidates the parent relation as soon as this object is destroyed. + class WaylandPortalParent + { + public: + ~WaylandPortalParent(); + + WaylandPortalParent(const WaylandPortalParent&) = delete; + WaylandPortalParent& operator=(const WaylandPortalParent&) = delete; + + [[nodiscard]] const std::string& Handle() const; + + private: + WaylandPortalParent() = default; + + bool Initialize(GLFWwindow* window); + + static void OnRegistryGlobal(void* context, wl_registry* registry, uint32_t name, const char* interface, uint32_t version); + static void OnExportedHandle(void* context, zxdg_exported_v2* exported, const char* handle); + + wl_event_queue* m_event_queue = nullptr; + wl_registry* m_registry = nullptr; + zxdg_exporter_v2* m_exporter = nullptr; + zxdg_exported_v2* m_exported = nullptr; + std::string m_handle; + + friend std::unique_ptr CreateWaylandPortalParent(GLFWwindow* window); + }; + + [[nodiscard]] std::unique_ptr CreateWaylandPortalParent(GLFWwindow* window); + + // Blocks the worker thread until the user accepts, cancels, or the portal fails. + [[nodiscard]] PortalFileDialogResult OpenPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title); + + // Used only when the portal service is unavailable or rejects the request. + [[nodiscard]] std::string OpenLinuxFallbackFileDialog(unsigned long x11_parent_window, bool use_x11, std::span extensions, std::string_view default_directory, std::string_view title); +} // namespace ZEngine::Windows::Platform + +#endif diff --git a/ZEngine/ZEngine/Windows/Platform/xdg-foreign-unstable-v2.xml b/ZEngine/ZEngine/Windows/Platform/xdg-foreign-unstable-v2.xml new file mode 100644 index 00000000..9e6d6164 --- /dev/null +++ b/ZEngine/ZEngine/Windows/Platform/xdg-foreign-unstable-v2.xml @@ -0,0 +1,45 @@ + + + + Copyright © 2015-2016 Red Hat Inc. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ZEngine/tests/Misc/EngineAssetRoot_test.cpp b/ZEngine/tests/Misc/EngineAssetRoot_test.cpp new file mode 100644 index 00000000..cf96bc5b --- /dev/null +++ b/ZEngine/tests/Misc/EngineAssetRoot_test.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include + +namespace +{ + class EngineAssetRootTest : public ::testing::Test + { + protected: + void SetUp() override + { + static std::atomic next_id = 0; + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + m_root = std::filesystem::temp_directory_path() / ("zengine_engine_asset_root_" + std::to_string(now) + "_" + std::to_string(next_id.fetch_add(1, std::memory_order_relaxed))); + std::error_code error; + std::filesystem::remove_all(m_root, error); + std::filesystem::create_directories(m_root, error); + ASSERT_FALSE(error); + } + + void TearDown() override + { + std::error_code error; + std::filesystem::remove_all(m_root, error); + } + + std::filesystem::path m_root; + }; +} // namespace + +TEST_F(EngineAssetRootTest, ResolvesPackageBesideTheExecutable) +{ + const std::filesystem::path executable = m_root / "bin" / "Obelisk"; + + const auto result = ZEngine::Core::ResolveEngineAssetRoot(executable); + + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Root, m_root / "bin" / "ZodiacEngine"); +} + +TEST_F(EngineAssetRootTest, ExplicitOverrideTakesPrecedence) +{ + const std::filesystem::path executable = m_root / "bin" / "Obelisk"; + const std::filesystem::path override = m_root / "tooling-assets"; + + const auto result = ZEngine::Core::ResolveEngineAssetRoot(executable, override); + + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Root, override); +} + +TEST_F(EngineAssetRootTest, RootValidationRejectsMissingPackageDirectory) +{ + const auto result = ZEngine::Core::ValidateEngineAssetRoot(m_root / "missing"); + + EXPECT_FALSE(result.Succeeded()); + EXPECT_NE(result.Diagnostic.find("missing"), std::string::npos); +} + +TEST_F(EngineAssetRootTest, RootValidationRejectsEmptyPath) +{ + const auto result = ZEngine::Core::ValidateEngineAssetRoot({}); + + EXPECT_FALSE(result.Succeeded()); + EXPECT_NE(result.Diagnostic.find("empty"), std::string::npos); +} + +TEST_F(EngineAssetRootTest, RootValidationAcceptsPackageDirectory) +{ + const std::filesystem::path package_root = m_root / "ZodiacEngine"; + std::filesystem::create_directories(package_root); + + const auto result = ZEngine::Core::ValidateEngineAssetRoot(package_root); + + EXPECT_TRUE(result.Succeeded()) << result.Diagnostic; +} diff --git a/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp b/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp index 6ba2a5ed..9824f66f 100644 --- a/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp +++ b/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp @@ -12,6 +12,8 @@ TEST(AppRenderPipelineMailboxTest, RenderStateProvidesLatestCoherentCameraSnapsh first.ResizeSequence = 1; first.RenderTargetW = 1280; first.RenderTargetH = 720; + first.FramebufferW = 1500; + first.FramebufferH = 800; first.SkyRevision = 4; first.Sky.EnvironmentIntensity = 1.0f; pipeline.PublishFrameState(first); @@ -21,6 +23,8 @@ TEST(AppRenderPipelineMailboxTest, RenderStateProvidesLatestCoherentCameraSnapsh latest.ResizeSequence = 2; latest.RenderTargetW = 1920; latest.RenderTargetH = 1080; + latest.FramebufferW = 2478; + latest.FramebufferH = 1514; latest.SkyRevision = 5; latest.Sky.EnvironmentIntensity = 2.0f; pipeline.PublishFrameState(latest); @@ -31,6 +35,8 @@ TEST(AppRenderPipelineMailboxTest, RenderStateProvidesLatestCoherentCameraSnapsh EXPECT_EQ(read.ResizeSequence, 2u); EXPECT_EQ(read.RenderTargetW, 1920u); EXPECT_EQ(read.RenderTargetH, 1080u); + EXPECT_EQ(read.FramebufferW, 2478u); + EXPECT_EQ(read.FramebufferH, 1514u); EXPECT_EQ(read.SkyRevision, 5u); EXPECT_FLOAT_EQ(read.Sky.EnvironmentIntensity, 2.0f); } diff --git a/ZEngine/tests/Rendering/AssetMaterialLifetime_test.cpp b/ZEngine/tests/Rendering/AssetMaterialLifetime_test.cpp new file mode 100644 index 00000000..c47fdd59 --- /dev/null +++ b/ZEngine/tests/Rendering/AssetMaterialLifetime_test.cpp @@ -0,0 +1,62 @@ +#include +#include +#include +#include +#include + +using ZEngine::Core::Containers::String; +using ZEngine::Core::Memory::ArenaAllocator; +using ZEngine::Importers::AssetMaterial; + +TEST(AssetMaterialLifetimeTest, CloneOwnsEveryStringAfterSourceScratchIsReused) +{ + ZEngine::Core::Memory::MemoryManager memory; + memory.Initialize(ZMega(2), {}); + ArenaAllocator scratch; + ArenaAllocator persistent; + memory.MainArena.CreateSubArena(ZMega(1), &scratch); + memory.MainArena.CreateSubArena(ZKilo(256), &persistent); + + AssetMaterial source; + const std::array source_strings = {&source.Name, &source.AlbedoTexPath, &source.EmissiveTexPath, &source.NormalTexPath, &source.OpacityTexPath, &source.SpecularTexPath}; + std::array expected; + for (size_t i = 0; i < expected.size(); ++i) + { + expected[i] = "/textures/" + std::string(80, static_cast('a' + i)) + ".png"; + source_strings[i]->init(&scratch, expected[i].c_str()); + } + source.MaterialUUID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + source.AlbedoTexUUID = source.MaterialUUID; + source.NormalTexUUID = source.MaterialUUID; + source.AlbedoColor[2] = 0.75f; + source.Factors[0] = 0.5f; + + auto copy = source.Clone(&persistent); + const std::array copied_strings = {©.Name, ©.AlbedoTexPath, ©.EmissiveTexPath, ©.NormalTexPath, ©.OpacityTexPath, ©.SpecularTexPath}; + for (size_t i = 0; i < copied_strings.size(); ++i) + EXPECT_NE(copied_strings[i]->c_str(), source_strings[i]->c_str()); + scratch.Clear(); + ASSERT_NE(scratch.Allocate(ZKilo(4)), nullptr); // allocation zeroes and overwrites the old strings + for (size_t i = 0; i < copied_strings.size(); ++i) + EXPECT_STREQ(copied_strings[i]->c_str(), expected[i].c_str()); + EXPECT_EQ(copy.MaterialUUID, source.MaterialUUID); + EXPECT_EQ(copy.AlbedoTexUUID, source.AlbedoTexUUID); + EXPECT_EQ(copy.NormalTexUUID, source.NormalTexUUID); + EXPECT_FLOAT_EQ(copy.AlbedoColor[2], 0.75f); + EXPECT_FLOAT_EQ(copy.Factors[0], 0.5f); +} + +TEST(AssetMaterialLifetimeTest, CloneSupportsUninitializedOptionalStrings) +{ + ZEngine::Core::Memory::MemoryManager memory; + memory.Initialize(ZKilo(64), {}); + const AssetMaterial source; + const auto copy = source.Clone(&memory.MainArena); + EXPECT_TRUE(copy.Name.empty()); + EXPECT_TRUE(copy.AlbedoTexPath.empty()); + EXPECT_TRUE(copy.EmissiveTexPath.empty()); + EXPECT_TRUE(copy.NormalTexPath.empty()); + EXPECT_TRUE(copy.OpacityTexPath.empty()); + EXPECT_TRUE(copy.SpecularTexPath.empty()); + EXPECT_TRUE(copy.MaterialUUID.is_nil()); +} diff --git a/ZEngine/tests/Rendering/FlyCameraController_test.cpp b/ZEngine/tests/Rendering/FlyCameraController_test.cpp index 752d0906..59a7f2bf 100644 --- a/ZEngine/tests/Rendering/FlyCameraController_test.cpp +++ b/ZEngine/tests/Rendering/FlyCameraController_test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using namespace ZEngine; using namespace ZEngine::Controllers; @@ -31,6 +32,16 @@ namespace { return m_input_enabled; } + + bool IsHovered() const + { + return m_state == CamState::Hover; + } + + float ViewportBound(uint32_t index) const + { + return m_vp[index]; + } }; struct FlyCameraControllerTest : ::testing::Test @@ -91,3 +102,74 @@ TEST_F(FlyCameraControllerTest, PauseAndResumeControlCameraInputLifecycle) Controller.ResumeEventProcessing(); EXPECT_TRUE(Controller.IsInputEnabled()); } + +TEST_F(FlyCameraControllerTest, ScaledUiViewportUsesWindowCoordinatesForHoverAndRays) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + const float input_scale[2] = {0.8f, 0.75f}; + Controller.SetViewportFromUI(rect, input_scale); + EXPECT_FLOAT_EQ(Controller.ViewportBound(0), 125.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(1), 200.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(2), 1125.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(3), 1000.f); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 1000.f / 800.f); + + // This raw cursor lies inside the displayed viewport, but outside the old + // unconverted UI bounds. It must remain available to camera navigation. + Input.AccumulateCursorPosition(1000, 600); + Controller.Update(Core::TimeStep(0.f)); + EXPECT_TRUE(Controller.IsHovered()); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportX, 875.f); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportY, 400.f); + + const auto ray = Camera.GetRayFromViewport(500.f, 400.f); + const auto forward = Camera.GetForward(); + EXPECT_NEAR(ray.Direction.x, forward.x, 0.0001f); + EXPECT_NEAR(ray.Direction.y, forward.y, 0.0001f); + EXPECT_NEAR(ray.Direction.z, forward.z, 0.0001f); + + Input.AccumulateCursorPosition(1200, 600); + Controller.Update(Core::TimeStep(0.f)); + EXPECT_FALSE(Controller.IsHovered()); +} + +TEST_F(FlyCameraControllerTest, NativeWaylandAndRetinaInputNeedsNoExtraDpiScaling) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + const float input_scale[2] = {1.f, 1.f}; + Controller.SetViewportFromUI(rect, input_scale); + for (uint32_t index = 0; index < 4; ++index) + EXPECT_FLOAT_EQ(Controller.ViewportBound(index), rect[index]); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 800.f / 600.f); + + Input.AccumulateCursorPosition(500, 450); + Controller.Update(Core::TimeStep(0.f)); + EXPECT_TRUE(Controller.IsHovered()); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportX, 400.f); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportY, 300.f); +} + +TEST_F(FlyCameraControllerTest, DpiTransitionUpdatesBoundsAndExtentWithoutStaleScale) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + const float scaled[2] = {0.8f, 0.75f}; + const float native[2] = {1.f, 1.f}; + Controller.SetViewportFromUI(rect, scaled); + Controller.SetViewportFromUI(rect, native); + EXPECT_FLOAT_EQ(Controller.ViewportBound(0), 100.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(2), 900.f); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 800.f / 600.f); +} + +TEST_F(FlyCameraControllerTest, InvalidInputScalesDoNotDivideByZeroOrProduceNan) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + for (float invalid_scale : {0.f, -1.f, std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity()}) + { + const float scales[2] = {invalid_scale, invalid_scale}; + Controller.SetViewportFromUI(rect, scales); + EXPECT_FLOAT_EQ(Controller.ViewportBound(0), 100.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(3), 750.f); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 800.f / 600.f); + } +} diff --git a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp index d103e5e3..72dffbae 100644 --- a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp +++ b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp @@ -1,10 +1,13 @@ #include #include +#include #include #include +#include #include #include #include +#include using namespace ZEngine::Core::Memory; using namespace ZEngine::Logging; @@ -64,11 +67,15 @@ namespace device_extension_count = 1; #endif - VkDeviceCreateInfo device_info = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; - device_info.queueCreateInfoCount = 1; - device_info.pQueueCreateInfos = &queue_info; - device_info.enabledExtensionCount = device_extension_count; - device_info.ppEnabledExtensionNames = device_extension_count > 0 ? device_extensions : nullptr; + VkDeviceCreateInfo device_info = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + device_info.queueCreateInfoCount = 1; + device_info.pQueueCreateInfos = &queue_info; + device_info.enabledExtensionCount = device_extension_count; + device_info.ppEnabledExtensionNames = device_extension_count > 0 ? device_extensions : nullptr; + + VkPhysicalDeviceVulkan13Features features13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES, .synchronization2 = VK_TRUE}; + VkPhysicalDeviceVulkan12Features features12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, .pNext = &features13, .timelineSemaphore = VK_TRUE}; + device_info.pNext = &features12; return vkCreateDevice(PhysicalDevice, &device_info, nullptr, &Device) == VK_SUCCESS; } @@ -301,3 +308,181 @@ TEST_F(GpuAllocatorTest, StagingRingSharesDeclaredPool) allocator().FreeBuffer(one_shot); } + +// Reuse the headless Vulkan fixture to exercise real upload submissions without +// creating a window, swapchain images, or the rest of the resource manager. +struct RRMUploadBatchTestHelper +{ + static void Initialize(ZEngine::Rendering::RenderResourceManager& manager, ZEngine::Hardwares::VulkanDevice& device, ZEngine::Hardwares::CommandBufferManager& commands, ZEngine::Rendering::Primitives::Semaphore* timeline, ArenaAllocator& arena, VkQueue queue) + { + device.m_queue_map.init(&arena, 2); + device.m_queue_map.insert(ZEngine::Rendering::QueueType::GRAPHIC_QUEUE, queue); + manager.m_device = &device; + manager.m_upload_cmd_mgr = &commands; + manager.m_batch_timeline = timeline; + manager.m_batch_frames.init(&arena, 3, 3); + for (auto& frame : manager.m_batch_frames) + frame = ZEngine::Rendering::RenderResourceManager::BatchFrameState{}; + } + + static void Append(ZEngine::Rendering::RenderResourceManager& manager, BufferView& target, uint8_t frame_index, uint32_t value, VkDeviceSize offset) + { + manager.EnsureBatchOpen(frame_index); + manager.AppendToGlobalBuffer(target, &value, sizeof(value), offset, frame_index); + } + + static bool IsOpen(const ZEngine::Rendering::RenderResourceManager& manager) + { + return manager.m_batch_mode; + } + + static uint32_t StagingCount(const ZEngine::Rendering::RenderResourceManager& manager, uint32_t frame_index) + { + return manager.m_batch_frames[frame_index].StagingCount; + } + + static uint64_t LastSignal(const ZEngine::Rendering::RenderResourceManager& manager, uint32_t frame_index) + { + return manager.m_batch_frames[frame_index].LastSignal; + } +}; + +class RenderResourceManagerBatchTest : public GpuAllocatorTest +{ +protected: + MemoryManager Memory{}; + std::unique_ptr Device = std::make_unique(); + std::unique_ptr Manager = std::make_unique(); + ZEngine::Hardwares::DeviceSwapchain Swapchain{}; + ZEngine::Hardwares::CommandBufferManager UploadCommands{}; + std::array, 3> Commands{}; + std::unique_ptr Timeline{}; + VkCommandPool Pool = VK_NULL_HANDLE; + BufferView Target{}; + bool Initialized = false; + + void SetUp() override + { + ASSERT_NE(s_allocator, nullptr); + Memory.Initialize(ZMega(4), {}); + auto& arena = Memory.MainArena; + Device->Arena = &arena; + Device->LogicalDevice = s_vk->Device; + Device->GraphicFamilyIndex = 0; + Device->SwapchainPtr = &Swapchain; + Device->GpuMem.Allocator = s_allocator->Allocator; + + VkQueue queue = VK_NULL_HANDLE; + vkGetDeviceQueue(s_vk->Device, 0, 0, &queue); + VkCommandPoolCreateInfo pool_info = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = 0}; + ASSERT_EQ(vkCreateCommandPool(s_vk->Device, &pool_info, nullptr, &Pool), VK_SUCCESS); + UploadCommands.Device = Device.get(); + UploadCommands.TotalThreadCount = 1; + const uint32_t stride = UploadCommands.MaxBufferPerPool * UploadCommands.MaxBufferPerPool; + UploadCommands.InstantGraphicsCommandBuffers.init(&arena, 3 * stride, 3 * stride); + for (uint32_t frame = 0; frame < Commands.size(); ++frame) + { + Commands[frame] = std::make_unique(Device.get(), Pool, ZEngine::Rendering::QueueType::GRAPHIC_QUEUE, true); + UploadCommands.InstantGraphicsCommandBuffers[frame * stride] = Commands[frame].get(); + } + Timeline = std::make_unique(Device.get(), true); + RRMUploadBatchTestHelper::Initialize(*Manager, *Device, UploadCommands, Timeline.get(), arena, queue); + Target = allocator().AllocateBuffer(64, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "batch_test_target"); + Initialized = true; + } + + void TearDown() override + { + if (Initialized) + { + Manager->EndFrame(); + vkDeviceWaitIdle(s_vk->Device); + Manager->BeginFrame(0); // Retire all submitted staging allocations. + allocator().FreeBuffer(Target); + } + for (auto& command : Commands) + command.reset(); + Timeline.reset(); + if (Device->LogicalDevice) + Device->PendingFree.Drain(&Device->GpuMem, Device->LogicalDevice, UINT64_MAX); + if (Pool) + vkDestroyCommandPool(s_vk->Device, Pool, nullptr); + } + + void Append(uint8_t frame, uint32_t value, VkDeviceSize offset = 0) + { + RRMUploadBatchTestHelper::Append(*Manager, Target, frame, value, offset); + } + + uint32_t Read(uint64_t signal_value, VkDeviceSize offset = 0) + { + Timeline->Wait(signal_value); + uint32_t value = 0; + EXPECT_EQ(vmaCopyAllocationToMemory(allocator().Allocator, Target.Allocation, offset, &value, sizeof(value)), VK_SUCCESS); + return value; + } +}; + +TEST_F(RenderResourceManagerBatchTest, StartupResizeSubmitsInitializationBeforeSwitchingSlots) +{ + Append(0, 42); + Manager->BeginFrame(1); // First acquired slot changes after startup recreation. + ASSERT_FALSE(RRMUploadBatchTestHelper::IsOpen(*Manager)); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 1u); + + ZEngine::Hardwares::AsyncGPUOperationHandle operation{}; + ASSERT_TRUE(Device->AsyncGPUOperations.pop(operation)); + EXPECT_EQ(operation.Timeline, Timeline.get()); + EXPECT_EQ(operation.SignalValue, 1u); + EXPECT_EQ(operation.StageFlags, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT); + + Append(1, 99, sizeof(uint32_t)); + Manager->EndFrame(); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 1), 2u); + EXPECT_EQ(Read(2), 42u); + EXPECT_EQ(Read(2, sizeof(uint32_t)), 99u); +} + +TEST_F(RenderResourceManagerBatchTest, SameSlotInitializationJoinsTheFirstFrame) +{ + Append(0, 42); + Manager->BeginFrame(0); + EXPECT_TRUE(RRMUploadBatchTestHelper::IsOpen(*Manager)); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 1u); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 0u); + + Append(0, 99, sizeof(uint32_t)); + Manager->EndFrame(); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 1u); + EXPECT_EQ(Read(1), 42u); + EXPECT_EQ(Read(1, sizeof(uint32_t)), 99u); +} + +TEST_F(RenderResourceManagerBatchTest, UploadJoinSubmitsThePreviousSlotsBatch) +{ + Append(0, 42); + Append(2, 99, sizeof(uint32_t)); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 1u); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 2), 1u); + Manager->EndFrame(); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 2), 2u); + EXPECT_EQ(Read(2), 42u); + EXPECT_EQ(Read(2, sizeof(uint32_t)), 99u); +} + +TEST_F(RenderResourceManagerBatchTest, PreviousSignalNeverRetiresAnOpenBatchsStaging) +{ + Append(0, 42); + Manager->EndFrame(); + EXPECT_EQ(Read(1), 42u); + Manager->BeginFrame(0); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 0u); + + Append(0, 99); + Manager->BeginFrame(0); // LastSignal=1 is complete, but the new copy is not submitted. + ASSERT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 1u); + Manager->EndFrame(); + EXPECT_EQ(Read(2), 99u); + Manager->BeginFrame(2); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 0u); +} diff --git a/ZEngine/tests/Rendering/ModelImporterOutput_test.cpp b/ZEngine/tests/Rendering/ModelImporterOutput_test.cpp new file mode 100644 index 00000000..64d9809c --- /dev/null +++ b/ZEngine/tests/Rendering/ModelImporterOutput_test.cpp @@ -0,0 +1,1661 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ZEngine::Core::VFS; +using namespace ZEngine::Importers; +using namespace ZEngine::Importers::AssetCodec; +using ZEngine::Core::Containers::ArrayView; +using ZEngine::Core::Memory::ArenaAllocator; + +namespace +{ + enum class FailurePoint + { + None, + CreateDir, + Open, + Write, + ShortWrite, + Flush, + Close, + Rename + }; + + // Use the real in-memory VFS; inject only the operation under test. + struct FaultVFSContext : VFSContext + { + struct FileProxy : IVFSFile + { + IVFSFile* Inner = nullptr; + FailurePoint Point = FailurePoint::None; + + VFSResult Read(ArrayView buffer, uint64_t offset) override + { + return Inner->Read(buffer, offset); + } + VFSResult Write(ArrayView buffer, uint64_t offset) override + { + if (Point == FailurePoint::Write) + return VFSResult::Fail(VFSError::IOError); + if (Point == FailurePoint::ShortWrite) + return Inner->Write({buffer.data(), buffer.size() - 1}, offset); + return Inner->Write(buffer, offset); + } + VFSResult Size() const override + { + return Inner->Size(); + } + VFSResult Stat() const override + { + return Inner->Stat(); + } + VFSResult Flush() override + { + return Point == FailurePoint::Flush ? VFSResult::Fail(VFSError::IOError) : Inner->Flush(); + } + const VFSPath& Path() const override + { + return Inner->Path(); + } + VFSResult Close() override + { + auto result = Inner->Close(); + return Point == FailurePoint::Close ? VFSResult::Fail(VFSError::IOError) : result; + } + } File; + + FailurePoint Point = FailurePoint::None; + std::string FailurePath; + size_t WriteNumber = 0; + size_t FailureWriteNumber = 0; + VFSError MetadataReadError = VFSError::OK; + std::string MetadataReadPath; + + bool Matches(const VFSPath& path) const + { + return (FailureWriteNumber == 0 || FailureWriteNumber == WriteNumber) && std::string(path.CStr()).find(FailurePath) != std::string::npos; + } + + VFSResult CreateDir(const VFSPath& path) override + { + if (Point == FailurePoint::CreateDir && Matches(path)) + return VFSResult::Fail(VFSError::PermissionDenied); + return VFSContext::CreateDir(path); + } + + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + if (!HasFlag(flags, VFSOpenFlags::Write)) + { + const std::string_view name(path.CStr()); + if (MetadataReadError != VFSError::OK && name.ends_with(".meta") && name.find(MetadataReadPath) != std::string_view::npos) + return VFSResult::Fail(MetadataReadError); + return VFSContext::Open(path, flags); + } + ++WriteNumber; + if (Point == FailurePoint::Open && Matches(path)) + return VFSResult::Fail(VFSError::PermissionDenied); + auto result = VFSContext::Open(path, flags); + if (result.Failed()) + return result; + File.Inner = result.Value(); + File.Point = Matches(path) ? Point : FailurePoint::None; + return VFSResult::Ok(&File); + } + + void Close(IVFSFile* file) override + { + if (file == &File) + { + VFSContext::Close(File.Inner); + File.Inner = nullptr; + } + else + VFSContext::Close(file); + } + + VFSResult Rename(const VFSPath& src, const VFSPath& dst) override + { + if (Point == FailurePoint::Rename && Matches(dst)) + return VFSResult::Fail(VFSError::IOError); + return VFSContext::Rename(src, dst); + } + }; + + struct ImportCallbacks + { + int Completed = 0; + int Failed = 0; + float Progress = 0; + std::string Error; + std::vector Outputs; + + static void Complete(void* context, ArrayView outputs) + { + auto& self = *static_cast(context); + ++self.Completed; + self.Outputs.clear(); + for (size_t i = 0; i < outputs.size(); ++i) + self.Outputs.push_back(outputs[i]); + } + static void Fail(void* context, std::string_view message) + { + auto& self = *static_cast(context); + ++self.Failed; + self.Error = message; + } + static void Advance(void* context, float progress) + { + static_cast(context)->Progress = progress; + } + }; + struct PausedImportCallbacks + { + ImportCallbacks Results; + std::promise Entered; + std::shared_future Release; + bool Paused = false; + + static void Complete(void* context, ArrayView outputs) + { + ImportCallbacks::Complete(&static_cast(context)->Results, outputs); + } + static void Fail(void* context, std::string_view message) + { + ImportCallbacks::Fail(&static_cast(context)->Results, message); + } + static void Advance(void* context, float progress) + { + auto& self = *static_cast(context); + self.Results.Progress = progress; + if (!self.Paused) + { + self.Paused = true; + self.Entered.set_value(); + self.Release.wait(); + } + } + }; +} // namespace + +class ModelImporterOutputTest : public ::testing::Test +{ +protected: + // Declaration order keeps arena storage alive through backend/importer destruction. + ZEngine::Core::Memory::MemoryManager Memory; + ArenaAllocator& Arena = Memory.MainArena; + VFSMemoryBackend Backend; + FaultVFSContext Context; + GltfImporter Importer; + ImportConfiguration Config; + ImportCallbacks Callbacks; + std::filesystem::path TempDirectory; + + void SetUp() override + { + // FBX reserves a 512 MiB sub-arena; pages are committed only as used. + Memory.Initialize(ZMega(768), {}); + Backend.Initialize(&Arena); + Context.Initialize(&Arena, 4); + ASSERT_TRUE(Context.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + Importer.Initialize(&Arena); + + Config.OutputWorkingSpacePath.init(&Arena, "/project"); + Config.OutputAssetsPath.init(&Arena, "/meshes"); + Config.OutputMaterialPath.init(&Arena, "/materials"); + Config.OutputTextureFilesPath.init(&Arena, "/textures"); + Config.OutputAssetFile.init(&Arena, "triangle.zemesh"); + Config.AssetName.init(&Arena, "triangle"); + Config.VFS = &Context; + Config.Options.ImportMaterials = false; + Config.Options.ImportTextures = false; + Config.Options.NormalsMode = 0; + + std::random_device rd; + std::mt19937 random(rd()); + uuids::uuid_random_generator uuid(random); + TempDirectory = std::filesystem::temp_directory_path() / ("zengine_import_output_" + uuids::to_string(uuid())); + ASSERT_TRUE(std::filesystem::create_directory(TempDirectory)); + Config.InputBaseAssetFilePath.init(&Arena, TempDirectory.string().c_str()); + } + + void TearDown() override + { + Context.Shutdown(); + std::error_code error; + std::filesystem::remove_all(TempDirectory, error); + } + + VFSPath Path(const char* raw) + { + return VFSPath::Parse(raw).Value(); + } + + bool Exists(const char* raw) + { + auto result = Context.Exists(Path(raw)); + return result.Succeeded() && result.Value(); + } + + void ExpectNoTemporaryFiles(const char* directory) + { + auto entries = Context.List(Path(directory), &Arena); + if (entries.Failed() && entries.Error() == VFSError::NotFound) + return; + ASSERT_TRUE(entries.Succeeded()); + for (const auto& entry : entries.Value()) + { + EXPECT_FALSE(std::string_view(entry.Path.CStr()).ends_with(".tmp")) << entry.Path.CStr(); + if (entry.IsDirectory) + ExpectNoTemporaryFiles(entry.Path.CStr()); + } + } + + std::vector ReadFile(const char* raw) + { + auto open = Context.Open(Path(raw), VFSOpenFlags::Read); + EXPECT_TRUE(open.Succeeded()); + if (open.Failed()) + return {}; + auto size = open.Value()->Size(); + EXPECT_TRUE(size.Succeeded()); + std::vector bytes(size.Succeeded() ? size.Value() : 0); + auto read = open.Value()->ReadAll({bytes.data(), bytes.size()}); + EXPECT_TRUE(read.Succeeded()); + Context.Close(open.Value()); + return bytes; + } + + std::string WriteGltf(size_t material_count = 0) + { + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + { + std::ofstream file(TempDirectory / "triangle.bin", std::ios::binary); + file.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + } + auto document = nlohmann::json::parse(R"({ + "asset": {"version": "2.0"}, + "buffers": [{"uri": "triangle.bin", "byteLength": 36}], + "bufferViews": [{"buffer": 0, "byteLength": 36}], + "accessors": [{"bufferView": 0, "componentType": 5126, "count": 3, "type": "VEC3", "min": [0,0,0], "max": [1,1,0]}], + "meshes": [{"primitives": [{"attributes": {"POSITION": 0}}]}], + "nodes": [{"mesh": 0}], "scenes": [{"nodes": [0]}], "scene": 0 + })"); + if (material_count) + { + document["meshes"][0]["primitives"][0]["material"] = 0; + for (size_t i = 0; i < material_count; ++i) + document["materials"].push_back({ + {"name", std::string(80, 'm') + std::to_string(i)} + }); + } + const auto filename = (TempDirectory / "triangle.gltf").string(); + std::ofstream(filename) << document.dump(); + return filename; + } + + void ImportGltf(const std::string& filename) + { + Importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + } + + std::string WriteTexturedGltf() + { + const auto filename = WriteGltf(1); + nlohmann::json document; + { + std::ifstream file(filename); + file >> document; + } + document["images"] = { + {{"name", "one_pixel"}, {"uri", "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a3ioAAAAASUVORK5CYII="}} + }; + document["textures"] = {{{"source", 0}}}; + document["materials"][0]["pbrMetallicRoughness"]["baseColorTexture"]["index"] = 0; + std::ofstream(filename) << document.dump(); + return filename; + } + + void ChangeTriangleVertices() + { + const std::array vertices = {0.25f, 0, 0, 1, 0, 0, 0, 1, 0}; + std::ofstream file(TempDirectory / "triangle.bin", std::ios::binary); + file.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + } + + std::string WriteObj(bool with_normals = false) + { + const auto filename = (TempDirectory / "triangle.obj").string(); + std::ofstream file(filename); + file << "o Triangle\nv 0 0 0\nv 1 0 0\nv 0 1 0\n"; + file << (with_normals ? "vn 0 0 1\nf 1//1 2//1 3//1\n" : "f 1 2 3\n"); + return filename; + } + + std::string WriteFbx(bool with_material = false) + { + const auto filename = (TempDirectory / "triangle.fbx").string(); + std::string contents = R"(; FBX 7.4.0 project file +FBXHeaderExtension: { FBXHeaderVersion: 1003 + FBXVersion: 7400 +} +Objects: { + Geometry: 1, "Geometry::Triangle", "Mesh" { + Vertices: *9 { a: 0,0,0,1,0,0,0,1,0 } + PolygonVertexIndex: *3 { a: 0,1,-3 } + } + Model: 2, "Model::Triangle", "Mesh" { Version: 232 } +} +Connections: { + C: "OO",1,2 + C: "OO",2,0 +} +)"; + if (with_material) + { + contents.insert(contents.find("}\nConnections:"), R"( Material: 3, "Material::TestMaterial", "" { + Version: 102 + ShadingModel: "phong" + Properties70: { P: "DiffuseColor", "Color", "", "A", 1, 0, 0 } + } +)"); + contents.insert(contents.rfind('}'), " C: \"OO\",3,2\n"); + } + std::ofstream(filename) << contents; + return filename; + } + + void ReadCookedMesh(AssetMesh& mesh, AssetNodeHierarchy& hierarchy) + { + const auto bytes = ReadFile("/meshes/triangle.zemesh"); + const auto filename = (TempDirectory / "triangle.zemesh").string(); + { + std::ofstream file(filename, std::ios::binary); + file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + DeserializeMeshAssetFile(&Arena, filename.c_str(), mesh, hierarchy); + } + + template + void VerifyInvalidConfigurations(ModelImporter& importer, const std::string& filename) + { + const auto start = importer.Arena.m_current_offset; + auto verify = [&](const ImportConfiguration& config, const char* source) { + const auto failures = Callbacks.Failed; + importer.ImportFile(source, config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, failures + 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(importer.Arena.m_current_offset, start); + EXPECT_NE(Callbacks.Error.find("VFS error"), std::string::npos); + }; + const std::string long_path = "/" + std::string(MAX_FILE_PATH_COUNT, 'x'); + auto invalid = Config; + invalid.VFS = nullptr; + verify(invalid, filename.c_str()); + invalid = Config; + invalid.OutputAssetsPath.init(&Arena, long_path.c_str()); + verify(invalid, filename.c_str()); + invalid = Config; + invalid.OutputAssetFile.init(&Arena, ""); + verify(invalid, filename.c_str()); + invalid.OutputAssetFile.init(&Arena, std::string(MAX_FILE_PATH_COUNT, 'x').c_str()); + verify(invalid, filename.c_str()); + invalid.OutputAssetFile.init(&Arena, std::string(MAX_FILE_PATH_COUNT - 1 - std::string_view("/meshes/").size(), 'x').c_str()); + verify(invalid, filename.c_str()); // Fits VFS, but not its .meta.tmp sidecar. + invalid = Config; + invalid.Options.ImportMaterials = true; + invalid.OutputMaterialPath.init(&Arena, long_path.c_str()); + verify(invalid, filename.c_str()); + invalid = Config; + invalid.Options.ImportMaterials = invalid.Options.ImportTextures = true; + invalid.OutputTextureFilesPath.init(&Arena, long_path.c_str()); + verify(invalid, filename.c_str()); + invalid.OutputTextureFilesPath = Config.OutputTextureFilesPath; + invalid.AssetName.init(&Arena, ""); + verify(invalid, filename.c_str()); + verify(Config, nullptr); + verify(Config, ""); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + } + + template + void VerifySameInstanceSerialization(ModelImporter& importer, const std::string& filename, bool runtime_call) + { + using namespace std::chrono_literals; + std::promise release; + PausedImportCallbacks paused; + paused.Release = release.get_future().share(); + auto entered = paused.Entered.get_future(); + ImportConfiguration second_config = Config; + second_config.OutputAssetFile.init(&Arena, "second.zemesh"); + + auto first = std::async(std::launch::async, [&] { importer.ImportFile(filename.c_str(), Config, &Arena, &paused, PausedImportCallbacks::Complete, PausedImportCallbacks::Advance, PausedImportCallbacks::Fail, nullptr); }); + const bool first_paused = entered.wait_for(5s) == std::future_status::ready; + EXPECT_TRUE(first_paused); + if (!first_paused) + { + release.set_value(); + first.get(); + return; + } + + std::promise second_started; + auto started = second_started.get_future(); + auto second = std::async(std::launch::async, [&] { + second_started.set_value(); + if (runtime_call) + { + // Even a failed runtime import must respect the editor import's lock. + auto result = importer.Import(Context, Path("/zengine_nonexistent_import_source"), {}); + EXPECT_TRUE(result.Failed()); + } + else + importer.ImportFile(filename.c_str(), second_config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + }); + EXPECT_EQ(started.wait_for(5s), std::future_status::ready); + EXPECT_EQ(second.wait_for(100ms), std::future_status::timeout); + release.set_value(); + first.get(); + second.get(); + EXPECT_EQ(paused.Results.Completed, 1) << paused.Results.Error; + EXPECT_EQ(paused.Results.Failed, 0); + if (!runtime_call) + { + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + EXPECT_EQ(Callbacks.Failed, 0); + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + EXPECT_EQ(Callbacks.Outputs[0].Path, "/meshes/second.zemesh"); + } + } +}; + +TEST_F(ModelImporterOutputTest, MeshOnlyGltfOwnsShortAndEmptyOutputStringsAfterImport) +{ + Config.OutputAssetsPath.init(&Arena, "/m"); + Config.OutputAssetFile.init(&Arena, "x.zemesh"); + Config.OutputWorkingSpacePath.init(&Arena, ""); + // A source material must be harmless when material/texture import is disabled. + ImportGltf(WriteGltf(1)); + + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + EXPECT_EQ(Callbacks.Outputs[0].Type, AssetFileType::MESH); + EXPECT_EQ(Callbacks.Outputs[0].Path, "/m/x.zemesh"); + EXPECT_TRUE(Callbacks.Outputs[0].RootPath.empty()); + EXPECT_TRUE(Exists("/m/x.zemesh")); + EXPECT_FLOAT_EQ(Callbacks.Progress, 1.0f); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); +} + +TEST_F(ModelImporterOutputTest, InvalidGltfNodeGraphsFailInEditorAndRuntimeWithoutPublication) +{ + for (int scenario = 0; scenario < 8; ++scenario) + { + SCOPED_TRACE(scenario); + Callbacks = {}; + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + switch (scenario) + { + case 0: + document["scene"] = 1000000; + break; + case 1: + document["scenes"][0]["nodes"] = {1000000}; + break; + case 2: + document["nodes"][0]["mesh"] = 1000000; + break; + case 3: + document["nodes"][0]["children"] = {1000000}; + break; + case 4: + document["nodes"][0]["children"] = {0}; + break; + case 5: + document["nodes"][0]["children"] = {1}; + document["nodes"].push_back({ + {"children", {0}} + }); + break; + case 6: + document["nodes"][0]["children"] = {2}; + document["nodes"].push_back({ + {"children", {2}} + }); + document["nodes"].push_back(nlohmann::json::object()); + document["scenes"][0]["nodes"] = {0, 1}; + break; + case 7: + document["scenes"][0]["nodes"] = {0, 0}; + break; + } + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_EQ(Importer.Import(Context, Path(filename.c_str()), {}).Error(), VFSError::Corrupted); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + } +} + +TEST_F(ModelImporterOutputTest, DeepGltfHierarchyUsesIterativeTraversalAndSizedMaps) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document["nodes"] = nlohmann::json::array(); + constexpr size_t count = 4096; + for (size_t i = 0; i + 1 < count; ++i) + document["nodes"].push_back({ + {"children", {i + 1}} + }); + document["nodes"].push_back({ + {"mesh", 0} + }); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Failed, 0); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(hierarchy.Hierarchies.size(), count + 1); + EXPECT_EQ(hierarchy.NodeNames.size(), count + 1); + EXPECT_EQ(hierarchy.NodeMeshes.size(), 1u); + EXPECT_EQ(hierarchy.Hierarchies.back().DepthLevel, count); +} + +TEST_F(ModelImporterOutputTest, MultipleGltfScenesCanReuseTheSameTree) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document.erase("scene"); + document["scenes"].push_back(document["scenes"][0]); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + EXPECT_EQ(hierarchy.Hierarchies.size(), 4u); + EXPECT_EQ(hierarchy.NodeMeshes.size(), 2u); +} + +TEST_F(ModelImporterOutputTest, OversizedValidGltfFailsCleanlyAndImporterRemainsReusable) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + constexpr size_t count = 1050000; + const std::vector vertices(count * 3, 0.0f); + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), vertices.size() * sizeof(float)); + } + document["buffers"][0]["byteLength"] = vertices.size() * sizeof(float); + document["bufferViews"][0]["byteLength"] = vertices.size() * sizeof(float); + document["accessors"][0]["count"] = count; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_NE(Callbacks.Error.find("scratch memory budget"), std::string::npos); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_EQ(Importer.Import(Context, Path(filename.c_str()), {}).Error(), VFSError::OutOfMemory); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + ImportGltf(WriteGltf()); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, DuplicateGltfImageNamesAreDistinctButSharedImagesKeepIdentity) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + const std::array red = {255, 0, 0, 255}; + const std::array green = {0, 255, 0, 255}; + ASSERT_TRUE(stbi_write_png((TempDirectory / "red.png").string().c_str(), 1, 1, 4, red.data(), 4)); + ASSERT_TRUE(stbi_write_png((TempDirectory / "green.png").string().c_str(), 1, 1, 4, green.data(), 4)); + nlohmann::json document; + std::ifstream(filename) >> document; + document["images"][0]["uri"] = "red.png"; + document["images"].push_back(document["images"][0]); + document["images"][1]["uri"] = "green.png"; + document["textures"].push_back({ + {"source", 1} + }); + document["textures"].push_back({ + {"source", 0} + }); + for (int i = 1; i < 3; ++i) + { + auto material = document["materials"][0]; + material["name"] = std::to_string(i); + material["pbrMetallicRoughness"]["baseColorTexture"]["index"] = i; + document["materials"].push_back(material); + } + std::ofstream(filename) << document.dump(); + auto textures = [&] { + std::vector result; + for (size_t i = 1; i < Callbacks.Outputs.size(); ++i) + { + const auto bytes = ReadFile(Callbacks.Outputs[i].Path.c_str()); + result.push_back(nlohmann::json::parse(bytes.begin(), bytes.end())["textures"]["albedo"]); + } + return result; + }; + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto first = textures(); + ASSERT_EQ(first.size(), 3u); + EXPECT_NE(first[0]["path"], first[1]["path"]); + EXPECT_NE(first[0]["uuid"], first[1]["uuid"]); + EXPECT_EQ(first[0], first[2]); + EXPECT_NE(ReadFile("/textures/triangle/one_pixel_0.png"), ReadFile("/textures/triangle/one_pixel_1.png")); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + EXPECT_EQ(textures(), first); +} + +TEST_F(ModelImporterOutputTest, NativeTextureCopiesWithSameBasenameKeepDifferentContents) +{ + ASSERT_TRUE(std::filesystem::create_directory(TempDirectory / "red")); + ASSERT_TRUE(std::filesystem::create_directory(TempDirectory / "green")); + std::ofstream(TempDirectory / "red/same.png", std::ios::binary) << "red image"; + std::ofstream(TempDirectory / "green/same.png", std::ios::binary) << "green image"; + ZEngine::Core::Containers::Array textures; + textures.init(&Arena, 2, 2); + textures[0].Path.init(&Arena, "red/same.png"); + textures[1].Path.init(&Arena, "green/same.png"); + ASSERT_TRUE(CopyTextureFiles(ArrayView{textures}, Config).Succeeded()); + EXPECT_STREQ(textures[0].Path.c_str(), "/textures/triangle/same_0.png"); + EXPECT_STREQ(textures[1].Path.c_str(), "/textures/triangle/same_1.png"); + EXPECT_NE(ReadFile(textures[0].Path.c_str()), ReadFile(textures[1].Path.c_str())); +} + +TEST_F(ModelImporterOutputTest, UnnamedSharedGltfImageUsesOneStableDestination) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document["images"][0].erase("name"); + document["textures"].push_back({ + {"source", 0} + }); + document["materials"][0]["normalTexture"] = { + {"index", 1} + }; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Failed, 0); + const auto bytes = ReadFile(Callbacks.Outputs[1].Path.c_str()); + const auto material = nlohmann::json::parse(bytes.begin(), bytes.end()); + EXPECT_EQ(material["textures"]["albedo"]["path"], "/textures/triangle/image_0.png"); + EXPECT_EQ(material["textures"]["albedo"], material["textures"]["normal"]); +} + +TEST_F(ModelImporterOutputTest, CorruptTextureIdentityIsRepairedBeforeMaterialPublication) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + ASSERT_TRUE(Context.CreateDir(Path("/textures/triangle")).Succeeded()); + const std::string metadata = R"({"uuid":"invalid","source_hash":0})"; + ASSERT_TRUE(WriteFileAtomically(Context, Path("/textures/triangle/one_pixel_0.png.meta"), {reinterpret_cast(metadata.data()), metadata.size()}).Succeeded()); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Failed, 0); + const auto bytes = ReadFile(Callbacks.Outputs[1].Path.c_str()); + const auto material = nlohmann::json::parse(bytes.begin(), bytes.end()); + auto meta = MetaFileIO::Read(Context, Path("/textures/triangle/one_pixel_0.png")); + ASSERT_TRUE(meta.Succeeded()); + EXPECT_FALSE(meta.Value().AssetUUID.is_nil()); + EXPECT_EQ(material["textures"]["albedo"]["uuid"], uuids::to_string(meta.Value().AssetUUID)); +} + +TEST_F(ModelImporterOutputTest, ModelMetadataReadErrorsNeverPublishReplacementIdentities) +{ + auto verify = [&](auto& importer, const std::string& filename) { + for (bool material : {false, true}) + { + Config.Options.ImportMaterials = material; + Context.MetadataReadPath = material ? "/materials/" : "/meshes/"; + for (auto error : {VFSError::PermissionDenied, VFSError::IOError, VFSError::OutOfMemory}) + { + SCOPED_TRACE(static_cast(error)); + Context.MetadataReadError = error; + Callbacks = {}; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1) << Callbacks.Error; + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + } + } + Context.MetadataReadError = VFSError::OK; + }; + verify(Importer, WriteGltf(1)); + AssimpImporter assimp; + assimp.Initialize(&Arena); + verify(assimp, WriteObj()); + FbxImporter fbx; + fbx.Initialize(&Arena); + verify(fbx, WriteFbx(true)); +} + +TEST_F(ModelImporterOutputTest, MaterialOutputsExceedOldCapacityAndOwnLongPaths) +{ + const std::string root = "/project/" + std::string(100, 'p'); + Config.OutputWorkingSpacePath.init(&Arena, root.c_str()); + Config.Options.ImportMaterials = true; + ImportGltf(WriteGltf(20)); + + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 21u); + for (size_t i = 0; i < 20; ++i) + { + const auto& output = Callbacks.Outputs[i + 1]; + EXPECT_EQ(output.Type, AssetFileType::MATERIAL); + EXPECT_EQ(output.Path, "/materials/triangle/" + std::string(80, 'm') + std::to_string(i) + "_" + std::to_string(i) + ".zematerial"); + EXPECT_EQ(output.RootPath, root); + EXPECT_TRUE(Exists(output.Path.c_str())); + } +} + +TEST_F(ModelImporterOutputTest, MeshOnlyGltfArtifactRoundTrips) +{ + ImportGltf(WriteGltf()); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.Vertices.size(), 24u); + ASSERT_EQ(mesh.Indices.size(), 3u); + ASSERT_EQ(mesh.SubMeshes.size(), 1u); + EXPECT_EQ(mesh.MeshUUID, hierarchy.MeshUUID); + EXPECT_EQ(mesh.SubMeshes[0].VertexCount, 3u); + EXPECT_EQ(mesh.SubMeshes[0].IndexCount, 3u); + EXPECT_TRUE(hierarchy.MaterialNames.empty()); +} + +TEST_F(ModelImporterOutputTest, MaterialWriteFailureNeverReportsSuccess) +{ + Config.Options.ImportMaterials = true; + Context.Point = FailurePoint::Write; + Context.FailurePath = "/materials/"; + ImportGltf(WriteGltf(2)); + + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_TRUE(Callbacks.Outputs.empty()); + EXPECT_LT(Callbacks.Progress, 1.0f); + EXPECT_NE(Callbacks.Error.find("/materials/triangle/" + std::string(80, 'm') + "0_0.zematerial"), std::string::npos); + EXPECT_NE(Callbacks.Error.find("VFS error 8"), std::string::npos); + ExpectNoTemporaryFiles("/materials"); +} + +TEST_F(ModelImporterOutputTest, AssimpWriteFailureReleasesScratchAndAllowsRetry) +{ + const auto filename = WriteObj(true); + Config.Options.ImportMaterials = true; + AssimpImporter importer; + importer.Initialize(&Arena); + Context.Point = FailurePoint::Write; + Context.FailurePath = "/meshes/"; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_NE(Callbacks.Error.find("/meshes/triangle.zemesh"), std::string::npos); + EXPECT_EQ(importer.Arena.m_current_offset, importer.Arena.m_initial_current_offset); + // The same importer must still work after failure. + Context.Point = FailurePoint::None; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Completed, 1); +} + +TEST_F(ModelImporterOutputTest, FbxWriteFailureReleasesScratchAndAllowsRetry) +{ + const auto filename = WriteFbx(); + FbxImporter importer; + importer.Initialize(&Arena); + Context.Point = FailurePoint::Write; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + + ASSERT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_NE(Callbacks.Error.find("/meshes/triangle.zemesh"), std::string::npos) << Callbacks.Error; + Context.Point = FailurePoint::None; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + EXPECT_EQ(Callbacks.Outputs[0].Path, "/meshes/triangle.zemesh"); +} + +TEST_F(ModelImporterOutputTest, AssimpWithoutNormalsOrMaterialsUsesFallbacks) +{ + const auto filename = WriteObj(); + AssimpImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.Vertices.size(), 24u); + ASSERT_EQ(mesh.SubMeshes.size(), 1u); + EXPECT_TRUE(mesh.SubMeshes[0].MaterialUUID.is_nil()); + EXPECT_TRUE(hierarchy.MaterialNames.empty()); + EXPECT_TRUE(hierarchy.NodeMaterials.empty()); + for (size_t vertex = 0; vertex < 3; ++vertex) + { + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 3], 0); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 4], 1); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 5], 0); + } +} + +TEST_F(ModelImporterOutputTest, AssimpPreservesProvidedNormalsAndMaterialIdentity) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteObj(true); + AssimpImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 2u); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.SubMeshes.size(), 1u); + EXPECT_FALSE(mesh.SubMeshes[0].MaterialUUID.is_nil()); + EXPECT_FALSE(hierarchy.NodeMaterials.empty()); + EXPECT_EQ(hierarchy.MaterialNames.size(), 1u); + for (size_t vertex = 0; vertex < 3; ++vertex) + { + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 3], 0); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 4], 0); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 5], 1); + } +} + +TEST_F(ModelImporterOutputTest, GltfEditorImportsShareOneScratchLock) +{ + VerifySameInstanceSerialization(Importer, WriteGltf(), false); +} + +TEST_F(ModelImporterOutputTest, GltfRuntimeAndEditorImportsShareOneScratchLock) +{ + VerifySameInstanceSerialization(Importer, WriteGltf(), true); +} + +TEST_F(ModelImporterOutputTest, AssimpEditorImportsShareOneScratchLock) +{ + AssimpImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteObj(), false); +} + +TEST_F(ModelImporterOutputTest, AssimpRuntimeAndEditorImportsShareOneScratchLock) +{ + AssimpImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteObj(), true); +} + +TEST_F(ModelImporterOutputTest, FbxEditorImportsShareOneScratchLock) +{ + FbxImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteFbx(), false); +} + +TEST_F(ModelImporterOutputTest, FbxRuntimeAndEditorImportsShareOneScratchLock) +{ + FbxImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteFbx(), true); +} + +TEST_F(ModelImporterOutputTest, SeparateImporterInstancesRemainIndependent) +{ + using namespace std::chrono_literals; + const auto filename = WriteGltf(); + GltfImporter other; + other.Initialize(&Arena); + std::promise release; + PausedImportCallbacks paused; + paused.Release = release.get_future().share(); + auto entered = paused.Entered.get_future(); + auto first = std::async(std::launch::async, [&] { Importer.ImportFile(filename.c_str(), Config, &Arena, &paused, PausedImportCallbacks::Complete, PausedImportCallbacks::Advance, PausedImportCallbacks::Fail, nullptr); }); + EXPECT_EQ(entered.wait_for(5s), std::future_status::ready); + auto second = std::async(std::launch::async, [&] { other.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); }); + EXPECT_EQ(second.wait_for(5s), std::future_status::ready); + release.set_value(); + first.get(); + second.get(); + EXPECT_EQ(paused.Results.Completed, 1) << paused.Results.Error; + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, CodecRejectsInvalidOutputInsteadOfReturningAnEmptyAsset) +{ + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + Config.OutputAssetFile.init(&Arena, ""); + EXPECT_EQ(SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config).Error(), VFSError::InvalidPath); + Config.OutputAssetFile.init(&Arena, "triangle.zemesh"); + Config.OutputAssetsPath.init(&Arena, ("/" + std::string(MAX_FILE_PATH_COUNT, 'x')).c_str()); + EXPECT_EQ(SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config).Error(), VFSError::InvalidPath); + Config.VFS = nullptr; + EXPECT_EQ(SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config).Error(), VFSError::Unsupported); +} + +TEST_F(ModelImporterOutputTest, GltfInvalidConfigurationsReportErrorsAndAllowRetry) +{ + VerifyInvalidConfigurations(Importer, WriteGltf()); +} + +TEST_F(ModelImporterOutputTest, AssimpInvalidConfigurationsReportErrorsAndAllowRetry) +{ + AssimpImporter importer; + importer.Initialize(&Arena); + VerifyInvalidConfigurations(importer, WriteObj()); +} + +TEST_F(ModelImporterOutputTest, FbxInvalidConfigurationsReportErrorsAndAllowRetry) +{ + FbxImporter importer; + importer.Initialize(&Arena); + VerifyInvalidConfigurations(importer, WriteFbx()); +} + +TEST_F(ModelImporterOutputTest, InvalidMaterialNameReportsErrorBeforePublishingMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(1); + nlohmann::json document; + { + std::ifstream file(filename); + file >> document; + } + document["materials"][0]["name"] = std::string(MAX_FILE_PATH_COUNT, 'm'); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); +} + +TEST_F(ModelImporterOutputTest, FailedMaterialRecookPreservesPreviouslyPublishedMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(2); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1); + const auto before = ReadFile("/meshes/triangle.zemesh"); + ChangeTriangleVertices(); + // Fail the second material, after the first dependency was published. + Context.FailurePath = std::string(80, 'm') + "1_1.zematerial"; + // Unique temporary names no longer contain the destination filename; + // fail its final rename instead, preserving the old material as well. + Context.Point = FailurePoint::Rename; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before); + ExpectNoTemporaryFiles("/materials"); + Context.Point = FailurePoint::None; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 2); + EXPECT_NE(ReadFile("/meshes/triangle.zemesh"), before); +} + +TEST_F(ModelImporterOutputTest, AssimpMaterialWriteFailurePreservesPreviouslyPublishedMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteObj(); + AssimpImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + ASSERT_EQ(Callbacks.Completed, 1); + const auto before = ReadFile("/meshes/triangle.zemesh"); + std::ofstream(filename) << "o Triangle\nv 0.25 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n"; + Context.Point = FailurePoint::Write; + Context.FailurePath = "/materials/"; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before); + ExpectNoTemporaryFiles("/materials"); +} + +TEST_F(ModelImporterOutputTest, FbxMaterialWriteFailurePreservesPreviouslyPublishedMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteFbx(true); + FbxImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Outputs.size(), 2u); + const auto before = ReadFile("/meshes/triangle.zemesh"); + std::string contents; + { + std::ifstream file(filename); + contents.assign(std::istreambuf_iterator(file), std::istreambuf_iterator()); + } + contents.replace(contents.find("0,0,0,1,0,0"), std::string_view("0,0,0,1,0,0").size(), "0.25,0,0,1,0,0"); + std::ofstream(filename) << contents; + Context.Point = FailurePoint::Write; + Context.FailurePath = "/materials/"; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before); + ExpectNoTemporaryFiles("/materials"); +} + +TEST_F(ModelImporterOutputTest, IndependentAtomicWritersUseDifferentTemporarySiblings) +{ + struct InterleavedContext : VFSContext + { + bool Interleaved = false; + bool NestedSucceeded = false; + std::vector TemporaryPaths; + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + if (HasFlag(flags, VFSOpenFlags::Write)) + TemporaryPaths.push_back(path); + return VFSContext::Open(path, flags); + } + VFSResult Rename(const VFSPath& source, const VFSPath& destination) override + { + if (!Interleaved) + { + Interleaved = true; + const std::array other = {9, 9}; + NestedSucceeded = WriteFileAtomically(*this, destination, {other.data(), other.size()}).Succeeded(); + } + return VFSContext::Rename(source, destination); + } + } interleaved; + interleaved.Initialize(&Arena, 4); + ASSERT_TRUE(interleaved.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + ASSERT_TRUE(interleaved.CreateDir(Path("/meshes")).Succeeded()); + const std::array bytes = {1, 2, 3, 4}; + EXPECT_TRUE(WriteFileAtomically(interleaved, Path("/meshes/collision.zemesh"), {bytes.data(), bytes.size()}).Succeeded()); + EXPECT_TRUE(interleaved.NestedSucceeded); + ASSERT_EQ(interleaved.TemporaryPaths.size(), 2u); + EXPECT_NE(interleaved.TemporaryPaths[0], interleaved.TemporaryPaths[1]); + EXPECT_EQ(ReadFile("/meshes/collision.zemesh"), (std::vector{1, 2, 3, 4})); + ExpectNoTemporaryFiles("/meshes"); + interleaved.Shutdown(); +} + +TEST_F(ModelImporterOutputTest, AtomicWriteSupportsMaximumLengthDestinationAndEmptyData) +{ + const std::string filename(MAX_FILE_PATH_COUNT - 1 - std::string_view("/meshes/").size(), 'x'); + auto path = VFSPath::Parse(("/meshes/" + filename).c_str()); + ASSERT_TRUE(path.Succeeded()); + EXPECT_EQ(MakeOutputPath("/meshes", filename.c_str()).Error(), VFSError::InvalidPath); + ASSERT_TRUE(Context.CreateDir(Path("/meshes")).Succeeded()); + EXPECT_TRUE(WriteFileAtomically(Context, path.Value(), {}).Succeeded()); + EXPECT_TRUE(ReadFile(path.Value().CStr()).empty()); + EXPECT_EQ(WriteFileAtomically(Context, VFSPath::Root(), {}).Error(), VFSError::InvalidPath); + EXPECT_EQ(WriteFileAtomically(Context, VFSPath{}, {}).Error(), VFSError::InvalidPath); + ExpectNoTemporaryFiles("/meshes"); +} + +TEST_F(ModelImporterOutputTest, CookedPathsReserveRoomForMetadataSidecars) +{ + const std::string filename(MAX_FILE_PATH_COUNT - 1 - std::string_view("/meshes/").size() - std::string_view(".meta").size(), 'x'); + EXPECT_TRUE(MakeOutputPath("/meshes", filename.c_str()).Succeeded()); + EXPECT_EQ(MakeOutputPath("/meshes", (filename + "x").c_str()).Error(), VFSError::InvalidPath); + EXPECT_EQ(MakeOutputPath("/meshes", "../outside.zemesh").Error(), VFSError::InvalidPath); +} + +TEST_F(ModelImporterOutputTest, CodecSupportsUninitializedOptionalRootAndMaterialName) +{ + Config.OutputWorkingSpacePath = {}; + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + auto mesh_result = SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config); + ASSERT_TRUE(mesh_result.Succeeded()); + EXPECT_TRUE(mesh_result.Value().RootPath.empty()); + AssetMaterial material; + auto material_result = SerializeMaterialAssetFile(&Arena, material, Config, 0); + ASSERT_TRUE(material_result.Succeeded()); + EXPECT_EQ(material_result.Value().Path, "/materials/triangle/material_0.zematerial"); + EXPECT_TRUE(material_result.Value().RootPath.empty()); + Config.AssetName = {}; + EXPECT_EQ(SerializeTextureAssetFiles(&Arena, {}, Config).Error(), VFSError::InvalidPath); +} + +TEST_F(ModelImporterOutputTest, SkippedPrimitivesDoNotShiftMaterialsOrHierarchySubmeshIndices) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(2); + nlohmann::json document; + { + std::ifstream file(filename); + file >> document; + } + auto triangle = document["meshes"][0]["primitives"][0]; + triangle["material"] = 1; + auto points = triangle; + points["mode"] = 0; + points["material"] = 0; + // Includes a mesh with no surviving primitives, plus a multi-primitive mesh. + document["meshes"] = {{{"primitives", {points}}}, {{"primitives", {points, triangle, triangle}}}, {{"primitives", {triangle}}}}; + document["nodes"] = {{{"mesh", 0}}, {{"mesh", 1}}, {{"mesh", 2}}}; + document["scenes"][0]["nodes"] = {0, 1, 2}; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Outputs.size(), 3u); + const auto material = nlohmann::json::parse(ReadFile(Callbacks.Outputs[2].Path.c_str())); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.SubMeshes.size(), 3u); + for (const auto& submesh : mesh.SubMeshes) + EXPECT_EQ(uuids::to_string(submesh.MaterialUUID), material["uuid"].get()); + ASSERT_EQ(hierarchy.NodeMeshes.size(), 3u); + std::array mapped = {}; + for (const auto& [node, index] : hierarchy.NodeMeshes) + { + ASSERT_LT(index, mapped.size()); + mapped[index] = true; + } + EXPECT_TRUE(mapped[0] && mapped[1] && mapped[2]); +} + +TEST_F(ModelImporterOutputTest, DisabledGltfMaterialsLeaveNoDanglingHierarchyMaterialIndices) +{ + ImportGltf(WriteGltf(1)); + ASSERT_EQ(Callbacks.Completed, 1); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + EXPECT_TRUE(hierarchy.NodeMaterials.empty()); + EXPECT_TRUE(hierarchy.MaterialNames.empty()); + EXPECT_TRUE(mesh.SubMeshes[0].MaterialUUID.is_nil()); +} + +TEST_F(ModelImporterOutputTest, SharedTextureCopyReportsMissingSourcesAndWriteErrors) +{ + ZEngine::Core::Containers::Array textures; + textures.init(&Arena, 1, 1); + textures[0].Path.init(&Arena, "missing.png"); + EXPECT_EQ(CopyTextureFiles(ArrayView{textures}, Config).Error(), VFSError::NotFound); + std::ofstream(TempDirectory / "present.png", std::ios::binary) << "test image bytes"; + textures[0].Path.init(&Arena, "present.png"); + Context.Point = FailurePoint::Write; + Context.FailurePath = "/textures/"; + EXPECT_EQ(CopyTextureFiles(ArrayView{textures}, Config).Error(), VFSError::IOError); + EXPECT_STREQ(textures[0].Path.c_str(), "present.png"); + ExpectNoTemporaryFiles("/textures/triangle"); + Context.Point = FailurePoint::None; + EXPECT_TRUE(CopyTextureFiles(ArrayView{textures}, Config).Succeeded()); + EXPECT_STREQ(textures[0].Path.c_str(), "/textures/triangle/present_0.png"); +} + +TEST_F(ModelImporterOutputTest, SharedGltfTextureUpdatesEveryMaterialSlotAndSurvivesRecook) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + auto& material = document["materials"][0]; + material["emissiveTexture"] = { + {"index", 0} + }; + material["normalTexture"] = { + {"index", 0} + }; + material["occlusionTexture"] = { + {"index", 0} + }; + material["pbrMetallicRoughness"]["metallicRoughnessTexture"] = { + {"index", 0} + }; + auto second = material; + second["name"] = "second"; + document["materials"].push_back(second); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 3u); + const auto first = nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str())); + const auto expected = first["textures"]["albedo"]; + EXPECT_EQ(expected["path"], "/textures/triangle/one_pixel_0.png"); + for (size_t m = 1; m < Callbacks.Outputs.size(); ++m) + { + const auto cooked = nlohmann::json::parse(ReadFile(Callbacks.Outputs[m].Path.c_str())); + for (const auto* slot : {"albedo", "emissive", "normal", "opacity", "specular"}) + EXPECT_EQ(cooked["textures"][slot], expected); + } + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2); + EXPECT_EQ(nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str()))["textures"]["albedo"], expected); +} + +TEST_F(ModelImporterOutputTest, UnnamedAndDuplicateMaterialsKeepDistinctStableOutputsAndDisplayNames) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(3); + nlohmann::json document; + std::ifstream(filename) >> document; + document["materials"][0].erase("name"); + document["materials"][1]["name"] = "same"; + document["materials"][2]["name"] = "same"; + const auto primitive = document["meshes"][0]["primitives"][0]; + for (size_t i = 1; i < 3; ++i) + { + auto next = primitive; + next["material"] = i; + document["meshes"][0]["primitives"].push_back(next); + } + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto outputs = Callbacks.Outputs; + EXPECT_EQ(outputs[1].Path, "/materials/triangle/material_0.zematerial"); + EXPECT_EQ(outputs[2].Path, "/materials/triangle/same_1.zematerial"); + EXPECT_EQ(outputs[3].Path, "/materials/triangle/same_2.zematerial"); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.SubMeshes.size(), 3u); + for (size_t m = 0; m < 3; ++m) + { + const auto cooked = nlohmann::json::parse(ReadFile(outputs[m + 1].Path.c_str())); + EXPECT_EQ(cooked["name"], m == 0 ? "material" : "same"); + EXPECT_EQ(cooked["uuid"], uuids::to_string(mesh.SubMeshes[m].MaterialUUID)); + MetaFileData meta; + meta.AssetUUID = mesh.SubMeshes[m].MaterialUUID; + ASSERT_TRUE(MetaFileIO::Write(Context, Path(outputs[m + 1].Path.c_str()), meta).Succeeded()); + } + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + for (size_t m = 0; m < 3; ++m) + { + EXPECT_EQ(Callbacks.Outputs[m + 1].Path, outputs[m + 1].Path); + EXPECT_EQ(nlohmann::json::parse(ReadFile(outputs[m + 1].Path.c_str()))["uuid"], uuids::to_string(mesh.SubMeshes[m].MaterialUUID)); + } +} + +TEST_F(ModelImporterOutputTest, DifferentModelsCannotOverwriteEachOthersMaterials) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(1); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto original_path = Callbacks.Outputs[1].Path; + const auto original_bytes = ReadFile(original_path.c_str()); + Config.AssetName.init(&Arena, "other_model"); + Config.OutputAssetFile.init(&Arena, "other_model.zemesh"); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + EXPECT_NE(Callbacks.Outputs[1].Path, original_path); + EXPECT_EQ(ReadFile(original_path.c_str()), original_bytes); +} + +TEST_F(ModelImporterOutputTest, InvalidImageBufferRangesAreRejectedInEditorAndRuntime) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + nlohmann::json original; + std::ifstream(filename) >> original; + for (size_t variant = 0; variant < 5; ++variant) + { + auto document = original; + document["bufferViews"].push_back({ + { "buffer", 0}, + {"byteLength", 36} + }); + document["images"][0] = { + { "name", "bad"}, + {"bufferView", 1}, + { "mimeType", "image/png"} + }; + if (variant == 0) + document["bufferViews"][1]["byteOffset"] = 1073741824; + else if (variant == 1) + document["bufferViews"][1]["byteLength"] = 37; + else if (variant == 2) + document["bufferViews"][1]["buffer"] = 1; + else if (variant == 3) + document["images"][0]["bufferView"] = 2; + else + document["textures"][0]["source"] = 1; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Callbacks.Failed, static_cast(variant + 1)); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + MetaFileData meta; + EXPECT_TRUE(Importer.Import(Context, VFSPath::FromNative(filename.c_str()).Value(), meta).Failed()); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + } + std::ofstream(filename) << original.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, InvalidTriangleCountsAndAccessorRangesReportErrorsAndAllowRetry) +{ + const auto filename = WriteGltf(); + nlohmann::json original; + std::ifstream(filename) >> original; + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0}; + for (size_t variant = 0; variant < 5; ++variant) + { + auto document = original; + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + } + document["buffers"][0]["byteLength"] = sizeof(vertices); + document["bufferViews"][0]["byteLength"] = sizeof(vertices); + if (variant == 0) + document["accessors"][0]["count"] = 4; + else if (variant == 1) + document["accessors"][0]["byteOffset"] = 16; + else if (variant == 2) + document["bufferViews"][0]["byteStride"] = 0; + else if (variant == 3) + document["meshes"][0]["primitives"][0]["attributes"]["POSITION"] = 1; + else + { + document["accessors"].push_back(document["accessors"][0]); + document["accessors"][1]["count"] = 4; + document["meshes"][0]["primitives"][0]["attributes"]["NORMAL"] = 1; + } + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, static_cast(variant + 1)); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + } + std::ofstream(filename) << original.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, IndexedTrianglesRejectInvalidCountsAndVertexReferences) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document["buffers"][0]["byteLength"] = 52; + document["bufferViews"].push_back({ + { "buffer", 0}, + {"byteOffset", 36}, + {"byteLength", 16} + }); + document["accessors"].push_back({ + { "bufferView", 1}, + {"componentType", 5125}, + { "count", 4}, + { "type", "SCALAR"} + }); + document["meshes"][0]["primitives"][0]["indices"] = 1; + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + for (size_t variant = 0; variant < 3; ++variant) + { + const std::array indices = {0, 1, variant == 1 ? 3u : 2u, 0}; + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + binary.write(reinterpret_cast(indices.data()), sizeof(indices)); + } + document["accessors"][1]["count"] = variant == 0 ? 4 : 3; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, variant == 2 ? 1 : 0) << Callbacks.Error; + EXPECT_EQ(Callbacks.Failed, variant == 0 ? 1 : 2); + } +} + +TEST_F(ModelImporterOutputTest, EmbeddedImageAtExactBufferBoundaryImportsSuccessfully) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto image = ReadFile("/textures/triangle/one_pixel_0.png"); + const auto cooked = nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str())); + nlohmann::json document; + std::ifstream(filename) >> document; + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + binary.write(reinterpret_cast(image.data()), image.size()); + } + document["buffers"][0]["byteLength"] = sizeof(vertices) + image.size(); + document["bufferViews"].push_back({ + { "buffer", 0}, + {"byteOffset", sizeof(vertices)}, + {"byteLength", image.size()} + }); + document["images"][0] = { + { "name", "one_pixel"}, + {"bufferView", 1}, + { "mimeType", "image/png"} + }; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + EXPECT_EQ(Callbacks.Failed, 0); + EXPECT_EQ(ReadFile("/textures/triangle/one_pixel_0.png"), image); + EXPECT_EQ(nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str()))["textures"], cooked["textures"]); +} + +TEST_F(ModelImporterOutputTest, MetadataWritersUseIndependentTemporaryFilesEvenWhenOneFails) +{ + struct InterleavedMetadataContext : VFSContext + { + bool InsideOtherWriter = false; + bool OtherSucceeded = false; + VFSPath AssetPath; + VFSPath FirstTemporary; + VFSPath OtherTemporary; + MetaFileData OtherData{}; + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + auto result = VFSContext::Open(path, flags); + if (result.Succeeded() && HasFlag(flags, VFSOpenFlags::Write)) + { + if (InsideOtherWriter) + OtherTemporary = path; + else + { + FirstTemporary = path; + InsideOtherWriter = true; + OtherSucceeded = MetaFileIO::Write(*this, AssetPath, OtherData).Succeeded(); + InsideOtherWriter = false; + } + } + return result; + } + VFSResult Rename(const VFSPath& src, const VFSPath& dst) override + { + return InsideOtherWriter ? VFSContext::Rename(src, dst) : VFSResult::Fail(VFSError::IOError); + } + } writers; + writers.Initialize(&Arena, 4); + ASSERT_TRUE(writers.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + ASSERT_TRUE(writers.CreateDir(Path("/textures")).Succeeded()); + writers.AssetPath = Path("/textures/shared.png"); + writers.OtherData.AssetUUID = uuids::uuid::from_string("22222222-2222-4222-8222-222222222222").value(); + MetaFileData first; + first.AssetUUID = uuids::uuid::from_string("11111111-1111-4111-8111-111111111111").value(); + EXPECT_TRUE(MetaFileIO::Write(writers, writers.AssetPath, first).Failed()); + ASSERT_TRUE(writers.OtherSucceeded); + EXPECT_NE(writers.FirstTemporary, writers.OtherTemporary); + auto published = MetaFileIO::Read(writers, writers.AssetPath); + ASSERT_TRUE(published.Succeeded()); + EXPECT_EQ(published.Value().AssetUUID, writers.OtherData.AssetUUID); + ExpectNoTemporaryFiles("/textures"); + writers.Shutdown(); +} + +TEST_F(ModelImporterOutputTest, ConcurrentMetadataCreationKeepsOneIdentity) +{ + ASSERT_TRUE(Context.CreateDir(Path("/textures")).Succeeded()); + auto create = [&] { return MetaFileIO::GetOrCreate(Context, Path("/textures/shared.png"), "Test", 1); }; + auto first = std::async(std::launch::async, create); + auto second = std::async(std::launch::async, create); + auto one = first.get(); + auto two = second.get(); + ASSERT_TRUE(one.Succeeded()); + ASSERT_TRUE(two.Succeeded()); + EXPECT_EQ(one.Value().AssetUUID, two.Value().AssetUUID); + EXPECT_FALSE(one.Value().AssetUUID.is_nil()); +} + +class ModelImporterMetadataFailureTest : public ModelImporterOutputTest, public ::testing::WithParamInterface +{ +}; + +TEST_P(ModelImporterMetadataFailureTest, TextureMetadataFailureStopsImportAndAllowsRetry) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + Context.Point = GetParam(); + Context.FailurePath = "/textures/"; + Context.FailureWriteNumber = 2; // Image publication succeeds; its metadata fails. + const auto filename = WriteTexturedGltf(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_FALSE(Exists("/textures/triangle/one_pixel_0.png.meta")); + ExpectNoTemporaryFiles("/textures/triangle"); + Context.Point = FailurePoint::None; + Context.FailureWriteNumber = 0; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + EXPECT_TRUE(Exists("/textures/triangle/one_pixel_0.png.meta")); +} + +TEST_P(ModelImporterMetadataFailureTest, FailedMetadataUpdatePreservesPreviousIdentityAndHash) +{ + ASSERT_TRUE(Context.CreateDir(Path("/textures")).Succeeded()); + const auto path = Path("/textures/shared.png"); + auto initial = MetaFileIO::GetOrCreate(Context, path, "Test", 1); + ASSERT_TRUE(initial.Succeeded()); + Context.WriteNumber = 0; + Context.FailureWriteNumber = 1; + Context.Point = GetParam(); + Context.FailurePath = "/textures/"; + EXPECT_TRUE(MetaFileIO::GetOrCreate(Context, path, "Test", 2).Failed()); + auto old = MetaFileIO::Read(Context, path); + ASSERT_TRUE(old.Succeeded()); + EXPECT_EQ(old.Value().SourceHash, 1u); + EXPECT_EQ(old.Value().AssetUUID, initial.Value().AssetUUID); + ExpectNoTemporaryFiles("/textures"); +} + +INSTANTIATE_TEST_SUITE_P(VFSStages, ModelImporterMetadataFailureTest, ::testing::Values(FailurePoint::Open, FailurePoint::Write, FailurePoint::ShortWrite, FailurePoint::Flush, FailurePoint::Close, FailurePoint::Rename)); + +class ModelImporterWriteFailureTest : public ModelImporterOutputTest, public ::testing::WithParamInterface +{ +}; + +TEST_P(ModelImporterWriteFailureTest, MeshWriteFailureStopsImportAndAllowsRetry) +{ + const auto filename = WriteGltf(); + Context.Point = GetParam(); + ImportGltf(filename); + + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_TRUE(Callbacks.Outputs.empty()); + EXPECT_LT(Callbacks.Progress, 1.0f); + EXPECT_NE(Callbacks.Error.find("/meshes/triangle.zemesh"), std::string::npos); + EXPECT_NE(Callbacks.Error.find("VFS error"), std::string::npos); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + ExpectNoTemporaryFiles("/meshes"); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + + Context.Point = FailurePoint::None; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_TRUE(Exists("/meshes/triangle.zemesh")); +} + +TEST_P(ModelImporterWriteFailureTest, FailedRecookPreservesExistingArtifact) +{ + const auto filename = WriteGltf(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1); + const auto before = ReadFile("/meshes/triangle.zemesh"); + ASSERT_FALSE(before.empty()); + + Context.Point = GetParam(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(before, ReadFile("/meshes/triangle.zemesh")); + ExpectNoTemporaryFiles("/meshes"); +} + +TEST_P(ModelImporterWriteFailureTest, TextureFailureStopsImportAndPreservesExistingMeshAndTexture) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto before_mesh = ReadFile("/meshes/triangle.zemesh"); + const auto before_texture = ReadFile("/textures/triangle/one_pixel_0.png"); + ChangeTriangleVertices(); + Context.Point = GetParam(); + Context.FailurePath = "/textures/"; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_LT(Callbacks.Progress, 1.0f); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before_mesh); + EXPECT_EQ(ReadFile("/textures/triangle/one_pixel_0.png"), before_texture); + ExpectNoTemporaryFiles("/textures/triangle"); +} + +INSTANTIATE_TEST_SUITE_P(VFSStages, ModelImporterWriteFailureTest, ::testing::Values(FailurePoint::CreateDir, FailurePoint::Open, FailurePoint::Write, FailurePoint::ShortWrite, FailurePoint::Flush, FailurePoint::Close, FailurePoint::Rename)); diff --git a/ZEngine/tests/Rendering/SwapchainExtent_test.cpp b/ZEngine/tests/Rendering/SwapchainExtent_test.cpp new file mode 100644 index 00000000..ede4f07c --- /dev/null +++ b/ZEngine/tests/Rendering/SwapchainExtent_test.cpp @@ -0,0 +1,226 @@ +#include +#include +#include +#include +#include +#include + +using namespace ZEngine::Hardwares; + +namespace +{ + VkSurfaceCapabilitiesKHR VariableExtentCapabilities() + { + VkSurfaceCapabilitiesKHR caps{}; + caps.currentExtent = {std::numeric_limits::max(), std::numeric_limits::max()}; + caps.minImageExtent = {64, 64}; + caps.maxImageExtent = {4096, 4096}; + return caps; + } +} // namespace + +TEST(SwapchainExtentTest, WaylandUsesPhysicalFramebufferWithinSurfaceLimits) +{ + const auto caps = VariableExtentCapabilities(); + auto extent = DeviceSwapchain::ResolveExtent(caps, {2478, 1514}); + EXPECT_EQ(extent.width, 2478u); + EXPECT_EQ(extent.height, 1514u); + + extent = DeviceSwapchain::ResolveExtent(caps, {32, 8192}); + EXPECT_EQ(extent.width, 64u); + EXPECT_EQ(extent.height, 4096u); +} + +TEST(SwapchainExtentTest, FixedSurfaceExtentRemainsAuthoritative) +{ + auto caps = VariableExtentCapabilities(); + caps.currentExtent = {1920, 1080}; + const auto extent = DeviceSwapchain::ResolveExtent(caps, {2478, 1514}); + EXPECT_EQ(extent.width, 1920u); + EXPECT_EQ(extent.height, 1080u); +} + +TEST(SwapchainExtentTest, ZeroSizeIsDeferredRatherThanClampedToMinimum) +{ + auto caps = VariableExtentCapabilities(); + for (const VkExtent2D framebuffer : { + VkExtent2D{ 0, 0}, + VkExtent2D{ 0, 1514}, + VkExtent2D{2478, 0} + }) + { + const auto extent = DeviceSwapchain::ResolveExtent(caps, framebuffer); + EXPECT_EQ(extent.width, 0u); + EXPECT_EQ(extent.height, 0u); + } + caps.currentExtent = {0, 0}; + const auto extent = DeviceSwapchain::ResolveExtent(caps, {2478, 1514}); + EXPECT_EQ(extent.width, 0u); + EXPECT_EQ(extent.height, 0u); +} + +TEST(SwapchainExtentTest, ResizeSchedulesRecreationWithoutVulkanError) +{ + DeviceSwapchain swapchain{}; + swapchain.FramebufferExtent = {1500, 800}; + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + EXPECT_EQ(swapchain.FramebufferExtent.width, 2478u); + EXPECT_EQ(swapchain.FramebufferExtent.height, 1514u); + + swapchain.Recreation = RecreationState::None; + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::None); + + swapchain.UpdateFramebufferExtent(2480, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + swapchain.Recreation = RecreationState::None; + swapchain.UpdateFramebufferExtent(2480, 1516); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + + // A DPI-only framebuffer change still requests recreation. + swapchain.UpdateFramebufferExtent(3098, 1893); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); +} + +TEST(SwapchainExtentTest, ZeroSizeSkipsAcquisitionAndRestoreSchedulesRecreation) +{ + DeviceSwapchain swapchain{}; + swapchain.FramebufferExtent = {2478, 1514}; + swapchain.UpdateFramebufferExtent(0, 0); + // No device or GPU objects are needed: zero size must not touch Vulkan/fences. + swapchain.AcquireNextImage(0); + EXPECT_FALSE(swapchain.IsFrameValid()); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + EXPECT_EQ(swapchain.FramebufferExtent.width, 2478u); + EXPECT_EQ(swapchain.FramebufferExtent.height, 1514u); +} + +TEST(SwapchainExtentTest, UnchangedExtentDoesNotClearAnAbortedAcquire) +{ + DeviceSwapchain swapchain{}; + swapchain.FramebufferExtent = {2478, 1514}; + swapchain.Recreation = RecreationState::FrameAborted; + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::FrameAborted); +} + +TEST(SwapchainExtentTest, InvalidFrameSkipsRecordingAndPresentation) +{ + // Device-owned queues exceed the default 1 MB Windows stack. + auto device = std::make_unique(); + CommandBufferManager command_buffers{}; + DeviceSwapchain swapchain{}; + ZEngine::Applications::AppRenderPipeline pipeline{}; + device->CommandBufferMgr = &command_buffers; + device->SwapchainPtr = &swapchain; + swapchain.Device = device.get(); + pipeline.Device = device.get(); + + // A zero-size framebuffer must not dereference GPU resources, including + // command pools, the render timeline, or CurrentCmdBuf. + EXPECT_FALSE(pipeline.BeginFrame()); + command_buffers.EnqueuedCommandBufferIndex = 1; + pipeline.EndFrame(); + EXPECT_EQ(command_buffers.EnqueuedCommandBufferIndex, 0u); + EXPECT_EQ(pipeline.CurrentCmdBuf, nullptr); + + // A concrete surface can also report zero size during recreation. In that + // case CurrentFrame exists, but ImageIndex is invalid and state is Pending. + FrameContext frame{}; + swapchain.CurrentFrame = &frame; + swapchain.Recreation = RecreationState::Pending; + command_buffers.EnqueuedCommandBufferIndex = 1; + pipeline.EndFrame(); + EXPECT_EQ(command_buffers.EnqueuedCommandBufferIndex, 0u); +} + +TEST(SwapchainExtentTest, AcquisitionWaitIsBounded) +{ + EXPECT_GT(DeviceSwapchain::ImageAcquireTimeoutNs, 0u); + EXPECT_LE(DeviceSwapchain::ImageAcquireTimeoutNs, 100'000'000u); +} + +TEST(SwapchainExtentTest, TimeoutAndNotReadyInvalidateTheOldImageWithoutRecreation) +{ + for (VkResult result : {VK_TIMEOUT, VK_NOT_READY}) + { + DeviceSwapchain swapchain{}; + FrameContext frame{}; + frame.ImageIndex = 2; + swapchain.RenderTimelineNextValue = 17; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, result, 0)); + EXPECT_FALSE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, std::numeric_limits::max()); + EXPECT_EQ(swapchain.Recreation, RecreationState::None); + EXPECT_EQ(swapchain.RenderTimelineNextValue, 17u); + + EXPECT_TRUE(swapchain.ApplyAcquireResult(frame, VK_SUCCESS, 1)); + EXPECT_TRUE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, 1u); + EXPECT_EQ(swapchain.Recreation, RecreationState::None); + } +} + +TEST(SwapchainExtentTest, SuboptimalAcquisitionHasAnImageAndSchedulesRecreation) +{ + DeviceSwapchain swapchain{}; + FrameContext frame{}; + EXPECT_TRUE(swapchain.ApplyAcquireResult(frame, VK_SUBOPTIMAL_KHR, 2)); + EXPECT_TRUE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, 2u); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); +} + +TEST(SwapchainExtentTest, FailedAcquisitionNeverExposesAnImage) +{ + for (VkResult result : {VK_ERROR_OUT_OF_DATE_KHR, VK_ERROR_DEVICE_LOST, VK_ERROR_SURFACE_LOST_KHR, VK_ERROR_OUT_OF_HOST_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) + { + DeviceSwapchain swapchain{}; + FrameContext frame{}; + frame.ImageIndex = 2; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, result, 0)); + EXPECT_FALSE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, std::numeric_limits::max()); + EXPECT_EQ(swapchain.Recreation, RecreationState::FrameAborted); + } +} + +TEST(SwapchainExtentTest, SuccessWithAnInvalidImageIndexIsRejected) +{ + DeviceSwapchain swapchain{}; + FrameContext frame{}; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, VK_SUCCESS, swapchain.SwapchainImageCount)); + EXPECT_FALSE(swapchain.IsFrameValid()); +} + +TEST(SwapchainExtentTest, TimeoutSkipsGpuSubmissionAndCancelsPendingCallbacks) +{ + ZEngine::Core::Memory::MemoryManager memory{}; + memory.Initialize(ZMega(4ULL), {}); + auto device = std::make_unique(); + CommandBufferManager command_buffers{}; + DeviceSwapchain swapchain{}; + ZEngine::Applications::AppRenderPipeline pipeline{}; + device->CommandBufferMgr = &command_buffers; + device->SwapchainPtr = &swapchain; + swapchain.Device = device.get(); + pipeline.Device = device.get(); + swapchain.RenderWorkSubmittedCallbacks.init(&memory.MainArena, 1); + uint32_t cancelled = 0; + swapchain.EnqueueRenderWorkSubmittedCallback([](void*, ZEngine::Rendering::Primitives::Semaphore*, uint64_t) { ADD_FAILURE() << "A timeout must not submit GPU work"; }, &cancelled, [](void* context) { ++*static_cast(context); }); + + FrameContext frame{}; + frame.ImageIndex = 2; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, VK_TIMEOUT, 0)); + command_buffers.EnqueuedCommandBufferIndex = 1; + pipeline.EndFrame(); // No command buffer, semaphore, fence or GPU is needed. + EXPECT_EQ(command_buffers.EnqueuedCommandBufferIndex, 0u); + EXPECT_EQ(cancelled, 1u); + EXPECT_EQ(swapchain.RenderWorkSubmittedCallbacks.size(), 0u); + EXPECT_EQ(swapchain.RenderTimelineNextValue, 0u); + memory.Shutdown(); +} diff --git a/ZEngine/tests/Rendering/ZUIRenderScale_test.cpp b/ZEngine/tests/Rendering/ZUIRenderScale_test.cpp new file mode 100644 index 00000000..f3be4ec9 --- /dev/null +++ b/ZEngine/tests/Rendering/ZUIRenderScale_test.cpp @@ -0,0 +1,107 @@ +#include +#include + +using namespace ZEngine::Rendering::Renderers; +using namespace ZEngine::UI; + +namespace +{ + ZUIRenderPayload Payload(float width, float height) + { + ZUIRenderPayload payload{}; + payload.Scale[0] = 2.f / width; + payload.Scale[1] = 2.f / height; + payload.Translate[0] = payload.Translate[1] = -1.f; + return payload; + } + + ZUIDrawListCmd Clip(float x, float y, float width, float height) + { + ZUIDrawListCmd cmd{}; + cmd.ClipX = x; + cmd.ClipY = y; + cmd.ClipW = width; + cmd.ClipH = height; + return cmd; + } +} // namespace + +TEST(ZUIRenderScaleTest, ClipUsesActualSwapchainRatherThanWindowDpi) +{ + const auto payload = Payload(1982, 1211); + const auto cmd = Clip(280, 50, 100, 20); + // Regression: window framebuffer=2478x1514, but old swapchain=1500x800. + // The old 1.25 multiplier produced x=350; projected geometry starts at x=212. + const auto rect = payload.GetScissorRect(cmd, {1500, 800}); + EXPECT_EQ(rect.offset.x, 211); + EXPECT_EQ(rect.offset.y, 33); + EXPECT_EQ(rect.extent.width, 77u); + EXPECT_EQ(rect.extent.height, 14u); +} + +TEST(ZUIRenderScaleTest, FractionalDpiRoundsOutwardPerAxis) +{ + const auto rect = Payload(1000, 800).GetScissorRect(Clip(10.2f, 20.2f, 30.1f, 40.1f), {1250, 1001}); + EXPECT_EQ(rect.offset.x, 12); + EXPECT_EQ(rect.offset.y, 25); + EXPECT_EQ(rect.extent.width, 39u); + EXPECT_EQ(rect.extent.height, 51u); +} + +TEST(ZUIRenderScaleTest, NativeAndRetinaScissorsStayAligned) +{ + const auto payload = Payload(1500, 800); + const auto cmd = Clip(280, 50, 100, 20); + const auto native = payload.GetScissorRect(cmd, {1500, 800}); + EXPECT_EQ(native.offset.x, 280); + EXPECT_EQ(native.offset.y, 50); + EXPECT_EQ(native.extent.width, 100u); + EXPECT_EQ(native.extent.height, 20u); + + const auto retina = payload.GetScissorRect(cmd, {3000, 1600}); + EXPECT_EQ(retina.offset.x, 560); + EXPECT_EQ(retina.offset.y, 100); + EXPECT_EQ(retina.extent.width, 200u); + EXPECT_EQ(retina.extent.height, 40u); +} + +TEST(ZUIRenderScaleTest, FullScreenClipCoversRoundedFractionalFramebuffer) +{ + const auto rect = Payload(1982, 1211).GetScissorRect(Clip(0, 0, 1982, 1211), {2478, 1514}); + EXPECT_EQ(rect.offset.x, 0); + EXPECT_EQ(rect.offset.y, 0); + EXPECT_EQ(rect.extent.width, 2478u); + EXPECT_EQ(rect.extent.height, 1514u); +} + +TEST(ZUIRenderScaleTest, ClampsPartiallyAndFullyOffscreenClips) +{ + const auto payload = Payload(100, 100); + auto rect = payload.GetScissorRect(Clip(-10, -20, 50, 60), {200, 200}); + EXPECT_EQ(rect.offset.x, 0); + EXPECT_EQ(rect.offset.y, 0); + EXPECT_EQ(rect.extent.width, 80u); + EXPECT_EQ(rect.extent.height, 80u); + + rect = payload.GetScissorRect(Clip(90, 90, 50, 60), {200, 200}); + EXPECT_EQ(rect.offset.x, 180); + EXPECT_EQ(rect.offset.y, 180); + EXPECT_EQ(rect.extent.width, 20u); + EXPECT_EQ(rect.extent.height, 20u); + + rect = payload.GetScissorRect(Clip(110, 110, 10, 10), {200, 200}); + EXPECT_EQ(rect.extent.width, 0u); + EXPECT_EQ(rect.extent.height, 0u); + rect = payload.GetScissorRect(Clip(-50, -50, 10, 10), {200, 200}); + EXPECT_EQ(rect.extent.width, 0u); + EXPECT_EQ(rect.extent.height, 0u); +} + +TEST(ZUIRenderScaleTest, EmptyOrUninitializedPayloadDoesNotDraw) +{ + const auto payload = Payload(100, 100); + EXPECT_EQ(payload.GetScissorRect(Clip(0, 0, 0, 10), {200, 200}).extent.width, 0u); + EXPECT_EQ(payload.GetScissorRect(Clip(0, 0, 10, -1), {200, 200}).extent.height, 0u); + EXPECT_EQ(payload.GetScissorRect(Clip(0, 0, 10, 10), {0, 0}).extent.width, 0u); + EXPECT_EQ(ZUIRenderPayload{}.GetScissorRect(Clip(0, 0, 10, 10), {200, 200}).extent.width, 0u); +} diff --git a/ZEngine/tests/VFS/VFSFileWatcherTest.cpp b/ZEngine/tests/VFS/VFSFileWatcherTest.cpp index ca7c935c..07bf5907 100644 --- a/ZEngine/tests/VFS/VFSFileWatcherTest.cpp +++ b/ZEngine/tests/VFS/VFSFileWatcherTest.cpp @@ -248,6 +248,26 @@ TEST_F(VFSFileWatcherTest, RenamePreservesOldPath) EXPECT_EQ(received.Kind, WatchEventKind::Renamed); } +TEST_F(VFSFileWatcherTest, DebouncePreservesObservationTimestamp) +{ + MockPlatformWatcher mock; + VFSFileWatcher watcher(&mock, std::chrono::milliseconds{10}); + watcher.Initialize(&m_manager.MainArena); + + uint64_t observed_at = 0; + watcher.Watch("/project", true, [&](const VFSWatchEvent& event) { observed_at = event.ObservedAtNanoseconds; }); + + VFSWatchEvent event = MakeEvent("/project/imported.zemesh", WatchEventKind::Renamed); + event.ObservedAtNanoseconds = 123456789; + mock.InjectEvent(event); + + watcher.Tick(); + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + watcher.Tick(); + + EXPECT_EQ(observed_at, 123456789u); +} + TEST_F(VFSFileWatcherTest, MultipleWatchesRouteCorrectly) { MockPlatformWatcher mock; diff --git a/ZEngine/tests/VFS/vfs_asset_identity_test.cpp b/ZEngine/tests/VFS/vfs_asset_identity_test.cpp new file mode 100644 index 00000000..06c3ed9f --- /dev/null +++ b/ZEngine/tests/VFS/vfs_asset_identity_test.cpp @@ -0,0 +1,398 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ZEngine::Core::Memory; +using namespace ZEngine::Core::VFS; + +namespace +{ + struct FaultFile : IVFSFile + { + IVFSFile* File = nullptr; + uint64_t ReadLimit = UINT64_MAX; + size_t MaxChunk = SIZE_MAX; + bool ReadIOError = false; + bool SizeIOError = false; + VFSResult Read(ZEngine::Core::Containers::ArrayView buffer, uint64_t offset) override + { + if (offset >= ReadLimit) + return ReadIOError ? VFSResult::Fail(VFSError::IOError) : VFSResult::Ok(0); + const size_t count = std::min({buffer.size(), MaxChunk, static_cast(ReadLimit - offset)}); + return File->Read({buffer.data(), count}, offset); + } + VFSResult Write(ZEngine::Core::Containers::ArrayView buffer, uint64_t offset) override + { + return File->Write(buffer, offset); + } + VFSResult Size() const override + { + return SizeIOError ? VFSResult::Fail(VFSError::IOError) : File->Size(); + } + VFSResult Stat() const override + { + return File->Stat(); + } + VFSResult Flush() override + { + return File->Flush(); + } + const VFSPath& Path() const override + { + return File->Path(); + } + VFSResult Close() override + { + return File->Close(); + } + }; + + // Faults apply to scanner operations; direct backend access remains available + // to verify that failed scans did not alter the original sidecar. + struct FaultContext : VFSContext + { + bool DenyMetaOnce = false; + bool FailAssetRead = false; + bool FailWrite = false; + bool FailRename = false; + bool FailList = false; + int Writes = 0; + FaultFile AssetFile; + bool WrapAssetReads = false; + bool WrapMetadataReads = false; + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + if (HasFlag(flags, VFSOpenFlags::Read)) + { + if (path.Extension().Equals(".meta") && DenyMetaOnce) + { + DenyMetaOnce = false; + return VFSResult::Fail(VFSError::PermissionDenied); + } + if (!path.Extension().Equals(".meta") && FailAssetRead) + return VFSResult::Fail(VFSError::IOError); + } + if (HasFlag(flags, VFSOpenFlags::Write)) + { + ++Writes; + if (FailWrite) + return VFSResult::Fail(VFSError::PermissionDenied); + } + auto result = VFSContext::Open(path, flags); + if (result.Succeeded() && HasFlag(flags, VFSOpenFlags::Read) && ((WrapAssetReads && !path.Extension().Equals(".meta")) || (WrapMetadataReads && path.Extension().Equals(".meta")))) + { + AssetFile.File = result.Value(); + return VFSResult::Ok(&AssetFile); + } + return result; + } + void Close(IVFSFile* file) override + { + VFSContext::Close(file == &AssetFile ? AssetFile.File : file); + } + VFSResult Rename(const VFSPath& from, const VFSPath& to) override + { + return FailRename ? VFSResult::Fail(VFSError::IOError) : VFSContext::Rename(from, to); + } + VFSResult> List(const VFSPath& path, ArenaAllocator* arena) override + { + if (FailList) + return VFSResult>::Fail(VFSError::IOError); + return VFSContext::List(path, arena); + } + }; +} // namespace + +class VFSAssetIdentityTest : public ::testing::Test +{ +protected: + MemoryManager Memory; + VFSMemoryBackend Backend; + FaultContext Context; + const uuids::uuid ID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + void SetUp() override + { + Memory.Initialize(ZGiga(1), {}); + Backend.Initialize(&Memory.MainArena); + Context.Initialize(&Memory.MainArena, 4); + ASSERT_TRUE(Context.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + } + void TearDown() override + { + Context.Shutdown(); + } + VFSPath P(const char* path) + { + return VFSPath::Parse(path).Value(); + } + void Put(const char* path, const std::string& data) + { + ASSERT_TRUE(Backend.WriteFile(P(path), {reinterpret_cast(data.data()), data.size()}).Succeeded()); + } + std::string Material() + { + return "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"textures\":{\"albedo\":{\"path\":\"" + std::string(200, 'p') + "\"}}}"; + } + void Scan(VFSScanner& scanner, VFSDirectoryCache& cache, ScanStats& stats) + { + std::atomic complete{false}; + struct Completion + { + ScanStats* Stats; + std::atomic* Complete; + } completion{&stats, &complete}; + scanner.SetOnScanComplete(&completion, [](void* ctx, ScanStats value) { + auto* done = static_cast(ctx); + *done->Stats = value; + done->Complete->store(true); + }); + scanner.Scan(&Context, VFSPath::Root(), &cache); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!complete.load() && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + // Keep callback context alive until all tasks finish, even on a timeout. + if (!complete.load()) + { + scanner.Cancel(); + while (scanner.IsScanning()) + std::this_thread::yield(); + } + ASSERT_TRUE(complete.load()); + } +}; + +TEST_F(VFSAssetIdentityTest, Reads1000MaterialIdentitiesWithoutAnArena) +{ + Put("/material.zematerial", Material()); + for (int i = 0; i < 1000; ++i) + { + auto result = ReadEmbeddedAssetUUID(Context, P("/material.zematerial")); + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Value(), ID); + } +} + +TEST_F(VFSAssetIdentityTest, StreamsLargeMaterialAndUsesOnlyTopLevelIdentity) +{ + const std::string json = "{\"textures\":{\"albedo\":{\"uuid\":\"550e8400-e29b-41d4-a716-446655440001\",\"path\":\"" + std::string(20000, 'p') + "\"}},\"uuid\":\"" + uuids::to_string(ID) + "\"}"; + Put("/large.zematerial", json); + auto result = ReadEmbeddedAssetUUID(Context, P("/large.zematerial")); + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Value(), ID); +} + +TEST_F(VFSAssetIdentityTest, OversizedMaterialIdentityIsRejectedWithoutReplacingCorruptMetadata) +{ + Put("/large.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(MATERIAL_IDENTITY_MAX_FILE_SIZE, 'n') + "\"}"); + Put("/large.zematerial.meta", "{corrupt metadata"); + + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/large.zematerial")).Error(), VFSError::SizeLimitExceeded); + + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + Context.Writes = 0; + ScanStats stats; + Scan(scanner, cache, stats); + + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(Context.Writes, 0); + auto sidecar = Context.Open(P("/large.zematerial.meta"), VFSOpenFlags::Read); + ASSERT_TRUE(sidecar.Succeeded()); + std::array contents{}; + auto read = sidecar.Value()->ReadAll({contents.data(), contents.size()}); + Context.Close(sidecar.Value()); + ASSERT_TRUE(read.Succeeded()); + EXPECT_STREQ(reinterpret_cast(contents.data()), "{corrupt metadata"); +} + +TEST_F(VFSAssetIdentityTest, RejectsInvalidMaterialsAndTruncatedMeshes) +{ + for (const char* json : {"{}", "[]", "{\"uuid\":null}", "{\"uuid\":\"invalid\"}", "{\"uuid\":\"00000000-0000-0000-0000-000000000000\"}", "{\"nested\":{\"uuid\":\"550e8400-e29b-41d4-a716-446655440000\"}}", "{\"uuid\":\"550e8400-e29b-41d4-a716-446655440000\"}garbage"}) + { + Put("/bad.zematerial", json); + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/bad.zematerial")).Error(), VFSError::Corrupted); + } + Put("/bad.zemesh", std::string(23, '\0')); + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/bad.zemesh")).Error(), VFSError::Corrupted); + Context.FailAssetRead = true; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/bad.zemesh")).Error(), VFSError::IOError); +} + +TEST_F(VFSAssetIdentityTest, ScannerPreservesSidecarOnMetadataHashAndPublicationFailures) +{ + Put("/material.zematerial", Material()); + MetaFileData data{}; + data.AssetUUID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440001").value(); + data.SourceHash = 123; + data.SettingsCount = 1; + std::strcpy(data.Settings[0].Key, "keep"); + std::strcpy(data.Settings[0].Value, "settings"); + ASSERT_TRUE(MetaFileIO::Write(Context, P("/material.zematerial"), data).Succeeded()); + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + for (int fault = 0; fault < 4; ++fault) + { + SCOPED_TRACE(fault); + Context.DenyMetaOnce = fault == 0; + Context.FailAssetRead = fault == 1; + Context.FailWrite = fault == 2; + Context.FailRename = fault == 3; + Context.Writes = 0; + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(stats.MetasCreated + stats.MetasUpdated + stats.MetasUpToDate, 0u); + if (fault < 2) + EXPECT_EQ(Context.Writes, 0); + Context.FailAssetRead = Context.FailWrite = Context.FailRename = false; + auto after = MetaFileIO::Read(Context, P("/material.zematerial")); + ASSERT_TRUE(after.Succeeded()); + EXPECT_EQ(after.Value().AssetUUID, data.AssetUUID); + EXPECT_EQ(after.Value().SourceHash, data.SourceHash); + EXPECT_EQ(after.Value().SettingsCount, 1u); + EXPECT_STREQ(after.Value().Settings[0].Value, "settings"); + } +} + +TEST_F(VFSAssetIdentityTest, ScannerReportsDirectoryErrorsAndDoesNotCreateIdentityForInvalidCookedAsset) +{ + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + Context.FailList = true; + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + Context.FailList = false; + Put("/bad.zematerial", "{}"); + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_FALSE(Context.Exists(P("/bad.zematerial.meta")).Value()); +} + +TEST_F(VFSAssetIdentityTest, ScannerSeedsLargeMaterialIdentityAndPreservesOversizedSidecar) +{ + Put("/large.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(20000, 'n') + "\"}"); + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 0u); + EXPECT_EQ(stats.MetasCreated, 1u); + auto meta = MetaFileIO::Read(Context, P("/large.zematerial")); + ASSERT_TRUE(meta.Succeeded()); + EXPECT_EQ(meta.Value().AssetUUID, ID); + Put("/large.zematerial.meta", std::string(MetaFileIO::MaxFileSize + 1, ' ')); + Context.Writes = 0; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(Context.Writes, 0); + EXPECT_EQ(MetaFileIO::Read(Context, P("/large.zematerial")).Error(), VFSError::SizeLimitExceeded); +} + +TEST_F(VFSAssetIdentityTest, IdentityReadsPropagateIOErrorsAndDetectPrematureEOF) +{ + Put("/material.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(5000, 'n') + "\"}"); + Context.WrapAssetReads = true; + Context.AssetFile.ReadLimit = 4096; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/material.zematerial")).Error(), VFSError::IOError); + Context.AssetFile.ReadIOError = true; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/material.zematerial")).Error(), VFSError::IOError); + Context.AssetFile.ReadLimit = UINT64_MAX; + Context.AssetFile.SizeIOError = true; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/material.zematerial")).Error(), VFSError::IOError); + Context.AssetFile.SizeIOError = false; + Context.AssetFile.MaxChunk = 7; + auto material = ReadEmbeddedAssetUUID(Context, P("/material.zematerial")); + ASSERT_TRUE(material.Succeeded()); + EXPECT_EQ(material.Value(), ID); + std::string mesh(24, '\0'); + uint32_t magic = ZEMESH_MAGIC, version = ASSET_FILE_VERSION; + std::memcpy(mesh.data(), &magic, 4); + std::memcpy(mesh.data() + 4, &version, 4); + std::memcpy(mesh.data() + 8, &ID, 16); + Put("/mesh.zemesh", mesh); + auto header = ReadEmbeddedAssetUUID(Context, P("/mesh.zemesh")); + ASSERT_TRUE(header.Succeeded()); + EXPECT_EQ(header.Value(), ID); + Context.AssetFile.ReadLimit = 12; + Context.AssetFile.ReadIOError = false; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/mesh.zemesh")).Error(), VFSError::Corrupted); +} + +TEST_F(VFSAssetIdentityTest, ScannerDoesNotPersistHashFromPrematureEOFOrFailedSizeQuery) +{ + Put("/material.zematerial", Material()); + MetaFileData before{}; + before.AssetUUID = ID; + before.SourceHash = 123; + ASSERT_TRUE(MetaFileIO::Write(Context, P("/material.zematerial"), before).Succeeded()); + Context.Writes = 0; + Context.WrapAssetReads = true; + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + for (bool size_failure : {false, true}) + { + Context.AssetFile.ReadLimit = size_failure ? UINT64_MAX : 12; + Context.AssetFile.SizeIOError = size_failure; + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(Context.Writes, 0); + auto retained = MetaFileIO::Read(Context, P("/material.zematerial")); + ASSERT_TRUE(retained.Succeeded()); + EXPECT_EQ(retained.Value().SourceHash, before.SourceHash); + EXPECT_EQ(retained.Value().AssetUUID, before.AssetUUID); + } +} + +TEST_F(VFSAssetIdentityTest, ShortMetadataReadsAreIOErrorsAndNeverTriggerRepair) +{ + MetaFileData before{}; + before.AssetUUID = ID; + before.SourceHash = 123; + ASSERT_TRUE(MetaFileIO::Write(Context, P("/material.zematerial"), before).Succeeded()); + Context.Writes = 0; + Context.WrapMetadataReads = true; + Context.AssetFile.ReadLimit = 32; + EXPECT_EQ(MetaFileIO::Read(Context, P("/material.zematerial")).Error(), VFSError::IOError); + EXPECT_EQ(MetaFileIO::GetOrCreate(Context, P("/material.zematerial"), "Test", 456, true).Error(), VFSError::IOError); + EXPECT_EQ(Context.Writes, 0); + Context.WrapMetadataReads = false; + auto retained = MetaFileIO::Read(Context, P("/material.zematerial")); + ASSERT_TRUE(retained.Succeeded()); + EXPECT_EQ(retained.Value().AssetUUID, before.AssetUUID); + EXPECT_EQ(retained.Value().SourceHash, before.SourceHash); +} + +TEST_F(VFSAssetIdentityTest, HashIsStableAcrossPartialBackendReads) +{ + Put("/material.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(5000, 'n') + "\"}"); + auto expected = MetaFileIO::ComputeHash(Context, P("/material.zematerial")); + ASSERT_TRUE(expected.Succeeded()); + Context.WrapAssetReads = true; + Context.AssetFile.MaxChunk = 7; + auto fragmented = MetaFileIO::ComputeHash(Context, P("/material.zematerial")); + ASSERT_TRUE(fragmented.Succeeded()); + EXPECT_EQ(fragmented.Value(), expected.Value()); +} diff --git a/ZEngine/tests/VFS/vfs_disk_test.cpp b/ZEngine/tests/VFS/vfs_disk_test.cpp index e362094c..96df978a 100644 --- a/ZEngine/tests/VFS/vfs_disk_test.cpp +++ b/ZEngine/tests/VFS/vfs_disk_test.cpp @@ -80,6 +80,34 @@ TEST_F(VFSDiskBackendTest, OpenMissingFileFailsWithNotFound) EXPECT_EQ(r.Error(), VFSError::NotFound); } +TEST_F(VFSDiskBackendTest, DirectoryOpenFailureIsNotReportedAsMissing) +{ + ASSERT_TRUE(std::filesystem::create_directory(std::filesystem::path(m_root.c_str()) / "existing")); + auto opened = m_backend.Open(VFSPath::Parse("/existing").Value(), VFSOpenFlags::Write); + ASSERT_TRUE(opened.Failed()); + EXPECT_NE(opened.Error(), VFSError::NotFound); +} + +#if !defined(_WIN32) +TEST_F(VFSDiskBackendTest, UnreadableExistingFileReturnsPermissionDenied) +{ + WriteFile("/restricted.meta", "existing metadata"); + const auto path = std::filesystem::path(m_root.c_str()) / "restricted.meta"; + std::error_code error; + std::filesystem::permissions(path, std::filesystem::perms::none, error); + ASSERT_FALSE(error); + auto opened = m_backend.Open(VFSPath::Parse("/restricted.meta").Value(), VFSOpenFlags::Read); + std::filesystem::permissions(path, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write, error); + ASSERT_FALSE(error); + if (opened.Succeeded()) + { + m_backend.Close(opened.Value()); + GTEST_SKIP() << "This user can bypass filesystem permission checks"; + } + EXPECT_EQ(opened.Error(), VFSError::PermissionDenied); +} +#endif + TEST_F(VFSDiskBackendTest, WriteThenReadBack) { cstring contents = "hello vfs world"; diff --git a/ZEngine/tests/VFS/vfs_import_publication_test.cpp b/ZEngine/tests/VFS/vfs_import_publication_test.cpp new file mode 100644 index 00000000..e257f568 --- /dev/null +++ b/ZEngine/tests/VFS/vfs_import_publication_test.cpp @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +using ZEngine::Core::Memory::MemoryManager; + +namespace ZEngine::Core::VFS +{ + // Keeps watcher ingress private in production while allowing this focused + // regression test to model an inotify event delivered after publication end. + struct VFSContextTestAccess + { + static void Configure(VFSContext& context, const char* project_root, FileChangeListener listener, void* listener_context) + { + Helpers::secure_strcpy(context.m_project_root_native, sizeof(context.m_project_root_native), project_root); + context.m_file_change_listener = listener; + context.m_file_change_context = listener_context; + } + + static void Deliver(VFSContext& context, const VFSWatchEvent& event) + { + context.HandleWatchEvent(event); + } + }; +} // namespace ZEngine::Core::VFS + +namespace +{ + struct ListenerCounter + { + int Count = 0; + }; + + void CountEvent(void* context, const ZEngine::Core::VFS::VFSPath&, ZEngine::Core::VFS::WatchEventKind) + { + static_cast(context)->Count++; + } + + ZEngine::Core::VFS::VFSWatchEvent MakeEvent(const char* path) + { + ZEngine::Core::VFS::VFSWatchEvent event{}; + ZEngine::Helpers::secure_strcpy(event.Path, sizeof(event.Path), path); + event.Kind = ZEngine::Core::VFS::WatchEventKind::Modified; + event.ObservedAtNanoseconds = ZEngine::Core::VFS::VFSWatchTimestampNowNanoseconds(); + return event; + } +} // namespace + +TEST(VFSImportPublicationTest, SuppressesArtifactEventDeliveredAfterPublicationEnds) +{ + MemoryManager manager; + + manager.Initialize(ZMega(2), {}); + { + ZEngine::Core::VFS::VFSContext context; + ListenerCounter listener; + + context.Initialize(&manager.MainArena); + ZEngine::Core::VFS::VFSContextTestAccess::Configure(context, "/project", &CountEvent, &listener); + + const auto artifact = ZEngine::Core::VFS::VFSPath::Parse("/Assets/Meshes/robot.zemesh"); + ASSERT_TRUE(artifact.Succeeded()); + + const ZEngine::Core::VFS::VFSImportPublication publication = context.BeginImportPublication(); + context.RecordImportPublicationArtifact(publication, artifact.Value()); + context.EndImportPublication(publication); + + // The raw event is observed after EndImportPublication. It must still be + // absorbed by the publication grace period instead of notifying the import + // coordinator through the generic file-change listener. + ZEngine::Core::VFS::VFSContextTestAccess::Deliver(context, MakeEvent("/project/Assets/Meshes/robot.zemesh")); + EXPECT_EQ(listener.Count, 0); + + // The grace period only suppresses artifacts owned by this import. Other + // project edits retain the normal watcher behavior. + ZEngine::Core::VFS::VFSContextTestAccess::Deliver(context, MakeEvent("/project/Assets/Meshes/other.zemesh")); + EXPECT_EQ(listener.Count, 1); + } + + manager.Shutdown(); +} diff --git a/ZEngine/tests/VFS/vfs_meta_test.cpp b/ZEngine/tests/VFS/vfs_meta_test.cpp index a3db3f1c..28a680b3 100644 --- a/ZEngine/tests/VFS/vfs_meta_test.cpp +++ b/ZEngine/tests/VFS/vfs_meta_test.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -64,8 +65,9 @@ class MetaFileIOTest : public ::testing::Test TEST_F(MetaFileIOTest, MetaPathForAppendsMetaSuffix) { VFSPath asset = P("/project/textures/diffuse.png"); - VFSPath meta = MetaFileIO::MetaPathFor(asset); - EXPECT_STREQ(meta.CStr(), "/project/textures/diffuse.png.meta"); + auto meta = MetaFileIO::MetaPathFor(asset); + ASSERT_TRUE(meta.Succeeded()); + EXPECT_STREQ(meta.Value().CStr(), "/project/textures/diffuse.png.meta"); } // Test 2 — Read returns Fail when file does not exist @@ -84,6 +86,29 @@ TEST_F(MetaFileIOTest, MalformedJSONReturnsError) EXPECT_EQ(result.Error(), VFSError::Corrupted); } +TEST_F(MetaFileIOTest, InvalidIdentitiesAreCorruptAndGetOrCreateRepairsThem) +{ + for (const auto* document : {"{}", "{\"uuid\":42}", "{\"uuid\":\"invalid\"}", "{\"uuid\":\"00000000-0000-0000-0000-000000000000\"}"}) + { + SCOPED_TRACE(document); + WriteRaw("/x.glb.meta", document); + EXPECT_EQ(MetaFileIO::Read(m_ctx, P("/x.glb")).Error(), VFSError::Corrupted); + auto repaired = MetaFileIO::GetOrCreate(m_ctx, P("/x.glb"), "Test", 0); + ASSERT_TRUE(repaired.Succeeded()); + EXPECT_FALSE(repaired.Value().AssetUUID.is_nil()); + EXPECT_EQ(repaired.Value().Status, ImportStatus::New); + auto persisted = MetaFileIO::Read(m_ctx, P("/x.glb")); + ASSERT_TRUE(persisted.Succeeded()); + EXPECT_EQ(persisted.Value().AssetUUID, repaired.Value().AssetUUID); + } +} + +TEST_F(MetaFileIOTest, WritingNilIdentityIsRejected) +{ + EXPECT_EQ(MetaFileIO::Write(m_ctx, P("/x.glb"), {}).Error(), VFSError::Corrupted); + EXPECT_FALSE(FileExists("/x.glb.meta")); +} + // Test 4 — Round-trip: Write then Read returns identical data TEST_F(MetaFileIOTest, RoundTrip) { @@ -119,7 +144,7 @@ TEST_F(MetaFileIOTest, StatusNotSerialised) d.Status = ImportStatus::Stale; ASSERT_TRUE(MetaFileIO::Write(m_ctx, P("/x.glb"), d).Succeeded()); - auto open = m_ctx.Open(MetaFileIO::MetaPathFor(P("/x.glb")), VFSOpenFlags::Read); + auto open = m_ctx.Open(MetaFileIO::MetaPathFor(P("/x.glb")).Value(), VFSOpenFlags::Read); ASSERT_TRUE(open.Succeeded()); auto* file = open.Value(); uint8_t raw[4096] = {}; @@ -173,7 +198,25 @@ TEST_F(MetaFileIOTest, GetOrCreateChangedHashReturnsStale) EXPECT_EQ(second.Value().AssetUUID, original_uuid); } -// Test 9 — Truncated key/value strings do not overflow buffers +// Generated asset data is published by rename, so watchers never see a partial file. +TEST_F(MetaFileIOTest, AtomicWritePublishesDestinationWithoutTemporaryFile) +{ + constexpr char content[] = "texture-bytes"; + ASSERT_TRUE(WriteFileAtomically(m_ctx, P("/generated/diffuse.png"), Bytes(content, sizeof(content) - 1)).Succeeded()); + + EXPECT_TRUE(FileExists("/generated/diffuse.png")); + EXPECT_FALSE(FileExists("/generated/diffuse.png.tmp")); + + auto open = m_ctx.Open(P("/generated/diffuse.png"), VFSOpenFlags::Read); + ASSERT_TRUE(open.Succeeded()); + uint8_t bytes[sizeof(content)] = {}; + ASSERT_TRUE(open.Value()->ReadAll({bytes, sizeof(content) - 1}).Succeeded()); + m_ctx.Close(open.Value()); + + EXPECT_STREQ(reinterpret_cast(bytes), content); +} + +// Test 10 — Truncated key/value strings do not overflow buffers TEST_F(MetaFileIOTest, OverlongSettingsAreTruncated) { char long_key[201]; @@ -196,7 +239,7 @@ TEST_F(MetaFileIOTest, OverlongSettingsAreTruncated) EXPECT_EQ(result.Value().Settings[0].Key[63], '\0'); } -// Test 10 — Settings count is capped at META_MAX_SETTINGS +// Test 11 — Settings count is capped at META_MAX_SETTINGS TEST_F(MetaFileIOTest, SettingsCountCappedAtMax) { char json[8192]; @@ -221,3 +264,73 @@ TEST_F(MetaFileIOTest, SettingsCountCappedAtMax) ASSERT_TRUE(result.Succeeded()); EXPECT_EQ(result.Value().SettingsCount, META_MAX_SETTINGS); } + +TEST_F(MetaFileIOTest, SidecarPathBoundaryNeverOverwritesTheAsset) +{ + const std::string longest = "/" + std::string(MAX_FILE_PATH_COUNT - 2 - std::string_view(".meta").size(), 'x'); + auto sidecar = MetaFileIO::MetaPathFor(P(longest.c_str())); + ASSERT_TRUE(sidecar.Succeeded()); + EXPECT_EQ(sidecar.Value().Length(), MAX_FILE_PATH_COUNT - 1); + const auto invalid = longest + "x"; + WriteRaw(invalid.c_str(), "original asset bytes"); + MetaFileData data; + EXPECT_EQ(MetaFileIO::Write(m_ctx, P(invalid.c_str()), data).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::Read(m_ctx, P(invalid.c_str())).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::GetOrCreate(m_ctx, P(invalid.c_str()), "Test", 0).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::MetaPathFor(VFSPath{}).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::MetaPathFor(VFSPath::Root()).Error(), VFSError::InvalidPath); + auto file = m_ctx.Open(P(invalid.c_str()), VFSOpenFlags::Read); + ASSERT_TRUE(file.Succeeded()); + uint8_t bytes[32] = {}; + ASSERT_TRUE(file.Value()->ReadAll({bytes, sizeof(bytes) - 1}).Succeeded()); + m_ctx.Close(file.Value()); + EXPECT_STREQ(reinterpret_cast(bytes), "original asset bytes"); +} + +TEST_F(MetaFileIOTest, MaximumEscapedFieldsRoundTrip) +{ + MetaFileData data{}; + data.AssetUUID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + auto fill = [](auto& field) { std::memset(field, '\x01', sizeof(field) - 1); }; + fill(data.ImporterName); + fill(data.SourcePath); + fill(data.ArtifactPath); + data.SettingsCount = META_MAX_SETTINGS; + for (auto& setting : data.Settings) + { + fill(setting.Key); + fill(setting.Value); + } + ASSERT_TRUE(MetaFileIO::Write(m_ctx, P("/max.glb"), data).Succeeded()); + auto result = MetaFileIO::Read(m_ctx, P("/max.glb")); + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Value().AssetUUID, data.AssetUUID); + EXPECT_STREQ(result.Value().ImporterName, data.ImporterName); + EXPECT_STREQ(result.Value().SourcePath, data.SourcePath); + EXPECT_STREQ(result.Value().ArtifactPath, data.ArtifactPath); + EXPECT_EQ(result.Value().SettingsCount, META_MAX_SETTINGS); + for (uint32_t i = 0; i < META_MAX_SETTINGS; ++i) + { + EXPECT_STREQ(result.Value().Settings[i].Key, data.Settings[i].Key); + EXPECT_STREQ(result.Value().Settings[i].Value, data.Settings[i].Value); + } +} + +TEST_F(MetaFileIOTest, SizeBoundaryAcceptsLimitAndPreservesOversizedMetadata) +{ + std::string json = "{\"uuid\":\"550e8400-e29b-41d4-a716-446655440000\"}"; + json.resize(MetaFileIO::MaxFileSize, ' '); + WriteRaw("/large.glb.meta", json.c_str()); + ASSERT_TRUE(MetaFileIO::Read(m_ctx, P("/large.glb")).Succeeded()); + json.push_back(' '); + WriteRaw("/large.glb.meta", json.c_str()); + EXPECT_EQ(MetaFileIO::Read(m_ctx, P("/large.glb")).Error(), VFSError::SizeLimitExceeded); + EXPECT_EQ(MetaFileIO::GetOrCreate(m_ctx, P("/large.glb"), "Test", 123).Error(), VFSError::SizeLimitExceeded); + auto open = m_ctx.Open(P("/large.glb.meta"), VFSOpenFlags::Read); + ASSERT_TRUE(open.Succeeded()); + std::string retained(json.size(), '\0'); + auto read = open.Value()->ReadAll({reinterpret_cast(retained.data()), retained.size()}); + m_ctx.Close(open.Value()); + ASSERT_TRUE(read.Succeeded()); + EXPECT_EQ(retained, json); +} diff --git a/dependencies.cmake b/dependencies.cmake index 1f25e5a5..b916d248 100644 --- a/dependencies.cmake +++ b/dependencies.cmake @@ -1,5 +1,14 @@ include(FetchContent) +# GLFW 3.5 can include both native Linux backends and selects the active one +# at runtime. Clear the pre-3.4 cache option so an existing build directory +# can upgrade without GLFW rejecting its old GLFW_USE_WAYLAND entry. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + unset(GLFW_USE_WAYLAND CACHE) + set(GLFW_BUILD_X11 ON CACHE BOOL "Build GLFW X11 backend" FORCE) + set(GLFW_BUILD_WAYLAND ON CACHE BOOL "Build GLFW Wayland backend" FORCE) +endif() + FetchContent_Declare( fmt GIT_REPOSITORY https://github.com/fmtlib/fmt.git @@ -26,7 +35,7 @@ FetchContent_Declare( glfw3 GIT_REPOSITORY https://github.com/glfw/glfw.git GIT_SHALLOW TRUE - GIT_TAG 3.3.10 + GIT_TAG 3.5.1 ) FetchContent_Declare( diff --git a/docs/engine-architecture.md b/docs/engine-architecture.md index cdb72b58..8cc28f54 100644 --- a/docs/engine-architecture.md +++ b/docs/engine-architecture.md @@ -468,7 +468,7 @@ the render thread resets global buffer cursors to 0 so new geometry starts fresh ```mermaid graph TD ctx["VFSContext"] - m1["Mount /ZodiacEngine\n→ VFSDiskBackend\n→ cwd/ZodiacEngine/\npriority = -1"] + m1["Mount /ZodiacEngine\n→ VFSDiskBackend\n→ executable-dir/ZodiacEngine/\npriority = -1"] m2["Mount /\n→ VFSDiskBackend\n→ project root\npriority = 0"] path["VFSPath\nnormalized, immutable\nno-alloc value type"] scanner["VFSScanner\nasync directory walker\npopulates content browser cache"] @@ -529,6 +529,20 @@ sequenceDiagram Hard dependencies: Device before VFS (surface), RRM before fallback texture, watcher after working directory. +### Packaged engine assets + +Engine-owned files are resolved from `ZodiacEngine/` beside the executable, not +from the process working directory. This keeps a Debug or installed package +relocatable: it may be launched from the repository root, its output directory, +or any other directory. Startup validates the package root before mounting it +at `/ZodiacEngine`; all files under that directory then use the ordinary VFS +lookup path, including shaders. + +Development tools and package tests may explicitly set `ZENGINE_ASSET_ROOT` to +an alternate `ZodiacEngine` directory. The override takes precedence over the +executable-relative package location; normal application launches should not +need it. + --- ## Shutdown Order