From 358210b738322bcad8b1facea775e74154766ba5 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Fri, 25 Sep 2026 21:30:15 +0900 Subject: [PATCH 01/27] fix(editor): resolve packaged assets and refresh imports --- Obelisk/EntryPoint.cpp | 3 - Resources/Editor/Settings/FallbackTexture.png | Bin 0 -> 78 bytes Tetragrama/Panels/AssetImporterPanel.cpp | 120 +++++++++++++----- Tetragrama/Panels/AssetImporterPanel.h | 62 +++++---- Tetragrama/Panels/ProjectViewPanel.cpp | 6 + Tetragrama/Panels/ProjectViewPanel.h | 3 + .../Panels/ZUIPanelManagerComponent.cpp | 8 +- ZEngine/ZEngine/Core/EngineAssetRoot.cpp | 97 ++++++++++++++ ZEngine/ZEngine/Core/EngineAssetRoot.h | 34 +++++ ZEngine/ZEngine/Engine.cpp | 33 +++-- ZEngine/ZEngine/Engine.h | 1 + .../Rendering/RenderResourceManager.cpp | 26 +++- ZEngine/ZEngine/Windows/GameWindow.cpp | 3 +- ZEngine/tests/Misc/EngineAssetRoot_test.cpp | 78 ++++++++++++ docs/engine-architecture.md | 16 ++- 15 files changed, 408 insertions(+), 82 deletions(-) create mode 100644 Resources/Editor/Settings/FallbackTexture.png create mode 100644 ZEngine/ZEngine/Core/EngineAssetRoot.cpp create mode 100644 ZEngine/ZEngine/Core/EngineAssetRoot.h create mode 100644 ZEngine/tests/Misc/EngineAssetRoot_test.cpp diff --git a/Obelisk/EntryPoint.cpp b/Obelisk/EntryPoint.cpp index ce957004..fb08fa8b 100644 --- a/Obelisk/EntryPoint.cpp +++ b/Obelisk/EntryPoint.cpp @@ -66,9 +66,6 @@ int applicationEntryPoint(int argc, char* argv[]) app->Run(); app->Shutdown(); - // Step 15 — join worker threads before logger/memory teardown - Helpers::ThreadPoolHelper::Shutdown(); - // Step 16 — flush and dispose logger Logger::Flush(); Logger::Dispose(); diff --git a/Resources/Editor/Settings/FallbackTexture.png b/Resources/Editor/Settings/FallbackTexture.png new file mode 100644 index 0000000000000000000000000000000000000000..41799f67d9976bc134d5292c82aefa8c46999b82 GIT binary patch literal 78 zcmeAS@N?(olHy`uVBq!ia0vp^EFjFm1|(O0oL2{=L_J*`Ln`8u|L7b2v}YDdNOE9q aVP#n7!?gB8;Uy2C0tQc4KbLh*2~7Zc=o4rF literal 0 HcmV?d00001 diff --git a/Tetragrama/Panels/AssetImporterPanel.cpp b/Tetragrama/Panels/AssetImporterPanel.cpp index bb21cf48..61ee8590 100644 --- a/Tetragrama/Panels/AssetImporterPanel.cpp +++ b/Tetragrama/Panels/AssetImporterPanel.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -32,9 +33,10 @@ namespace Tetragrama::Panels // Initialize - void AssetImporterPanel::Initialize(Tetragrama::Layers::ZUILayer* layer) + void AssetImporterPanel::Initialize(Tetragrama::Layers::ZUILayer* layer, ProjectViewPanel* project_view) { - m_layer = layer; + m_layer = layer; + m_project_view = project_view; if (!layer) return; @@ -82,9 +84,6 @@ namespace Tetragrama::Panels StartImport(); } - // Consume TriggerScan posted by background thread (actor creation on main thread) - TriggerScan(); - ZUIBox* bg = ZUIBeginColumn(ctx, "##imp_bg", ZFill(), ZFill()); bg->Flags = bg->Flags | ZUI_DrawBackground | ZUI_Scrollable; ZUIBoxSetColorArr(bg, kBg); @@ -595,6 +594,9 @@ namespace Tetragrama::Panels void AssetImporterPanel::StartImport() { + if (m_state.value.load(std::memory_order_acquire) == ImporterState::Importing) + return; + if (!m_gltf_importer || !m_fbx_importer || !m_assimp_importer) return; @@ -672,7 +674,10 @@ namespace Tetragrama::Panels void AssetImporterPanel::RunImportTask(void* context) { - auto* task = static_cast(context); + auto* task = static_cast(context); + if (!task) + return; + auto* panel = task->Panel; if (!panel) return; @@ -681,29 +686,49 @@ namespace Tetragrama::Panels switch (task->Kind) { case ImporterKind::Gltf: - panel->m_gltf_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, panel, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); + panel->m_gltf_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, task, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); break; case ImporterKind::Fbx: - panel->m_fbx_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, panel, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); + panel->m_fbx_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, task, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); break; case ImporterKind::Assimp: - panel->m_assimp_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, panel, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); + panel->m_assimp_importer->ImportFile(task->SourcePath, config, &panel->m_local_arena, task, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); break; } - } - // TriggerScan (main-thread only) + if (task->Outcome == ImportTaskOutcome::Pending) + { + task->Error = "Importer returned without reporting a result"; + task->Outcome = ImportTaskOutcome::Failed; + } + + // ImportFile's local scratch storage is gone once it returns. The worker + // callbacks have already copied the required output paths into ImportTask. + ZEngine::Core::MainThreadScheduler::Post(task, &AssetImporterPanel::FinalizeImportTask); + } - void AssetImporterPanel::TriggerScan() + void AssetImporterPanel::FinalizeImportTask(void* context) { - if (!m_pending_actor.valid) + auto* task = static_cast(context); + if (!task || !task->Panel) return; - auto* app = m_layer ? reinterpret_cast(m_layer->CurrentApp) : nullptr; - auto* scene = app ? reinterpret_cast(app->CurrentScene) : nullptr; - auto* ctx = ZEngine::Engine::GetContext(); + auto* panel = task->Panel; + if (task->Outcome == ImportTaskOutcome::Completed) + { + ZEngine::Core::Containers::ArrayView outputs(task->Outputs.data(), task->Outputs.size()); + CompleteImportOnMainThread(panel, outputs); + return; + } + + CompleteImportErrorOnMainThread(panel, task->Error.empty() ? "Importer returned without an error message" : task->Error); + } + + void AssetImporterPanel::CreateMeshActor(uuids::uuid mesh_uuid, uint32_t render_instance_id, const char* name) + { + auto* ctx = ZEngine::Engine::GetContext(); - if (scene && ctx && ctx->ActorManager) + if (ctx && ctx->ActorManager) { using namespace ZEngine::ECS::Components; ZEngine::ECS::ActorHandle handle = ctx->ActorManager->Create(); @@ -711,16 +736,15 @@ namespace Tetragrama::Panels if (actor) { NameComponent nc = {}; - secure_strncpy(nc.Value, sizeof(nc.Value), m_pending_actor.name, secure_strlen(m_pending_actor.name)); + secure_strncpy(nc.Value, sizeof(nc.Value), name, secure_strlen(name)); actor->AddComponent(nc); actor->AddComponent({}); MeshComponent mc = {}; - mc.MeshUUID = m_pending_actor.uuid; - mc.RenderInstanceId = m_pending_actor.render_id; + mc.MeshUUID = mesh_uuid; + mc.RenderInstanceId = render_instance_id; actor->AddComponent(mc); } } - m_pending_actor = {}; } // PushLog / PushHistory @@ -737,7 +761,6 @@ namespace Tetragrama::Panels m_log_head = (m_log_head + 1) % kLogMax; if (m_log_count < kLogMax) ++m_log_count; - m_scroll_log = true; } void AssetImporterPanel::PushHistory(const char* name, bool ok, const char* msg) @@ -765,7 +788,23 @@ namespace Tetragrama::Panels void AssetImporterPanel::OnImportFileComplete(void* ctx, ZEngine::Core::Containers::ArrayView outputs) { - auto* self = reinterpret_cast(ctx); + auto* task = static_cast(ctx); + if (!task) + return; + + task->Outputs.clear(); + if (outputs.size() > 0) + task->Outputs.assign(outputs.data(), outputs.data() + outputs.size()); + task->Error.clear(); + task->Outcome = ImportTaskOutcome::Completed; + } + + void AssetImporterPanel::CompleteImportOnMainThread(void* ctx, ZEngine::Core::Containers::ArrayView outputs) + { + auto* self = reinterpret_cast(ctx); + + if (self->m_project_view) + self->m_project_view->RequestRefresh(); bool has_mesh = false; cstring mesh_path = nullptr; @@ -882,11 +921,7 @@ namespace Tetragrama::Panels snprintf(iname, sizeof(iname), "%.*s", (int) s.Length, s.Data); } } - uint32_t render_id = scene->AddMeshInstance(header.Id, iname); - self->m_pending_actor.uuid = header.Id; - self->m_pending_actor.render_id = render_id; - self->m_pending_actor.valid = true; - secure_strncpy(self->m_pending_actor.name, sizeof(self->m_pending_actor.name), iname, sizeof(self->m_pending_actor.name) - 1); + self->CreateMeshActor(header.Id, scene->AddMeshInstance(header.Id, iname), iname); } } self->m_add_to_scene = false; @@ -923,19 +958,33 @@ namespace Tetragrama::Panels self->m_progress.value.store(1.f, std::memory_order_relaxed); self->m_state.value.store(ImporterState::Idle, std::memory_order_release); - ZEngine::Core::MainThreadScheduler::Post(self, [](void* c) { reinterpret_cast(c)->TriggerScan(); }); } void AssetImporterPanel::OnImportProgress(void* ctx, float pct) { - auto* self = reinterpret_cast(ctx); - char msg[128]; + auto* task = static_cast(ctx); + auto* self = task ? task->Panel : nullptr; + if (!self) + return; + + char msg[128]; snprintf(msg, sizeof(msg), "Processing... %.0f%%", pct * 100.f); self->PushLog(msg, kWhite[0], kWhite[1], kWhite[2]); self->m_progress.value.store(pct, std::memory_order_relaxed); } void AssetImporterPanel::OnImportError(void* ctx, std::string_view err) + { + auto* task = static_cast(ctx); + if (!task) + return; + + task->Outputs.clear(); + task->Error.assign(err.data(), err.size()); + task->Outcome = ImportTaskOutcome::Failed; + } + + void AssetImporterPanel::CompleteImportErrorOnMainThread(void* ctx, std::string_view err) { auto* self = reinterpret_cast(ctx); char msg[512]; @@ -952,13 +1001,16 @@ namespace Tetragrama::Panels } self->PushHistory(fn, false, msg); self->m_state.value.store(ImporterState::Idle, std::memory_order_release); - ZEngine::Core::MainThreadScheduler::Post(self, [](void* c) { reinterpret_cast(c)->TriggerScan(); }); } void AssetImporterPanel::OnImportLog(void* ctx, std::string_view msg) { - auto* self = reinterpret_cast(ctx); - char buf[256]; + auto* task = static_cast(ctx); + auto* self = task ? task->Panel : nullptr; + if (!self) + return; + + char buf[256]; snprintf(buf, sizeof(buf), "%.*s", (int) msg.size(), msg.data()); self->PushLog(buf, kWhite[0], kWhite[1], kWhite[2]); } diff --git a/Tetragrama/Panels/AssetImporterPanel.h b/Tetragrama/Panels/AssetImporterPanel.h index e94c951c..5ecf923a 100644 --- a/Tetragrama/Panels/AssetImporterPanel.h +++ b/Tetragrama/Panels/AssetImporterPanel.h @@ -9,13 +9,16 @@ #include #include #include +#include +#include namespace Tetragrama::Panels { + struct ProjectViewPanel; + /// @brief Three-state asset importer panel (Idle → Options → Importing). /// Supports glTF/GLB, FBX, and Assimp-backed formats. Import runs - /// on a background thread; ECS actor creation is posted back to the - /// main thread via TriggerScan(). + /// on a background thread; completion is finalized on the main thread. struct AssetImporterPanel : ZEngine::UI::ZUIPanelView { AssetImporterPanel() @@ -25,7 +28,8 @@ namespace Tetragrama::Panels /// @brief Allocates importers and arenas; must be called once before first use. /// @param layer Owning ZUI layer (provides arena and app pointers). - void Initialize(Tetragrama::Layers::ZUILayer* layer); + /// @param project_view Content browser refreshed after a successful import. + void Initialize(Tetragrama::Layers::ZUILayer* layer, ProjectViewPanel* project_view); /// @brief Dispatches to BuildIdle/BuildOptions/BuildImporting based on current state. /// @param ctx ZUI context for the current frame. @@ -34,6 +38,7 @@ namespace Tetragrama::Panels private: Tetragrama::Layers::ZUILayer* m_layer = nullptr; + ProjectViewPanel* m_project_view = nullptr; // Scratch arena for ImportConfiguration strings (cleared before each import) ZEngine::Core::Memory::ArenaAllocator m_local_arena = {}; @@ -53,12 +58,24 @@ namespace Tetragrama::Panels Assimp, }; + enum class ImportTaskOutcome : uint8_t + { + Pending, + Completed, + Failed, + }; + struct ImportTask { - AssetImporterPanel* Panel = nullptr; - ZEngine::Importers::AssetCodec::ImportConfiguration Configuration = {}; - ImporterKind Kind = ImporterKind::Gltf; - char SourcePath[MAX_FILE_PATH_COUNT] = {}; + AssetImporterPanel* Panel = nullptr; + ZEngine::Importers::AssetCodec::ImportConfiguration Configuration = {}; + ImporterKind Kind = ImporterKind::Gltf; + char SourcePath[MAX_FILE_PATH_COUNT] = {}; + // The importer releases its scratch arena after it calls its callback. + // Retain the small output list until the main thread consumes it. + std::vector Outputs = {}; + std::string Error = {}; + ImportTaskOutcome Outcome = ImportTaskOutcome::Pending; }; ImportTask m_import_task = {}; @@ -87,17 +104,8 @@ namespace Tetragrama::Panels bool m_same_settings = false; int m_options_filter = 5; // 0=General 1=Mesh 2=Material 3=Anim 4=LOD 5=All - // Pending ECS actor creation — written on bg thread, consumed on main thread - struct PendingActor - { - uuids::uuid uuid = {}; - uint32_t render_id = UINT32_MAX; - char name[128] = {}; - bool valid = false; - } m_pending_actor = {}; - // Import history ring buffer (Idle state, newest first) - static constexpr int kHistMax = 16; + static constexpr int kHistMax = 16; struct HistEntry { char name[256] = {}; @@ -117,7 +125,6 @@ namespace Tetragrama::Panels LogEntry m_log[kLogMax] = {}; int m_log_head = 0; int m_log_count = 0; - bool m_scroll_log = false; std::mutex m_log_mutex; // Build helpers @@ -134,25 +141,28 @@ namespace Tetragrama::Panels std::future BrowseFileAsync(); /// @brief Begin the import process with the current settings. void StartImport(); - /// @brief Trigger a directory scan for importable assets. - void TriggerScan(); // main-thread only + /// @brief Add an imported mesh instance to the ECS scene on the main thread. + void CreateMeshActor(uuids::uuid mesh_uuid, uint32_t render_instance_id, const char* name); /// @brief Append a log message to the import log. void PushLog(const char* text, float r, float g, float b); /// @brief Record the completed import in the history list. void PushHistory(const char* name, bool ok, const char* msg); - /// @brief Runs one import on a worker through the allocation-free task API. + /// @brief Runs one import on a worker through the C-style task API. static void RunImportTask(void* context); + /// @brief Consumes a completed worker task on the main thread. + static void FinalizeImportTask(void* context); - // Static callbacks for ImportFile (called from background thread) - /// @brief Callback invoked when the import file step finishes. + // Static callbacks for ImportFile. They execute on a worker, so they only + // copy the result into ImportTask; FinalizeImportTask performs editor work. static void OnImportFileComplete(void* ctx, ZEngine::Core::Containers::ArrayView outputs); - /// @brief Callback invoked with progress updates during import. static void OnImportProgress(void* ctx, float pct); - /// @brief Callback invoked when an import error occurs. static void OnImportError(void* ctx, std::string_view err); - /// @brief Callback invoked for each import log message. static void OnImportLog(void* ctx, std::string_view msg); + + // Main-thread-only completion handlers. + static void CompleteImportOnMainThread(void* ctx, ZEngine::Core::Containers::ArrayView outputs); + static void CompleteImportErrorOnMainThread(void* ctx, std::string_view err); }; } // namespace Tetragrama::Panels diff --git a/Tetragrama/Panels/ProjectViewPanel.cpp b/Tetragrama/Panels/ProjectViewPanel.cpp index 38571420..75b1d5b5 100644 --- a/Tetragrama/Panels/ProjectViewPanel.cpp +++ b/Tetragrama/Panels/ProjectViewPanel.cpp @@ -211,6 +211,12 @@ namespace Tetragrama::Panels Title = "Project"; } + void ProjectViewPanel::RequestRefresh() + { + m_needs_refresh = true; + m_tree_cache_count = 0; + } + // VFS listing void ProjectViewPanel::RefreshListing(IVFSContext* vfs) { diff --git a/Tetragrama/Panels/ProjectViewPanel.h b/Tetragrama/Panels/ProjectViewPanel.h index 087a9dd7..1bad8377 100644 --- a/Tetragrama/Panels/ProjectViewPanel.h +++ b/Tetragrama/Panels/ProjectViewPanel.h @@ -27,6 +27,9 @@ namespace Tetragrama::Panels /// @param rect Panel bounding rect [x0, y0, x1, y1]. void BuildContent(ZEngine::UI::ZUIContext* ctx, float rect[4]) override; + /// @brief Invalidates cached listings after a main-thread file operation. + void RequestRefresh(); + private: // Navigation ZEngine::Core::VFS::VFSPath m_current_dir = {}; diff --git a/Tetragrama/Panels/ZUIPanelManagerComponent.cpp b/Tetragrama/Panels/ZUIPanelManagerComponent.cpp index 03e8f509..10fd2c7e 100644 --- a/Tetragrama/Panels/ZUIPanelManagerComponent.cpp +++ b/Tetragrama/Panels/ZUIPanelManagerComponent.cpp @@ -64,12 +64,14 @@ namespace Tetragrama::Panels auto* p_proj = Manager.AddPanel(ZUIDockHashName("Project")); Manager.AddView(p_proj, &project); - importer.Initialize(parent); // allocate importers from ImportPipeline budget + importer.Initialize(parent, &project); // allocate importers from ImportPipeline budget - // EngineAssetsBackend is rooted at /ZodiacEngine (mounted at /ZodiacEngine). + // EngineAssetsBackend is rooted at the packaged ZodiacEngine directory + // beside the executable (mounted at /ZodiacEngine). // Settings/ is created by the engine build. Verify it exists via the backend, // then use VFSPath::ResolveNative to produce the correct native path — no - // manual string concatenation. Falls back to the working directory if absent. + // manual string concatenation. A missing settings directory uses the local + // layout filename so the editor can still start with its default layout. char layout_path[512] = {}; bool settings_ok = false; auto* eng = ZEngine::Engine::GetContext(); diff --git a/ZEngine/ZEngine/Core/EngineAssetRoot.cpp b/ZEngine/ZEngine/Core/EngineAssetRoot.cpp new file mode 100644 index 00000000..08dc95b5 --- /dev/null +++ b/ZEngine/ZEngine/Core/EngineAssetRoot.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#elif defined(__APPLE__) +#include +#elif defined(__linux__) +#include +#endif + +namespace ZEngine::Core +{ + namespace + { + constexpr const char* kEngineAssetDirectory = "ZodiacEngine"; + + std::filesystem::path NormalizePath(const std::filesystem::path& path) + { + std::error_code error; + std::filesystem::path absolute = std::filesystem::absolute(path, error); + return error ? path.lexically_normal() : absolute.lexically_normal(); + } + } // namespace + + std::filesystem::path GetExecutablePath() + { +#if defined(_WIN32) + std::vector buffer(MAX_PATH); + for (;;) + { + const DWORD length = GetModuleFileNameW(nullptr, buffer.data(), static_cast(buffer.size())); + if (length == 0) + return {}; + if (length < buffer.size()) + return std::filesystem::path(std::wstring(buffer.data(), length)); + buffer.resize(buffer.size() * 2); + } +#elif defined(__APPLE__) + uint32_t size = 0; + if (_NSGetExecutablePath(nullptr, &size) != -1 || size == 0) + return {}; + + std::vector buffer(size); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) + return {}; + return std::filesystem::path(buffer.data()); +#elif defined(__linux__) + std::vector buffer(1024); + for (;;) + { + const ssize_t length = readlink("/proc/self/exe", buffer.data(), buffer.size()); + if (length < 0) + return {}; + if (static_cast(length) < buffer.size()) + return std::filesystem::path(std::string(buffer.data(), static_cast(length))); + buffer.resize(buffer.size() * 2); + } +#else + return {}; +#endif + } + + EngineAssetRootResolution ResolveEngineAssetRoot(const std::filesystem::path& executable_path, const std::filesystem::path& override_root) + { + if (!override_root.empty()) + return {.Root = NormalizePath(override_root)}; + + if (executable_path.empty()) + return {.Diagnostic = "Unable to determine the executable path while resolving the engine asset package. Set ZENGINE_ASSET_ROOT to the ZodiacEngine directory."}; + + return {.Root = NormalizePath(executable_path.parent_path() / kEngineAssetDirectory)}; + } + + EngineAssetRootResolution ResolveEngineAssetRoot() + { + const char* const override_root = std::getenv(EngineAssetRootOverrideEnvironment); + return ResolveEngineAssetRoot(GetExecutablePath(), override_root && override_root[0] != '\0' ? std::filesystem::path(override_root) : std::filesystem::path{}); + } + + EngineAssetRootResolution ValidateEngineAssetRoot(const std::filesystem::path& root) + { + if (root.empty()) + return {.Diagnostic = "Engine asset root is empty"}; + + EngineAssetRootResolution result = {.Root = NormalizePath(root)}; + std::error_code error; + if (!std::filesystem::is_directory(result.Root, error)) + { + result.Diagnostic = "Engine asset root is missing or not a directory: " + result.Root.string(); + } + return result; + } + +} // namespace ZEngine::Core diff --git a/ZEngine/ZEngine/Core/EngineAssetRoot.h b/ZEngine/ZEngine/Core/EngineAssetRoot.h new file mode 100644 index 00000000..b9e0fb9f --- /dev/null +++ b/ZEngine/ZEngine/Core/EngineAssetRoot.h @@ -0,0 +1,34 @@ +#pragma once +#include +#include + +namespace ZEngine::Core +{ + inline constexpr const char* EngineAssetRootOverrideEnvironment = "ZENGINE_ASSET_ROOT"; + + struct EngineAssetRootResolution + { + std::filesystem::path Root = {}; + std::string Diagnostic = {}; + + [[nodiscard]] bool Succeeded() const + { + return Diagnostic.empty(); + } + }; + + // Returns the executable's native path, independent of the process working directory. + [[nodiscard]] std::filesystem::path GetExecutablePath(); + + // Resolves the directory containing packaged engine files. An explicit override + // takes precedence; otherwise assets live in /ZodiacEngine. + [[nodiscard]] EngineAssetRootResolution ResolveEngineAssetRoot(const std::filesystem::path& executable_path, const std::filesystem::path& override_root = {}); + + // Uses the current executable and the optional ZENGINE_ASSET_ROOT environment + // override. The override exists for development tools and package tests. + [[nodiscard]] EngineAssetRootResolution ResolveEngineAssetRoot(); + + // Checks that the native package directory exists before it is mounted. + [[nodiscard]] EngineAssetRootResolution ValidateEngineAssetRoot(const std::filesystem::path& root); + +} // namespace ZEngine::Core diff --git a/ZEngine/ZEngine/Engine.cpp b/ZEngine/ZEngine/Engine.cpp index 07c71204..72019594 100644 --- a/ZEngine/ZEngine/Engine.cpp +++ b/ZEngine/ZEngine/Engine.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -119,11 +121,22 @@ namespace ZEngine ZENGINE_VALIDATE_ASSERT(Logging::Logger::IsInitialized(), "Engine::Initialize: Logger not initialized — Obelisk must call Logger::Initialize first") ZENGINE_VALIDATE_ASSERT(Helpers::ThreadPoolHelper::IsInitialized(), "Engine::Initialize: ThreadPool not initialized — Obelisk must call ThreadPoolHelper::Initialize first") - auto& arena = memory->BootstrapArena; + const Core::EngineAssetRootResolution resolved_asset_root = Core::ResolveEngineAssetRoot(); + const Core::EngineAssetRootResolution validated_asset_root = resolved_asset_root.Succeeded() ? Core::ValidateEngineAssetRoot(resolved_asset_root.Root) : resolved_asset_root; + if (!validated_asset_root.Succeeded()) + { + ZENGINE_CORE_CRITICAL("Engine startup aborted: {}", validated_asset_root.Diagnostic) + ZENGINE_EXIT_FAILURE() + } - g_engine_ctx = ZPushStructCtor(&arena, EngineContext); + auto& arena = memory->BootstrapArena; - auto window = ZPushStructCtor(&arena, Windows::GameWindow); + g_engine_ctx = ZPushStructCtor(&arena, EngineContext); + const std::string engine_assets_path = validated_asset_root.Root.string(); + g_engine_ctx->EngineAssetsNativeRoot = ZPushString(&arena, engine_assets_path.size() + 1); + ZENGINE_VALIDATE_ASSERT(Helpers::secure_memcpy(g_engine_ctx->EngineAssetsNativeRoot, engine_assets_path.size() + 1, engine_assets_path.c_str(), engine_assets_path.size() + 1) == Helpers::MEMORY_OP_SUCCESS, "Engine::Initialize: failed to persist engine asset root") + + auto window = ZPushStructCtor(&arena, Windows::GameWindow); window->SetCallbackFunction(std::bind(&Applications::GameApplication::ProcessEvent, app, std::placeholders::_1)); window->Initialize(&arena, *window_cfg_ptr); g_engine_ctx->Window = window; @@ -142,13 +155,12 @@ namespace ZEngine g_engine_ctx->VFS = vfs_ctx; { - const std::string engine_dir = std::filesystem::current_path().string() + "/ZodiacEngine"; - g_engine_ctx->EngineAssetsBackend.Initialize(engine_dir.c_str(), Core::VFS::VFSBackendCaps::Read | Core::VFS::VFSBackendCaps::Write | Core::VFS::VFSBackendCaps::List, &g_engine_ctx->VFSArena); + g_engine_ctx->EngineAssetsBackend.Initialize(g_engine_ctx->EngineAssetsNativeRoot, Core::VFS::VFSBackendCaps::Read | Core::VFS::VFSBackendCaps::Write | Core::VFS::VFSBackendCaps::List, &g_engine_ctx->VFSArena); auto mount_path = Core::VFS::VFSPath::Parse("/ZodiacEngine"); - if (mount_path.Succeeded()) + if (mount_path.Failed() || vfs_ctx->Mount(&g_engine_ctx->EngineAssetsBackend, mount_path.Value(), -1).Failed()) { - auto res = vfs_ctx->Mount(&g_engine_ctx->EngineAssetsBackend, mount_path.Value(), -1); - (void) res; + ZENGINE_CORE_CRITICAL("Engine startup aborted: failed to mount engine assets at /ZodiacEngine from {}", g_engine_ctx->EngineAssetsNativeRoot) + ZENGINE_EXIT_FAILURE() } const bool has_writable_workspace = app->WorkingSpacePath && app->WorkingSpacePath[0] != '\0' && app->VFSBackend && Core::VFS::HasCap(app->VFSBackend->Capabilities(), Core::VFS::VFSBackendCaps::Write); @@ -264,6 +276,11 @@ namespace ZEngine g_engine_ctx->RenderThread.join(); } + // Import and VFS work run on this pool. Join it while the scheduler and + // every engine-owned service are still valid, so an in-flight import cannot + // publish a completion into torn-down state. + Helpers::ThreadPoolHelper::Shutdown(); + Core::MainThreadScheduler::Shutdown(); // Step 3 — ECS shutdown: ActorManager before Scene (lifecycle order) diff --git a/ZEngine/ZEngine/Engine.h b/ZEngine/ZEngine/Engine.h index c63aa880..8ccf8c2b 100644 --- a/ZEngine/ZEngine/Engine.h +++ b/ZEngine/ZEngine/Engine.h @@ -33,6 +33,7 @@ namespace ZEngine // Mounted at /ZodiacEngine with priority -1 so the workspace backend // (priority 0) takes precedence for any overlapping paths. Core::VFS::VFSDiskBackend EngineAssetsBackend = {}; + char* EngineAssetsNativeRoot = nullptr; // Writable project-owned backend mounted specifically at /cache/pso. // It resolves to /.zodiacengine/cache/pso and is distinct diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index 1e112ff0..edf6bb15 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -2,6 +2,7 @@ // stbi_load pixel buffers stay on the slab rather than the system heap. // Falls back to malloc/free/realloc on the main thread (slab = nullptr). #include +#include #include #include #define STBI_MALLOC(sz) (ZEngine::Helpers::GetWorkerSlab() ? ZEngine::Helpers::GetWorkerSlab()->Alloc(sz) : std::malloc(sz)) @@ -56,7 +57,7 @@ namespace ZEngine::Rendering uint16_t FloatToHalf(float value) { uint32_t bits = 0; - std::memcpy(&bits, &value, sizeof(bits)); + ZENGINE_VALIDATE_ASSERT(secure_memcpy(&bits, sizeof(bits), &value, sizeof(value)) == MEMORY_OP_SUCCESS, "FloatToHalf: failed to copy float bits") const uint16_t sign = static_cast((bits >> 16) & 0x8000u); const int32_t exponent = static_cast((bits >> 23) & 0xFFu) - 127 + 15; @@ -2091,11 +2092,20 @@ namespace ZEngine::Rendering Rendering::Textures::TextureHandle RenderResourceManager::GetOrCreateFallbackTexture() { - static constexpr const char* kFallbackPath = "ZodiacEngine/Settings/FallbackTexture.png"; + static constexpr const char* kFallbackRelativePath = "Settings/FallbackTexture.png"; - if (!std::filesystem::exists(kFallbackPath)) + const auto* engine_context = Engine::GetContext(); + if (!engine_context || !engine_context->EngineAssetsNativeRoot) { - // 4×4 (255, 20, 147, 255) fallback color for missing textures + ZENGINE_CORE_ERROR("Fallback texture cannot be loaded because the engine asset root is unavailable") + return {}; + } + + const std::string fallback_path = (std::filesystem::path(engine_context->EngineAssetsNativeRoot) / kFallbackRelativePath).string(); + if (!std::filesystem::exists(fallback_path)) + { + // Keep GetOrCreateFallbackTexture's recovery behavior for development + // packages that were produced without the bundled fallback image. static constexpr int W = 4, H = 4; static constexpr uint8_t R = 255, G = 20, B = 147, A = 255; uint8_t pixels[W * H * 4]; @@ -2106,10 +2116,14 @@ namespace ZEngine::Rendering pixels[i * 4 + 2] = B; pixels[i * 4 + 3] = A; } - stbi_write_png(kFallbackPath, W, H, 4, pixels, W * 4); + if (!stbi_write_png(fallback_path.c_str(), W, H, 4, pixels, W * 4)) + { + ZENGINE_CORE_ERROR("Failed to create fallback texture: {}", fallback_path) + return {}; + } } - auto result = SubmitTextureFile(kFallbackPath); + auto result = SubmitTextureFile(fallback_path.c_str()); if (result.Valid()) { auto texture = m_device->GlobalTextures.Access(result); diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index 638e4f49..d5babcc1 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -133,7 +133,8 @@ namespace ZEngine::Windows } { - auto icon_path = (std::filesystem::current_path() / "ZodiacEngine/Settings/Icons/AppIconBadge.png").string(); + const auto* engine_context = Engine::GetContext(); + const auto icon_path = engine_context && engine_context->EngineAssetsNativeRoot ? (std::filesystem::path(engine_context->EngineAssetsNativeRoot) / "Settings/Icons/AppIconBadge.png").string() : std::string{}; int w = 0, h = 0, channels = 0; unsigned char* pixels = stbi_load(icon_path.c_str(), &w, &h, &channels, STBI_rgb_alpha); if (pixels) diff --git a/ZEngine/tests/Misc/EngineAssetRoot_test.cpp b/ZEngine/tests/Misc/EngineAssetRoot_test.cpp new file mode 100644 index 00000000..cf96bc5b --- /dev/null +++ b/ZEngine/tests/Misc/EngineAssetRoot_test.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include + +namespace +{ + class EngineAssetRootTest : public ::testing::Test + { + protected: + void SetUp() override + { + static std::atomic next_id = 0; + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + m_root = std::filesystem::temp_directory_path() / ("zengine_engine_asset_root_" + std::to_string(now) + "_" + std::to_string(next_id.fetch_add(1, std::memory_order_relaxed))); + std::error_code error; + std::filesystem::remove_all(m_root, error); + std::filesystem::create_directories(m_root, error); + ASSERT_FALSE(error); + } + + void TearDown() override + { + std::error_code error; + std::filesystem::remove_all(m_root, error); + } + + std::filesystem::path m_root; + }; +} // namespace + +TEST_F(EngineAssetRootTest, ResolvesPackageBesideTheExecutable) +{ + const std::filesystem::path executable = m_root / "bin" / "Obelisk"; + + const auto result = ZEngine::Core::ResolveEngineAssetRoot(executable); + + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Root, m_root / "bin" / "ZodiacEngine"); +} + +TEST_F(EngineAssetRootTest, ExplicitOverrideTakesPrecedence) +{ + const std::filesystem::path executable = m_root / "bin" / "Obelisk"; + const std::filesystem::path override = m_root / "tooling-assets"; + + const auto result = ZEngine::Core::ResolveEngineAssetRoot(executable, override); + + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Root, override); +} + +TEST_F(EngineAssetRootTest, RootValidationRejectsMissingPackageDirectory) +{ + const auto result = ZEngine::Core::ValidateEngineAssetRoot(m_root / "missing"); + + EXPECT_FALSE(result.Succeeded()); + EXPECT_NE(result.Diagnostic.find("missing"), std::string::npos); +} + +TEST_F(EngineAssetRootTest, RootValidationRejectsEmptyPath) +{ + const auto result = ZEngine::Core::ValidateEngineAssetRoot({}); + + EXPECT_FALSE(result.Succeeded()); + EXPECT_NE(result.Diagnostic.find("empty"), std::string::npos); +} + +TEST_F(EngineAssetRootTest, RootValidationAcceptsPackageDirectory) +{ + const std::filesystem::path package_root = m_root / "ZodiacEngine"; + std::filesystem::create_directories(package_root); + + const auto result = ZEngine::Core::ValidateEngineAssetRoot(package_root); + + EXPECT_TRUE(result.Succeeded()) << result.Diagnostic; +} diff --git a/docs/engine-architecture.md b/docs/engine-architecture.md index cdb72b58..8cc28f54 100644 --- a/docs/engine-architecture.md +++ b/docs/engine-architecture.md @@ -468,7 +468,7 @@ the render thread resets global buffer cursors to 0 so new geometry starts fresh ```mermaid graph TD ctx["VFSContext"] - m1["Mount /ZodiacEngine\n→ VFSDiskBackend\n→ cwd/ZodiacEngine/\npriority = -1"] + m1["Mount /ZodiacEngine\n→ VFSDiskBackend\n→ executable-dir/ZodiacEngine/\npriority = -1"] m2["Mount /\n→ VFSDiskBackend\n→ project root\npriority = 0"] path["VFSPath\nnormalized, immutable\nno-alloc value type"] scanner["VFSScanner\nasync directory walker\npopulates content browser cache"] @@ -529,6 +529,20 @@ sequenceDiagram Hard dependencies: Device before VFS (surface), RRM before fallback texture, watcher after working directory. +### Packaged engine assets + +Engine-owned files are resolved from `ZodiacEngine/` beside the executable, not +from the process working directory. This keeps a Debug or installed package +relocatable: it may be launched from the repository root, its output directory, +or any other directory. Startup validates the package root before mounting it +at `/ZodiacEngine`; all files under that directory then use the ordinary VFS +lookup path, including shaders. + +Development tools and package tests may explicitly set `ZENGINE_ASSET_ROOT` to +an alternate `ZodiacEngine` directory. The override takes precedence over the +executable-relative package location; normal application launches should not +need it. + --- ## Shutdown Order From 3b936e9921ec7ce332b68cfee01ddcfb025d34e9 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 11:16:30 +0900 Subject: [PATCH 02/27] feat(linux): support Wayland file dialogs --- .github/workflows/job-cmakebuild-linux.yml | 2 +- ZEngine/ZEngine/CMakeLists.txt | 27 +- ZEngine/ZEngine/Windows/GameWindow.cpp | 137 ++++++---- .../Platform/WaylandPortalFileDialog.cpp | 245 ++++++++++++++++++ .../Platform/WaylandPortalFileDialog.h | 54 ++++ .../Platform/xdg-foreign-unstable-v2.xml | 45 ++++ dependencies.cmake | 11 +- 7 files changed, 465 insertions(+), 56 deletions(-) create mode 100644 ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp create mode 100644 ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h create mode 100644 ZEngine/ZEngine/Windows/Platform/xdg-foreign-unstable-v2.xml diff --git a/.github/workflows/job-cmakebuild-linux.yml b/.github/workflows/job-cmakebuild-linux.yml index 15413d6d..a473e8c8 100644 --- a/.github/workflows/job-cmakebuild-linux.yml +++ b/.github/workflows/job-cmakebuild-linux.yml @@ -35,7 +35,7 @@ jobs: run: sudo apt-get update - name: Install Dependencies - run: sudo apt-get install libasound2-dev libgl1-mesa-dev libpulse-dev libdbus-1-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxrandr-dev libxss-dev libxt-dev libxxf86vm-dev cmake apt-transport-https software-properties-common gcc g++ powershell clang-format-22 glslang-tools libwayland-dev libxkbcommon-dev + run: sudo apt-get install libasound2-dev libgl1-mesa-dev libpulse-dev libdbus-1-dev libglib2.0-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxrandr-dev libxss-dev libxt-dev libxxf86vm-dev cmake apt-transport-https software-properties-common gcc g++ powershell clang-format-22 glslang-tools libwayland-dev libxkbcommon-dev pkg-config - name: CMake Build run: .\Scripts\BuildEngine.ps1 -Configurations ${{inputs.configuration}} diff --git a/ZEngine/ZEngine/CMakeLists.txt b/ZEngine/ZEngine/CMakeLists.txt index 230fb4cb..e0ac45f9 100644 --- a/ZEngine/ZEngine/CMakeLists.txt +++ b/ZEngine/ZEngine/CMakeLists.txt @@ -185,7 +185,32 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") target_compile_definitions(zEngineLib PUBLIC NOMINMAX GLFW_EXPOSE_NATIVE_WIN32) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") target_link_libraries(zEngineLib PUBLIC stdc++fs dl) - target_compile_definitions(zEngineLib PUBLIC GLFW_EXPOSE_NATIVE_WAYLAND) + # GLFW 3.5 includes both backends and selects the active session at runtime. + # The file-dialog layer needs both native window types for correct parenting. + target_compile_definitions(zEngineLib PUBLIC GLFW_EXPOSE_NATIVE_X11 GLFW_EXPOSE_NATIVE_WAYLAND) + + find_package(PkgConfig REQUIRED) + pkg_check_modules(GIO REQUIRED IMPORTED_TARGET gio-2.0) + pkg_check_modules(WAYLAND_CLIENT REQUIRED IMPORTED_TARGET wayland-client) + target_link_libraries(zEngineLib PRIVATE PkgConfig::GIO PkgConfig::WAYLAND_CLIENT) + + set(ZENGINE_XDG_FOREIGN_PROTOCOL ${CMAKE_CURRENT_SOURCE_DIR}/Windows/Platform/xdg-foreign-unstable-v2.xml) + set(ZENGINE_XDG_FOREIGN_HEADER ${CMAKE_CURRENT_BINARY_DIR}/xdg-foreign-unstable-v2-client-protocol.h) + set(ZENGINE_XDG_FOREIGN_CODE ${CMAKE_CURRENT_BINARY_DIR}/xdg-foreign-unstable-v2-client-protocol.c) + + find_program(WAYLAND_SCANNER_EXECUTABLE NAMES wayland-scanner REQUIRED) + add_custom_command( + OUTPUT ${ZENGINE_XDG_FOREIGN_HEADER} ${ZENGINE_XDG_FOREIGN_CODE} + COMMAND ${WAYLAND_SCANNER_EXECUTABLE} client-header ${ZENGINE_XDG_FOREIGN_PROTOCOL} ${ZENGINE_XDG_FOREIGN_HEADER} + COMMAND ${WAYLAND_SCANNER_EXECUTABLE} private-code ${ZENGINE_XDG_FOREIGN_PROTOCOL} ${ZENGINE_XDG_FOREIGN_CODE} + DEPENDS ${ZENGINE_XDG_FOREIGN_PROTOCOL} + VERBATIM + ) + target_sources(zEngineLib PRIVATE + ${ZENGINE_XDG_FOREIGN_HEADER} + ${ZENGINE_XDG_FOREIGN_CODE} + ) + target_include_directories(zEngineLib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_libraries(zEngineLib PRIVATE "-lobjc" "-framework AppKit" "-framework CoreGraphics" "-framework CoreServices") target_compile_definitions(zEngineLib PUBLIC GLFW_EXPOSE_NATIVE_COCOA) diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index d5babcc1..80c103ea 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -21,6 +21,10 @@ extern "C" void ZEngineSetDockIcon(const char* path); #include #include +#if defined(__linux__) +#include +#endif + #ifdef _WIN32 #include @@ -438,66 +442,93 @@ namespace ZEngine::Windows #elif defined(__linux__) { - // Zenity and kdialog block until the picker closes. Run them off - // the UI thread, then resume this coroutine on the main thread. std::vector extensions; extensions.reserve(type_filters.size()); for (std::string_view filter : type_filters) extensions.emplace_back(filter); - std::string default_dir_copy(default_dir); - std::string message_copy(message); - std::future result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy)] { - std::string filter; - for (const std::string& extension : extensions) - { - if (!filter.empty()) - filter += ' '; - filter += '*'; - filter += extension; - } - - const std::string start_dir = default_dir.empty() ? "." : default_dir; - const std::string title = message.empty() ? "Select a file" : message; - - std::string cmd; - if (system("which zenity > /dev/null 2>&1") == 0) - { - cmd = "zenity --file-selection --title='" + title + "'"; - if (!default_dir.empty()) - cmd += " --filename='" + start_dir + "/'"; - if (!filter.empty()) - cmd += " --file-filter='" + filter + "'"; - } - else if (system("which kdialog > /dev/null 2>&1") == 0) - { - cmd = "kdialog --getopenfilename " + start_dir + " '"; + std::string default_dir_copy(default_dir); + std::string message_copy(message); + if (glfwGetPlatform() == GLFW_PLATFORM_WAYLAND) + { + // The exported object remains alive in this coroutine frame until + // the portal response arrives, keeping the dialog parent valid. + auto parent = Platform::CreateWaylandPortalParent(m_native_window); + if (!parent) + ZENGINE_CORE_WARN("[FileDialog] Wayland compositor does not support xdg-foreign; opening the portal without a transient parent") + + const std::string parent_handle = parent ? parent->Handle() : std::string{}; + std::future result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy), parent_handle] { return Platform::OpenWaylandPortalFileDialog(parent_handle, extensions, default_dir, message); }); + path = co_await result; + } + else + { + // Zenity and kdialog block until the picker closes. Run them off + // the UI thread, then resume this coroutine on the main thread. + const unsigned long parent_window = glfwGetX11Window(m_native_window); + std::future result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy), parent_window] { + std::string filter; for (const std::string& extension : extensions) - cmd += '*' + extension + ' '; - if (!cmd.empty() && cmd.back() == ' ') - cmd.pop_back(); - cmd += "'"; - } - - std::string selected_path; - if (cmd.empty()) + { + if (!filter.empty()) + filter += ' '; + filter += '*'; + filter += extension; + } + + const std::string start_dir = default_dir.empty() ? "." : default_dir; + const std::string title = message.empty() ? "Select a file" : message; + + std::string cmd; + // `command` is provided by the POSIX shell used by system(), unlike + // the optional `which` utility. Some minimal Linux installations + // have Zenity but do not ship `which`, which previously made a click + // appear to do nothing. + if (system("command -v zenity >/dev/null 2>&1") == 0) + { + // Zenity needs to use the same X11 server as GLFW for + // --attach to produce a valid transient parent. + cmd = "GDK_BACKEND=x11 zenity --file-selection --modal --title='" + title + "'"; + if (parent_window != 0) + cmd += " --attach=" + std::to_string(parent_window); + if (!default_dir.empty()) + cmd += " --filename='" + start_dir + "/'"; + if (!filter.empty()) + cmd += " --file-filter='" + filter + "'"; + } + else if (system("command -v kdialog >/dev/null 2>&1") == 0) + { + cmd = "kdialog"; + if (parent_window != 0) + cmd += " --attach " + std::to_string(parent_window); + cmd += " --getopenfilename " + start_dir + " '"; + for (const std::string& extension : extensions) + cmd += '*' + extension + ' '; + if (!cmd.empty() && cmd.back() == ' ') + cmd.pop_back(); + cmd += "'"; + } + + std::string selected_path; + if (cmd.empty()) + return selected_path; + + FILE* pipe = popen(cmd.c_str(), "r"); + if (!pipe) + return selected_path; + + char buf[4096] = {}; + if (fgets(buf, sizeof(buf), pipe)) + { + selected_path = buf; + if (!selected_path.empty() && selected_path.back() == '\n') + selected_path.pop_back(); + } + pclose(pipe); return selected_path; - - FILE* pipe = popen(cmd.c_str(), "r"); - if (!pipe) - return selected_path; - - char buf[4096] = {}; - if (fgets(buf, sizeof(buf), pipe)) - { - selected_path = buf; - if (!selected_path.empty() && selected_path.back() == '\n') - selected_path.pop_back(); - } - pclose(pipe); - return selected_path; - }); - path = co_await result; + }); + path = co_await result; + } } #endif diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp new file mode 100644 index 00000000..c54567fb --- /dev/null +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp @@ -0,0 +1,245 @@ +#include +#include + +#if defined(__linux__) + +#include +#include +#include +#include +#include +#include +#include +#include +#include "xdg-foreign-unstable-v2-client-protocol.h" + +namespace ZEngine::Windows::Platform +{ + namespace + { + constexpr char kPortalService[] = "org.freedesktop.portal.Desktop"; + constexpr char kPortalObjectPath[] = "/org/freedesktop/portal/desktop"; + constexpr char kPortalFileChooser[] = "org.freedesktop.portal.FileChooser"; + constexpr char kPortalRequest[] = "org.freedesktop.portal.Request"; + constexpr char kPortalResponse[] = "Response"; + constexpr char kSupportedFilesLabel[] = "Supported files"; + + struct PortalRequest + { + GMainLoop* Loop = nullptr; + std::string Path; + std::string SelectedPath; + }; + + std::atomic_uint64_t s_portal_token = 0; + + void LogError(const char* operation, GError* error) + { + ZENGINE_CORE_ERROR("[FileDialog] {}: {}", operation, error ? error->message : "unknown error") + if (error) + g_error_free(error); + } + + GVariant* BuildPortalOptions(std::span extensions, std::string_view default_directory) + { + GVariantBuilder options; + g_variant_builder_init(&options, G_VARIANT_TYPE("a{sv}")); + + const std::string handle_token = "zengine" + std::to_string(s_portal_token.fetch_add(1, std::memory_order_relaxed)); + g_variant_builder_add(&options, "{sv}", "handle_token", g_variant_new_string(handle_token.c_str())); + g_variant_builder_add(&options, "{sv}", "modal", g_variant_new_boolean(TRUE)); + + if (!extensions.empty()) + { + GVariantBuilder patterns; + g_variant_builder_init(&patterns, G_VARIANT_TYPE("a(us)")); + for (const std::string& extension : extensions) + { + const std::string pattern = "*" + extension; + g_variant_builder_add(&patterns, "(us)", 0u, pattern.c_str()); + } + + GVariantBuilder filters; + g_variant_builder_init(&filters, G_VARIANT_TYPE("a(sa(us))")); + g_variant_builder_add(&filters, "(s@a(us))", kSupportedFilesLabel, g_variant_builder_end(&patterns)); + g_variant_builder_add(&options, "{sv}", "filters", g_variant_builder_end(&filters)); + } + + if (!default_directory.empty()) + { + GVariantBuilder folder; + g_variant_builder_init(&folder, G_VARIANT_TYPE("ay")); + for (const unsigned char character : default_directory) + g_variant_builder_add(&folder, "y", character); + g_variant_builder_add(&folder, "y", 0u); + g_variant_builder_add(&options, "{sv}", "current_folder", g_variant_builder_end(&folder)); + } + + return g_variant_builder_end(&options); + } + + void OnPortalResponse(GDBusConnection*, const gchar*, const gchar* object_path, const gchar*, const gchar*, GVariant* parameters, gpointer user_data) + { + auto* request = static_cast(user_data); + if (!object_path || request->Path != object_path) + return; + + guint32 response = 1; + GVariant* results = nullptr; + g_variant_get(parameters, "(u@a{sv})", &response, &results); + + if (response == 0) + { + GVariant* uris = g_variant_lookup_value(results, "uris", G_VARIANT_TYPE("as")); + if (uris) + { + gsize count = 0; + gchar** values = g_variant_dup_strv(uris, &count); + if (count > 0) + { + gchar* native_path = g_filename_from_uri(values[0], nullptr, nullptr); + if (native_path) + { + request->SelectedPath = native_path; + g_free(native_path); + } + } + g_strfreev(values); + g_variant_unref(uris); + } + } + + g_variant_unref(results); + g_main_loop_quit(request->Loop); + } + } // namespace + + WaylandPortalParent::~WaylandPortalParent() + { + if (m_exported) + zxdg_exported_v2_destroy(m_exported); + if (m_exporter) + zxdg_exporter_v2_destroy(m_exporter); + if (m_registry) + wl_registry_destroy(m_registry); + if (m_event_queue) + wl_event_queue_destroy(m_event_queue); + } + + const std::string& WaylandPortalParent::Handle() const + { + return m_handle; + } + + bool WaylandPortalParent::Initialize(GLFWwindow* window) + { + if (!window || glfwGetPlatform() != GLFW_PLATFORM_WAYLAND) + return false; + + wl_display* display = glfwGetWaylandDisplay(); + wl_surface* surface = glfwGetWaylandWindow(window); + if (!display || !surface) + return false; + + m_event_queue = wl_display_create_queue(display); + if (!m_event_queue) + return false; + + static const wl_registry_listener registry_listener = { + .global = &WaylandPortalParent::OnRegistryGlobal, + .global_remove = nullptr, + }; + static const zxdg_exported_v2_listener exported_listener = { + .handle = &WaylandPortalParent::OnExportedHandle, + }; + + m_registry = wl_display_get_registry(display); + if (!m_registry) + return false; + wl_proxy_set_queue(reinterpret_cast(m_registry), m_event_queue); + wl_registry_add_listener(m_registry, ®istry_listener, this); + + if (wl_display_roundtrip_queue(display, m_event_queue) < 0 || !m_exporter) + return false; + + m_exported = zxdg_exporter_v2_export_toplevel(m_exporter, surface); + if (!m_exported) + return false; + wl_proxy_set_queue(reinterpret_cast(m_exported), m_event_queue); + zxdg_exported_v2_add_listener(m_exported, &exported_listener, this); + + return wl_display_roundtrip_queue(display, m_event_queue) >= 0 && !m_handle.empty(); + } + + void WaylandPortalParent::OnRegistryGlobal(void* context, wl_registry* registry, uint32_t name, const char* interface, uint32_t version) + { + auto* parent = static_cast(context); + if (parent->m_exporter || std::strcmp(interface, zxdg_exporter_v2_interface.name) != 0) + return; + + parent->m_exporter = static_cast(wl_registry_bind(registry, name, &zxdg_exporter_v2_interface, std::min(version, 1u))); + if (parent->m_exporter) + wl_proxy_set_queue(reinterpret_cast(parent->m_exporter), parent->m_event_queue); + } + + void WaylandPortalParent::OnExportedHandle(void* context, zxdg_exported_v2*, const char* handle) + { + if (handle) + static_cast(context)->m_handle = "wayland:" + std::string(handle); + } + + std::unique_ptr CreateWaylandPortalParent(GLFWwindow* window) + { + auto parent = std::unique_ptr(new WaylandPortalParent{}); + if (!parent->Initialize(window)) + return {}; + return parent; + } + + std::string OpenWaylandPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title) + { + GMainContext* context = g_main_context_new(); + g_main_context_push_thread_default(context); + + GError* error = nullptr; + GDBusConnection* connection = g_bus_get_sync(G_BUS_TYPE_SESSION, nullptr, &error); + if (!connection) + { + LogError("Unable to connect to the desktop portal", error); + g_main_context_pop_thread_default(context); + g_main_context_unref(context); + return {}; + } + + PortalRequest request; + request.Loop = g_main_loop_new(context, FALSE); + + const guint subscription = g_dbus_connection_signal_subscribe(connection, kPortalService, kPortalRequest, kPortalResponse, nullptr, nullptr, G_DBUS_SIGNAL_FLAGS_NONE, &OnPortalResponse, &request, nullptr); + + const std::string parent(parent_handle); + const std::string dialog_title(title.empty() ? "Select a file" : title); + GVariant* reply = g_dbus_connection_call_sync(connection, kPortalService, kPortalObjectPath, kPortalFileChooser, "OpenFile", g_variant_new("(ss@a{sv})", parent.c_str(), dialog_title.c_str(), BuildPortalOptions(extensions, default_directory)), G_VARIANT_TYPE("(o)"), G_DBUS_CALL_FLAGS_NONE, -1, nullptr, &error); + + if (!reply) + { + LogError("Unable to open the desktop portal file picker", error); + } + else + { + const gchar* request_path = nullptr; + g_variant_get(reply, "(&o)", &request_path); + request.Path = request_path; + g_variant_unref(reply); + g_main_loop_run(request.Loop); + } + + g_dbus_connection_signal_unsubscribe(connection, subscription); + g_main_loop_unref(request.Loop); + g_object_unref(connection); + g_main_context_pop_thread_default(context); + g_main_context_unref(context); + return request.SelectedPath; + } +} // namespace ZEngine::Windows::Platform + +#endif diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h new file mode 100644 index 00000000..3298c0db --- /dev/null +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h @@ -0,0 +1,54 @@ +#pragma once + +#if defined(__linux__) + +#include +#include +#include +#include + +struct GLFWwindow; +struct wl_event_queue; +struct wl_registry; +struct zxdg_exported_v2; +struct zxdg_exporter_v2; + +namespace ZEngine::Windows::Platform +{ + // Owns the xdg-foreign export for as long as the portal dialog is open. + // Wayland invalidates the parent relation as soon as this object is destroyed. + class WaylandPortalParent + { + public: + ~WaylandPortalParent(); + + WaylandPortalParent(const WaylandPortalParent&) = delete; + WaylandPortalParent& operator=(const WaylandPortalParent&) = delete; + + [[nodiscard]] const std::string& Handle() const; + + private: + WaylandPortalParent() = default; + + bool Initialize(GLFWwindow* window); + + static void OnRegistryGlobal(void* context, wl_registry* registry, uint32_t name, const char* interface, uint32_t version); + static void OnExportedHandle(void* context, zxdg_exported_v2* exported, const char* handle); + + wl_event_queue* m_event_queue = nullptr; + wl_registry* m_registry = nullptr; + zxdg_exporter_v2* m_exporter = nullptr; + zxdg_exported_v2* m_exported = nullptr; + std::string m_handle; + + friend std::unique_ptr CreateWaylandPortalParent(GLFWwindow* window); + }; + + [[nodiscard]] std::unique_ptr CreateWaylandPortalParent(GLFWwindow* window); + + // Blocks the worker thread until the user accepts or cancels the portal. + // The coroutine caller resumes on the engine's main thread afterwards. + [[nodiscard]] std::string OpenWaylandPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title); +} // namespace ZEngine::Windows::Platform + +#endif diff --git a/ZEngine/ZEngine/Windows/Platform/xdg-foreign-unstable-v2.xml b/ZEngine/ZEngine/Windows/Platform/xdg-foreign-unstable-v2.xml new file mode 100644 index 00000000..9e6d6164 --- /dev/null +++ b/ZEngine/ZEngine/Windows/Platform/xdg-foreign-unstable-v2.xml @@ -0,0 +1,45 @@ + + + + Copyright © 2015-2016 Red Hat Inc. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dependencies.cmake b/dependencies.cmake index 1f25e5a5..b916d248 100644 --- a/dependencies.cmake +++ b/dependencies.cmake @@ -1,5 +1,14 @@ include(FetchContent) +# GLFW 3.5 can include both native Linux backends and selects the active one +# at runtime. Clear the pre-3.4 cache option so an existing build directory +# can upgrade without GLFW rejecting its old GLFW_USE_WAYLAND entry. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + unset(GLFW_USE_WAYLAND CACHE) + set(GLFW_BUILD_X11 ON CACHE BOOL "Build GLFW X11 backend" FORCE) + set(GLFW_BUILD_WAYLAND ON CACHE BOOL "Build GLFW Wayland backend" FORCE) +endif() + FetchContent_Declare( fmt GIT_REPOSITORY https://github.com/fmtlib/fmt.git @@ -26,7 +35,7 @@ FetchContent_Declare( glfw3 GIT_REPOSITORY https://github.com/glfw/glfw.git GIT_SHALLOW TRUE - GIT_TAG 3.3.10 + GIT_TAG 3.5.1 ) FetchContent_Declare( From 1aaed136b8c65b34eb92149bcad9cdf0fd405cb0 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 11:27:49 +0900 Subject: [PATCH 03/27] fix(import): publish generated textures through VFS --- ZEngine/ZEngine/Core/VFS/VFSContext.cpp | 10 +++ ZEngine/ZEngine/Importers/AssetCodec.cpp | 49 +++++++--- ZEngine/ZEngine/Importers/AssetCodec.h | 4 + ZEngine/ZEngine/Importers/AssimpImporter.cpp | 94 ++++++-------------- ZEngine/ZEngine/Importers/FbxImporter.cpp | 37 +++++--- ZEngine/ZEngine/Importers/GltfImporter.cpp | 33 +++---- ZEngine/tests/VFS/vfs_meta_test.cpp | 23 ++++- 7 files changed, 139 insertions(+), 111 deletions(-) diff --git a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp index f694c4c6..2269ab37 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp @@ -156,11 +156,21 @@ namespace ZEngine::Core::VFS // Notify AssetRegistry and ImportCoordinator for file (non-directory) events // Skip .meta sidecar files — they are written by the coordinator itself and must // not be fed back into the import queue. + auto is_temporary = [](const VFSPath& p) -> bool { + auto ext = p.Extension(); + return ext.Data && ext.Length == 4 && // ".tmp" + ext.Data[0] == '.' && ext.Data[1] == 't' && ext.Data[2] == 'm' && ext.Data[3] == 'p'; + }; auto is_meta = [](const VFSPath& p) -> bool { auto ext = p.Extension(); return ext.Data && ext.Length == 5 && // ".meta" ext.Data[0] == '.' && ext.Data[1] == 'm' && ext.Data[2] == 'e' && ext.Data[3] == 't' && ext.Data[4] == 'a'; }; + // All generated assets use an atomic .tmp -> rename write. The temporary + // file is not an importable project source and must not enter the import + // queue (or trigger a directory scan) before publication completes. + if (!ev.IsDirectory && !full_rescan && is_temporary(file_path)) + return; if (!ev.IsDirectory && !full_rescan && !is_meta(file_path)) { switch (ev.Kind) diff --git a/ZEngine/ZEngine/Importers/AssetCodec.cpp b/ZEngine/ZEngine/Importers/AssetCodec.cpp index 1955c3ae..9747b75a 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.cpp +++ b/ZEngine/ZEngine/Importers/AssetCodec.cpp @@ -34,33 +34,54 @@ namespace ZEngine::Importers::AssetCodec } } // namespace - // Write data atomically via VFS: open .tmp, write, flush, close, rename to out_path. - static bool WriteVFS(Core::VFS::IVFSContext* vfs, const VFSPath& out_path, const std::string& data) + Core::VFS::VFSResult WriteFileAtomically(Core::VFS::IVFSContext& ctx, const VFSPath& out_path, ArrayView data) { + constexpr char tmp_suffix[] = ".tmp"; + constexpr size_t suffix_len = sizeof(tmp_suffix) - 1; + if (out_path.Length() + suffix_len >= MAX_FILE_PATH_COUNT) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); + char tmp_buf[MAX_FILE_PATH_COUNT] = {}; const char* raw = out_path.CStr(); - size_t raw_len = secure_strlen(raw); - size_t copy_len = raw_len < MAX_FILE_PATH_COUNT - 5 ? raw_len : MAX_FILE_PATH_COUNT - 5; + size_t copy_len = out_path.Length(); secure_memcpy(tmp_buf, MAX_FILE_PATH_COUNT, raw, copy_len); - const char tmp_suffix[] = ".tmp"; secure_memcpy(tmp_buf + copy_len, MAX_FILE_PATH_COUNT - copy_len, tmp_suffix, sizeof(tmp_suffix)); - auto tmp_path = VFSPath::Parse(tmp_buf).Value(); + auto tmp_result = VFSPath::Parse(tmp_buf); + if (tmp_result.Failed()) + return Core::VFS::VFSResult::Fail(tmp_result.Error()); + auto tmp_path = tmp_result.Value(); - auto open_result = vfs->Open(tmp_path, Core::VFS::VFSOpenFlags::Write | Core::VFS::VFSOpenFlags::Create | Core::VFS::VFSOpenFlags::Truncate); + auto open_result = ctx.Open(tmp_path, Core::VFS::VFSOpenFlags::Write | Core::VFS::VFSOpenFlags::Create | Core::VFS::VFSOpenFlags::Truncate); if (open_result.Failed()) - return false; + return Core::VFS::VFSResult::Fail(open_result.Error()); Core::VFS::IVFSFile* file = open_result.Value(); - const auto* bytes = reinterpret_cast(data.data()); - auto w = file->Write({bytes, data.size()}, 0); + auto w = file->Write(data, 0); auto flush = file->Flush(); file->Close(); - vfs->Close(file); + ctx.Close(file); - if (w.Failed() || flush.Failed()) - return false; + auto fail = [&ctx, &tmp_path](Core::VFS::VFSError error) { + ctx.Remove(tmp_path); + return Core::VFS::VFSResult::Fail(error); + }; + if (w.Failed()) + return fail(w.Error()); + if (w.Value() != data.size()) + return fail(Core::VFS::VFSError::IOError); + if (flush.Failed()) + return fail(flush.Error()); + + return ctx.Rename(tmp_path, out_path); + } - return !vfs->Rename(tmp_path, out_path).Failed(); + // Write data atomically via VFS: open .tmp, write, flush, close, rename to out_path. + static bool WriteVFS(Core::VFS::IVFSContext* vfs, const VFSPath& out_path, const std::string& data) + { + if (!vfs) + return false; + const auto* bytes = reinterpret_cast(data.data()); + return WriteFileAtomically(*vfs, out_path, {bytes, data.size()}).Succeeded(); } AssetImporterOutput SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config) diff --git a/ZEngine/ZEngine/Importers/AssetCodec.h b/ZEngine/ZEngine/Importers/AssetCodec.h index c3930c80..4292ef5c 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.h +++ b/ZEngine/ZEngine/Importers/AssetCodec.h @@ -97,6 +97,10 @@ namespace ZEngine::Importers::AssetCodec AssetImporterOutput SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, const ImportConfiguration& config); + /// @brief Atomically publishes a file through the VFS using a temporary sibling and rename. + /// @details Use for generated project assets so file watchers never observe a partially written file. + Core::VFS::VFSResult WriteFileAtomically(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, Core::Containers::ArrayView data); + [[nodiscard]] uint32_t GetEnvironmentMapFullMipCount(uint32_t face_size); [[nodiscard]] bool IsEnvironmentMapFileHeaderValid(const EnvironmentMapFileHeader& header); [[nodiscard]] bool DoesEnvironmentMapHeaderMatchSource(const EnvironmentMapFileHeader& header, uint64_t source_hash); diff --git a/ZEngine/ZEngine/Importers/AssimpImporter.cpp b/ZEngine/ZEngine/Importers/AssimpImporter.cpp index 0f7f6349..60c1f3a3 100644 --- a/ZEngine/ZEngine/Importers/AssimpImporter.cpp +++ b/ZEngine/ZEngine/Importers/AssimpImporter.cpp @@ -11,6 +11,7 @@ #include #include #include +#include using namespace ZEngine::Helpers; using ZEngine::Core::VFS::VFSPath; @@ -583,88 +584,47 @@ namespace ZEngine::Importers } } - void AssimpImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) + void AssimpImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* /*arena*/, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) { - /* - * Normalize file naming - */ - char dst_dir_buf[MAX_FILE_PATH_COUNT] = {}; - (VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()).ResolveNative(config.OutputWorkingSpacePath.c_str(), dst_dir_buf, sizeof(dst_dir_buf)); - std::string dst_dir = dst_dir_buf; - - auto CreateBaseDirectoryFn = [](std::string_view filename) -> void { - auto base_dir = fs::absolute(filename).parent_path(); - - std::error_code err = {}; - if (!fs::exists(base_dir)) - { - fs::create_directories(base_dir, err); - } - }; + if (!config.VFS) + return; - auto scratch = ZGetScratch(arena); - Array src_tex_files = {}; - Array dst_tex_files = {}; - src_tex_files.init(scratch.Arena, textures.size()); - dst_tex_files.init(scratch.Arena, textures.size()); + auto texture_dir = VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str(); + auto create_dir = config.VFS->CreateDir(texture_dir); + if (create_dir.Failed() && create_dir.Error() != Core::VFS::VFSError::AlreadyExists) + return; for (auto& tex : textures) { if (tex.Path.empty()) + continue; + + fs::path tex_path(tex.Path.c_str()); + fs::path src = tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path; + std::ifstream in(src, std::ios::binary | std::ios::ate); + if (!in.is_open()) { + ZENGINE_CORE_WARN("[AssimpImporter] Texture not found: {}", src.string()) continue; } - fs::path tex_path(tex.Path.c_str()); - auto src_file = (tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path).string(); - auto dst_file = (fs::path(dst_dir) / tex_path.filename()).string(); - - CreateBaseDirectoryFn(dst_file); - - auto& sf = src_tex_files.push_use({}); - auto& df = dst_tex_files.push_use({}); - - sf.init(scratch.Arena, src_file.c_str()); - df.init(scratch.Arena, dst_file.c_str()); - } - /* - * Texture files processing - * (1) Ensuring Scene sub-dir is created - * (2) Copying files to destination - */ - - ZENGINE_VALIDATE_ASSERT(src_tex_files.size() == dst_tex_files.size(), "source files count can't be diff of destination files count") - for (int i = 0; i < src_tex_files.size(); ++i) - { - auto src = fs::absolute(src_tex_files[i].c_str()); - auto dst = fs::absolute(dst_tex_files[i].c_str()); - - std::ifstream in(src.c_str(), std::ios::binary); - std::ofstream out(dst.c_str(), std::ios::binary); + const std::streamsize byte_count = in.tellg(); + if (byte_count <= 0) + continue; + std::vector bytes(static_cast(byte_count)); + in.seekg(0, std::ios::beg); + if (!in.read(reinterpret_cast(bytes.data()), byte_count)) + continue; - if (!in.is_open() || !out.is_open()) + auto filename = src.filename().string(); + auto new_path = texture_dir / filename.c_str(); + if (AssetCodec::WriteFileAtomically(*config.VFS, new_path, {bytes.data(), bytes.size()}).Failed()) { - in.close(); - out.close(); + ZENGINE_CORE_WARN("[AssimpImporter] Failed to copy texture: {}", src.string()) continue; } - - out << in.rdbuf(); - - in.close(); - out.close(); - } - - ZReleaseScratch(scratch); - - /* - * Update texture path - */ - for (auto& tex : textures) - { - auto new_path = std::string((VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str() / tex.Path.c_str()).CStr()); tex.Path.clear(); - tex.Path.append(new_path.c_str()); + tex.Path.append(new_path.CStr()); } } diff --git a/ZEngine/ZEngine/Importers/FbxImporter.cpp b/ZEngine/ZEngine/Importers/FbxImporter.cpp index 183c0417..739d1896 100644 --- a/ZEngine/ZEngine/Importers/FbxImporter.cpp +++ b/ZEngine/ZEngine/Importers/FbxImporter.cpp @@ -11,6 +11,7 @@ #include #include #include +#include using ZEngine::Core::VFS::VFSPath; using namespace ZEngine::Core::Containers; @@ -529,15 +530,15 @@ namespace ZEngine::Importers ZReleaseScratch(scratch); } - void FbxImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) + void FbxImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* /*arena*/, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) { - if (textures.empty()) + if (textures.empty() || !config.VFS) return; - char dst_dir_buf[MAX_FILE_PATH_COUNT] = {}; - (VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()).ResolveNative(config.OutputWorkingSpacePath.c_str(), dst_dir_buf, sizeof(dst_dir_buf)); - if (config.VFS) - config.VFS->CreateDir(VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()); + auto texture_dir = VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str(); + auto create_dir = config.VFS->CreateDir(texture_dir); + if (create_dir.Failed() && create_dir.Error() != Core::VFS::VFSError::AlreadyExists) + return; for (auto& tex : textures) { @@ -546,20 +547,30 @@ namespace ZEngine::Importers fs::path tex_path(tex.Path.c_str()); fs::path src = tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path; - fs::path dst = fs::path(dst_dir_buf) / tex_path.filename(); - - std::ifstream in(src, std::ios::binary); + std::ifstream in(src, std::ios::binary | std::ios::ate); if (!in.is_open()) { ZENGINE_CORE_WARN("[FbxImporter] Texture not found: {}", src.string()) continue; } - std::ofstream out(dst, std::ios::binary); - out << in.rdbuf(); - auto new_path = std::string((VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str() / tex_path.filename().string().c_str()).CStr()); + const std::streamsize byte_count = in.tellg(); + if (byte_count <= 0) + continue; + std::vector bytes(static_cast(byte_count)); + in.seekg(0, std::ios::beg); + if (!in.read(reinterpret_cast(bytes.data()), byte_count)) + continue; + + auto filename = tex_path.filename().string(); + auto new_path = texture_dir / filename.c_str(); + if (AssetCodec::WriteFileAtomically(*config.VFS, new_path, {bytes.data(), bytes.size()}).Failed()) + { + ZENGINE_CORE_WARN("[FbxImporter] Failed to copy texture: {}", src.string()) + continue; + } tex.Path.clear(); - tex.Path.append(new_path.c_str()); + tex.Path.append(new_path.CStr()); } } } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/GltfImporter.cpp b/ZEngine/ZEngine/Importers/GltfImporter.cpp index c95c797e..338acff2 100644 --- a/ZEngine/ZEngine/Importers/GltfImporter.cpp +++ b/ZEngine/ZEngine/Importers/GltfImporter.cpp @@ -624,9 +624,14 @@ namespace ZEngine::Importers // Extract texture image bytes to disk and record project-relative paths if (config.Options.ImportTextures && config.Options.ImportMaterials) { - char dest_dir_buf[MAX_FILE_PATH_COUNT] = {}; - (ZEngine::Core::VFS::VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()).ResolveNative(config.OutputWorkingSpacePath.c_str(), dest_dir_buf, sizeof(dest_dir_buf)); - config.VFS->CreateDir(ZEngine::Core::VFS::VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()); + auto texture_dir = ZEngine::Core::VFS::VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str(); + auto create_dir = config.VFS->CreateDir(texture_dir); + if (create_dir.Failed() && create_dir.Error() != ZEngine::Core::VFS::VFSError::AlreadyExists) + { + if (on_error) + on_error(context, "Unable to create the texture output directory"); + return; + } // Pre-resolve base pointers for every buffer — for a GLB all images share // buffer 0 (the binary chunk). Resolving once avoids repeated get_if per texture. @@ -767,19 +772,17 @@ namespace ZEngine::Importers snprintf(stem_buf, sizeof(stem_buf), "tex_%zu", tex_idx); } - char out_path_buf[MAX_FILE_PATH_COUNT] = {}; - snprintf(out_path_buf, sizeof(out_path_buf), "%s/%s%s", dest_dir_buf, stem_buf, ext); - - // fwrite is faster than std::ofstream for plain binary blobs - if (FILE* f = fopen(out_path_buf, "wb")) + char output_name[sizeof(stem_buf) + 5] = {}; + snprintf(output_name, sizeof(output_name), "%s%s", stem_buf, ext); + auto output_path = texture_dir / output_name; + if (AssetCodec::WriteFileAtomically(*config.VFS, output_path, {bytes, nbytes}).Succeeded()) { - fwrite(bytes, 1, nbytes, f); - fclose(f); - ZENGINE_LOG_ASSET_INFO("GltfImporter: extracted texture '{}' ({} bytes)", out_path_buf, nbytes) - - char rel_buf[MAX_FILE_PATH_COUNT] = {}; - snprintf(rel_buf, sizeof(rel_buf), "%s/%s/%s%s", config.OutputTextureFilesPath.c_str(), config.AssetName.c_str(), stem_buf, ext); - textures[tex_idx].Path.init(&scratch, rel_buf); + ZENGINE_LOG_ASSET_INFO("GltfImporter: extracted texture '{}' ({} bytes)", output_path.CStr(), nbytes) + textures[tex_idx].Path.init(&scratch, output_path.CStr()); + } + else + { + ZENGINE_LOG_ASSET_WARN("GltfImporter: unable to write extracted texture '{}'", output_path.CStr()) } } diff --git a/ZEngine/tests/VFS/vfs_meta_test.cpp b/ZEngine/tests/VFS/vfs_meta_test.cpp index a3db3f1c..4565face 100644 --- a/ZEngine/tests/VFS/vfs_meta_test.cpp +++ b/ZEngine/tests/VFS/vfs_meta_test.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -173,7 +174,25 @@ TEST_F(MetaFileIOTest, GetOrCreateChangedHashReturnsStale) EXPECT_EQ(second.Value().AssetUUID, original_uuid); } -// Test 9 — Truncated key/value strings do not overflow buffers +// Generated asset data is published by rename, so watchers never see a partial file. +TEST_F(MetaFileIOTest, AtomicWritePublishesDestinationWithoutTemporaryFile) +{ + constexpr char content[] = "texture-bytes"; + ASSERT_TRUE(ZEngine::Importers::AssetCodec::WriteFileAtomically(m_ctx, P("/generated/diffuse.png"), Bytes(content, sizeof(content) - 1)).Succeeded()); + + EXPECT_TRUE(FileExists("/generated/diffuse.png")); + EXPECT_FALSE(FileExists("/generated/diffuse.png.tmp")); + + auto open = m_ctx.Open(P("/generated/diffuse.png"), VFSOpenFlags::Read); + ASSERT_TRUE(open.Succeeded()); + uint8_t bytes[sizeof(content)] = {}; + ASSERT_TRUE(open.Value()->ReadAll({bytes, sizeof(content) - 1}).Succeeded()); + m_ctx.Close(open.Value()); + + EXPECT_STREQ(reinterpret_cast(bytes), content); +} + +// Test 10 — Truncated key/value strings do not overflow buffers TEST_F(MetaFileIOTest, OverlongSettingsAreTruncated) { char long_key[201]; @@ -196,7 +215,7 @@ TEST_F(MetaFileIOTest, OverlongSettingsAreTruncated) EXPECT_EQ(result.Value().Settings[0].Key[63], '\0'); } -// Test 10 — Settings count is capped at META_MAX_SETTINGS +// Test 11 — Settings count is capped at META_MAX_SETTINGS TEST_F(MetaFileIOTest, SettingsCountCappedAtMax) { char json[8192]; From 8c5e7a4bd16b7e47ba06884ce7d62a5e4841a05b Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 11:45:13 +0900 Subject: [PATCH 04/27] fix(linux): fall back when desktop portal picker fails --- ZEngine/ZEngine/Windows/GameWindow.cpp | 105 ++++++------------ .../Platform/WaylandPortalFileDialog.cpp | 97 +++++++++++++++- .../Platform/WaylandPortalFileDialog.h | 22 +++- 3 files changed, 143 insertions(+), 81 deletions(-) diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index 80c103ea..a5a7a0c3 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -447,87 +448,43 @@ namespace ZEngine::Windows for (std::string_view filter : type_filters) extensions.emplace_back(filter); - std::string default_dir_copy(default_dir); - std::string message_copy(message); - if (glfwGetPlatform() == GLFW_PLATFORM_WAYLAND) + std::string default_dir_copy(default_dir); + std::string message_copy(message); + const int platform = glfwGetPlatform(); + + std::unique_ptr wayland_parent; + std::string portal_parent; + unsigned long x11_parent_window = 0; + if (platform == GLFW_PLATFORM_WAYLAND) { // The exported object remains alive in this coroutine frame until // the portal response arrives, keeping the dialog parent valid. - auto parent = Platform::CreateWaylandPortalParent(m_native_window); - if (!parent) + wayland_parent = Platform::CreateWaylandPortalParent(m_native_window); + if (!wayland_parent) ZENGINE_CORE_WARN("[FileDialog] Wayland compositor does not support xdg-foreign; opening the portal without a transient parent") - - const std::string parent_handle = parent ? parent->Handle() : std::string{}; - std::future result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy), parent_handle] { return Platform::OpenWaylandPortalFileDialog(parent_handle, extensions, default_dir, message); }); - path = co_await result; + else + portal_parent = wayland_parent->Handle(); } + else if (platform == GLFW_PLATFORM_X11) + { + x11_parent_window = glfwGetX11Window(m_native_window); + if (x11_parent_window != 0) + { + char parent_buffer[32] = {}; + std::snprintf(parent_buffer, sizeof(parent_buffer), "x11:0x%lx", x11_parent_window); + portal_parent = parent_buffer; + } + } + + std::future portal_result = std::async(std::launch::async, [extensions, default_dir = default_dir_copy, message = message_copy, portal_parent] { return Platform::OpenPortalFileDialog(portal_parent, extensions, default_dir, message); }); + auto result = co_await portal_result; + if (result.Status != Platform::PortalFileDialogStatus::Failed) + path = std::move(result.Path); else { - // Zenity and kdialog block until the picker closes. Run them off - // the UI thread, then resume this coroutine on the main thread. - const unsigned long parent_window = glfwGetX11Window(m_native_window); - std::future result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy), parent_window] { - std::string filter; - for (const std::string& extension : extensions) - { - if (!filter.empty()) - filter += ' '; - filter += '*'; - filter += extension; - } - - const std::string start_dir = default_dir.empty() ? "." : default_dir; - const std::string title = message.empty() ? "Select a file" : message; - - std::string cmd; - // `command` is provided by the POSIX shell used by system(), unlike - // the optional `which` utility. Some minimal Linux installations - // have Zenity but do not ship `which`, which previously made a click - // appear to do nothing. - if (system("command -v zenity >/dev/null 2>&1") == 0) - { - // Zenity needs to use the same X11 server as GLFW for - // --attach to produce a valid transient parent. - cmd = "GDK_BACKEND=x11 zenity --file-selection --modal --title='" + title + "'"; - if (parent_window != 0) - cmd += " --attach=" + std::to_string(parent_window); - if (!default_dir.empty()) - cmd += " --filename='" + start_dir + "/'"; - if (!filter.empty()) - cmd += " --file-filter='" + filter + "'"; - } - else if (system("command -v kdialog >/dev/null 2>&1") == 0) - { - cmd = "kdialog"; - if (parent_window != 0) - cmd += " --attach " + std::to_string(parent_window); - cmd += " --getopenfilename " + start_dir + " '"; - for (const std::string& extension : extensions) - cmd += '*' + extension + ' '; - if (!cmd.empty() && cmd.back() == ' ') - cmd.pop_back(); - cmd += "'"; - } - - std::string selected_path; - if (cmd.empty()) - return selected_path; - - FILE* pipe = popen(cmd.c_str(), "r"); - if (!pipe) - return selected_path; - - char buf[4096] = {}; - if (fgets(buf, sizeof(buf), pipe)) - { - selected_path = buf; - if (!selected_path.empty() && selected_path.back() == '\n') - selected_path.pop_back(); - } - pclose(pipe); - return selected_path; - }); - path = co_await result; + ZENGINE_CORE_WARN("[FileDialog] Desktop portal failed; trying the Linux fallback picker") + std::future fallback_result = std::async(std::launch::async, [extensions = std::move(extensions), default_dir = std::move(default_dir_copy), message = std::move(message_copy), x11_parent_window, use_x11 = platform == GLFW_PLATFORM_X11] { return Platform::OpenLinuxFallbackFileDialog(x11_parent_window, use_x11, extensions, default_dir, message); }); + path = co_await fallback_result; } } #endif diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp index c54567fb..ed64823b 100644 --- a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp @@ -9,7 +9,10 @@ #include #include #include +#include +#include #include +#include #include #include "xdg-foreign-unstable-v2-client-protocol.h" @@ -29,6 +32,7 @@ namespace ZEngine::Windows::Platform GMainLoop* Loop = nullptr; std::string Path; std::string SelectedPath; + uint32_t Response = 2; }; std::atomic_uint64_t s_portal_token = 0; @@ -87,6 +91,7 @@ namespace ZEngine::Windows::Platform guint32 response = 1; GVariant* results = nullptr; g_variant_get(parameters, "(u@a{sv})", &response, &results); + request->Response = response; if (response == 0) { @@ -112,6 +117,20 @@ namespace ZEngine::Windows::Platform g_variant_unref(results); g_main_loop_quit(request->Loop); } + + std::string ShellQuote(std::string_view value) + { + std::string quoted("'"); + for (char character : value) + { + if (character == '\'') + quoted += "'\"'\"'"; + else + quoted += character; + } + quoted += '\''; + return quoted; + } } // namespace WaylandPortalParent::~WaylandPortalParent() @@ -196,7 +215,7 @@ namespace ZEngine::Windows::Platform return parent; } - std::string OpenWaylandPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title) + PortalFileDialogResult OpenPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title) { GMainContext* context = g_main_context_new(); g_main_context_push_thread_default(context); @@ -228,9 +247,13 @@ namespace ZEngine::Windows::Platform { const gchar* request_path = nullptr; g_variant_get(reply, "(&o)", &request_path); - request.Path = request_path; + if (request_path) + request.Path = request_path; g_variant_unref(reply); - g_main_loop_run(request.Loop); + if (!request.Path.empty()) + g_main_loop_run(request.Loop); + else + ZENGINE_CORE_ERROR("[FileDialog] Desktop portal returned an invalid request path") } g_dbus_connection_signal_unsubscribe(connection, subscription); @@ -238,7 +261,73 @@ namespace ZEngine::Windows::Platform g_object_unref(connection); g_main_context_pop_thread_default(context); g_main_context_unref(context); - return request.SelectedPath; + if (request.Response == 0 && !request.SelectedPath.empty()) + return {PortalFileDialogStatus::Selected, std::move(request.SelectedPath)}; + if (request.Response == 1) + return {PortalFileDialogStatus::Cancelled, {}}; + + ZENGINE_CORE_ERROR("[FileDialog] Desktop portal returned an error response ({})", request.Response) + return {}; + } + + std::string OpenLinuxFallbackFileDialog(unsigned long x11_parent_window, bool use_x11, std::span extensions, std::string_view default_directory, std::string_view title) + { + std::string filter; + for (const std::string& extension : extensions) + { + if (!filter.empty()) + filter += ' '; + filter += '*'; + filter += extension; + } + + const std::string start_dir = default_directory.empty() ? "." : std::string(default_directory); + const std::string dialog_title(title.empty() ? "Select a file" : title); + + std::string command; + if (system("command -v zenity >/dev/null 2>&1") == 0) + { + command = use_x11 ? "GDK_BACKEND=x11 " : ""; + command += "zenity --file-selection --modal --title=" + ShellQuote(dialog_title); + if (use_x11 && x11_parent_window != 0) + command += " --attach=" + std::to_string(x11_parent_window); + if (!default_directory.empty()) + command += " --filename=" + ShellQuote(start_dir + "/"); + if (!filter.empty()) + command += " --file-filter=" + ShellQuote(filter); + } + else if (system("command -v kdialog >/dev/null 2>&1") == 0) + { + command = "kdialog"; + if (use_x11 && x11_parent_window != 0) + command += " --attach " + std::to_string(x11_parent_window); + command += " --getopenfilename " + ShellQuote(start_dir); + if (!filter.empty()) + command += " " + ShellQuote(filter); + } + else + { + ZENGINE_CORE_ERROR("[FileDialog] No Linux picker is available. Install xdg-desktop-portal, zenity, or kdialog") + return {}; + } + + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) + { + ZENGINE_CORE_ERROR("[FileDialog] Unable to launch the fallback picker") + return {}; + } + + char buffer[4096] = {}; + std::string selected_path; + if (fgets(buffer, sizeof(buffer), pipe)) + { + selected_path = buffer; + if (!selected_path.empty() && selected_path.back() == '\n') + selected_path.pop_back(); + } + pclose(pipe); + return selected_path; } } // namespace ZEngine::Windows::Platform diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h index 3298c0db..7be6737e 100644 --- a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h @@ -2,6 +2,7 @@ #if defined(__linux__) +#include #include #include #include @@ -15,6 +16,19 @@ struct zxdg_exporter_v2; namespace ZEngine::Windows::Platform { + enum class PortalFileDialogStatus : uint8_t + { + Selected, + Cancelled, + Failed, + }; + + struct PortalFileDialogResult + { + PortalFileDialogStatus Status = PortalFileDialogStatus::Failed; + std::string Path; + }; + // Owns the xdg-foreign export for as long as the portal dialog is open. // Wayland invalidates the parent relation as soon as this object is destroyed. class WaylandPortalParent @@ -46,9 +60,11 @@ namespace ZEngine::Windows::Platform [[nodiscard]] std::unique_ptr CreateWaylandPortalParent(GLFWwindow* window); - // Blocks the worker thread until the user accepts or cancels the portal. - // The coroutine caller resumes on the engine's main thread afterwards. - [[nodiscard]] std::string OpenWaylandPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title); + // Blocks the worker thread until the user accepts, cancels, or the portal fails. + [[nodiscard]] PortalFileDialogResult OpenPortalFileDialog(std::string_view parent_handle, std::span extensions, std::string_view default_directory, std::string_view title); + + // Used only when the portal service is unavailable or rejects the request. + [[nodiscard]] std::string OpenLinuxFallbackFileDialog(unsigned long x11_parent_window, bool use_x11, std::span extensions, std::string_view default_directory, std::string_view title); } // namespace ZEngine::Windows::Platform #endif From 6128a7b7ae0ed1b5099a840cbe800bf3f4b62860 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 12:08:36 +0900 Subject: [PATCH 05/27] fix(macos): launch editor with linked Vulkan loader --- ZEngine/ZEngine/Windows/GameWindow.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index a5a7a0c3..8e76494d 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -115,7 +115,13 @@ namespace ZEngine::Windows m_property.Title = cfg.Title.c_str(); m_property.VSync = cfg.EnableVsync; - int glfw_init = glfwInit(); +#if defined(__APPLE__) + // GLFW otherwise tries to dlopen the release Vulkan loader name, while + // Debug builds link the debug loader directly. + glfwInitVulkanLoader(vkGetInstanceProcAddr); +#endif + + int glfw_init = glfwInit(); if (glfw_init == GLFW_FALSE) { ZENGINE_CORE_CRITICAL("Unable to initialize glfw..") @@ -138,8 +144,9 @@ namespace ZEngine::Windows } { - const auto* engine_context = Engine::GetContext(); - const auto icon_path = engine_context && engine_context->EngineAssetsNativeRoot ? (std::filesystem::path(engine_context->EngineAssetsNativeRoot) / "Settings/Icons/AppIconBadge.png").string() : std::string{}; + const auto* engine_context = Engine::GetContext(); + const auto icon_path = engine_context && engine_context->EngineAssetsNativeRoot ? (std::filesystem::path(engine_context->EngineAssetsNativeRoot) / "Settings/Icons/AppIconBadge.png").string() : std::string{}; +#if !defined(__APPLE__) int w = 0, h = 0, channels = 0; unsigned char* pixels = stbi_load(icon_path.c_str(), &w, &h, &channels, STBI_rgb_alpha); if (pixels) @@ -148,7 +155,7 @@ namespace ZEngine::Windows glfwSetWindowIcon(m_native_window, 1, &icon); stbi_image_free(pixels); } -#if defined(__APPLE__) +#else ZEngineSetDockIcon(icon_path.c_str()); #endif } From db6866cfe61b71e5147f9576e1dee60a3fcc4bb8 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 12:08:36 +0900 Subject: [PATCH 06/27] refactor(linux): model portal response codes --- .../ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp | 4 ++-- .../ZEngine/Windows/Platform/WaylandPortalFileDialog.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp index ed64823b..69c9849f 100644 --- a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.cpp @@ -261,9 +261,9 @@ namespace ZEngine::Windows::Platform g_object_unref(connection); g_main_context_pop_thread_default(context); g_main_context_unref(context); - if (request.Response == 0 && !request.SelectedPath.empty()) + if (static_cast(request.Response) == PortalFileDialogStatus::Selected && !request.SelectedPath.empty()) return {PortalFileDialogStatus::Selected, std::move(request.SelectedPath)}; - if (request.Response == 1) + if (static_cast(request.Response) == PortalFileDialogStatus::Cancelled) return {PortalFileDialogStatus::Cancelled, {}}; ZENGINE_CORE_ERROR("[FileDialog] Desktop portal returned an error response ({})", request.Response) diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h index 7be6737e..82388478 100644 --- a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h @@ -16,11 +16,11 @@ struct zxdg_exporter_v2; namespace ZEngine::Windows::Platform { - enum class PortalFileDialogStatus : uint8_t + enum class PortalFileDialogStatus : uint32_t { - Selected, - Cancelled, - Failed, + Selected = 0, + Cancelled = 1, + Failed = 2, }; struct PortalFileDialogResult From aa2321aba6cc8d11fb58be9dd5ccc900a175bbbd Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 12:41:27 +0900 Subject: [PATCH 07/27] fix(linux): avoid X11 status macro collision --- ZEngine/ZEngine/Windows/GameWindow.cpp | 2 +- ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index 8e76494d..bbc19ff5 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -485,7 +485,7 @@ namespace ZEngine::Windows std::future portal_result = std::async(std::launch::async, [extensions, default_dir = default_dir_copy, message = message_copy, portal_parent] { return Platform::OpenPortalFileDialog(portal_parent, extensions, default_dir, message); }); auto result = co_await portal_result; - if (result.Status != Platform::PortalFileDialogStatus::Failed) + if (result.Outcome != Platform::PortalFileDialogStatus::Failed) path = std::move(result.Path); else { diff --git a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h index 82388478..15d6bda4 100644 --- a/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h +++ b/ZEngine/ZEngine/Windows/Platform/WaylandPortalFileDialog.h @@ -25,7 +25,7 @@ namespace ZEngine::Windows::Platform struct PortalFileDialogResult { - PortalFileDialogStatus Status = PortalFileDialogStatus::Failed; + PortalFileDialogStatus Outcome = PortalFileDialogStatus::Failed; std::string Path; }; From ed769b1ab5fa4c4fcba309af65a3ae4f240cc67b Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 13:09:32 +0900 Subject: [PATCH 08/27] fix(rendering): serialize streamed geometry uploads --- ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp | 11 +-- .../Rendering/RenderResourceManager.cpp | 77 ++++++++++++++----- .../ZEngine/Rendering/RenderResourceManager.h | 27 +++++-- 3 files changed, 80 insertions(+), 35 deletions(-) diff --git a/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp b/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp index a6388ca6..7950ceab 100644 --- a/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp +++ b/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp @@ -106,13 +106,10 @@ namespace ZEngine::Core::Memory // is this domain's dominant consumer. The generic one-off staging buffers // (AllocateBuffer with domain=HostStaging) share this same pool. // - // blockSize=0 (auto-sized), not a fixed StagingBytes block: a block needs some - // alignment/bookkeeping headroom beyond its raw byte count, so a fixed block - // exactly equal to the ring's own StagingBytes allocation can't actually fit it - // (confirmed: fails with VK_ERROR_OUT_OF_DEVICE_MEMORY). Same underlying reason as - // DeviceTexture's blockSize=0 below, just triggered by exact-equality instead of - // exceeding the block. - create_buffer_pool(GpuMemoryDomain::HostStaging, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VmaAllocationCreateInfo{.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}, 0, 1); + // blockSize=0 and maxBlockCount=0 let VMA add a block for one-off staging + // uploads after the ring's block is full. A one-block pool made every geometry + // staging allocation fall back to VMA's default allocator on discrete GPUs. + create_buffer_pool(GpuMemoryDomain::HostStaging, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VmaAllocationCreateInfo{.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}, 0, 0); // HostReadback has the inverse host-access contract of HostStaging. It is // deliberately a separate pool because readback allocations can stay leased diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index 8bfdb4a9..d790bf73 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -265,6 +265,7 @@ namespace ZEngine::Rendering while (m_pending_texture_decodes.value.load(std::memory_order_acquire) > 0) std::this_thread::yield(); + DiscardQueuedTextureDecodes(); m_device->QueueWaitAll(); ShutdownTextureTimelines(); DiscardTextureDeferrals(); @@ -318,6 +319,7 @@ namespace ZEngine::Rendering { m_active_frame_index = static_cast(frame_index); RetireBatchStagings(); + DispatchQueuedTextureDecodes(); m_streaming_mgr.Tick(frame_index); if (m_streaming_mgr.IsCompactionRequested()) RunCompaction(); @@ -703,17 +705,25 @@ namespace ZEngine::Rendering { m_batch_cmd->End(); - // Submission is deferred to SubmitAsyncUploads (AppRenderPipeline::EndFrame) instead - // of submitted-and-blocked-on here, so a mesh drop never stalls the render thread. + // Submit without blocking so a mesh drop never stalls the render thread. Present() + // consumes the resulting timeline operation in the graphics submission below. // Signals m_batch_timeline — a dedicated semaphore with exactly one writer (this // function) — rather than DeviceSwapchain::RenderTimeline, which Present() also // drives independently. uint64_t signal_value = ++m_batch_next_value; m_batch_frames[m_batch_frame_index].LastSignal = signal_value; - VkPipelineStageFlags2 wait_flag = VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; - m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, wait_flag, signal_value, UINT64_MAX, nullptr); - m_device->EnqueueAsyncGPUOperation({wait_flag, signal_value, m_batch_timeline}); + // The global geometry buffers are bound as whole-buffer storage descriptors. + // A fresh copy therefore needs to wait for the previous render submission before + // writing, even when its byte range is newly allocated: validation cannot infer + // the shader's per-draw subrange, and separate graphics queues may overlap. + auto* const swapchain = m_device->SwapchainPtr; + const uint64_t previous_render = swapchain->RenderTimelineNextValue; + auto* const render_timeline = swapchain->RenderTimeline; + constexpr auto copy_wait_stage = VK_PIPELINE_STAGE_2_TRANSFER_BIT; + constexpr auto consumer_wait_stage = VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT; + m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, copy_wait_stage, signal_value, previous_render, previous_render != 0 ? render_timeline : nullptr); + m_device->EnqueueAsyncGPUOperation({consumer_wait_stage, signal_value, m_batch_timeline}); // Left in m_batch_frames[m_batch_frame_index] for RetireBatchStagings (or the next // BeginBatchUpload for this same frame index) to free once m_batch_timeline proves @@ -1583,6 +1593,8 @@ namespace ZEngine::Rendering if (!ProcessTextureDeferral(frame_index, deferral)) m_tex_deferral_retry.push(deferral); } + + DispatchQueuedTextureDecodes(); } void RenderResourceManager::SubmitAsyncUploads() @@ -1894,38 +1906,63 @@ namespace ZEngine::Rendering if (!tex_handle.Valid()) return {}; - uint8_t decode_slab_index = UINT8_MAX; - if (!TryAcquireTextureDecodeSlab(&decode_slab_index)) - { - if (track_decode) - PublishTextureDecodeCompletion(tex_handle, false); - if (!existing.Valid()) - m_device->DestroyTexture(tex_handle); - ZENGINE_CORE_WARN("[RRM] Texture decode capacity ({}) reached — rejecting {}", MAX_CONCURRENT_TEXTURE_DECODES, filename) - return existing; - } - auto* task = static_cast(m_texture_task_slab.Alloc(sizeof(TextureDecodeTask))); ZConstruct(task, TextureDecodeTask); task->Owner = this; task->Specification = spec; task->Texture = tex_handle; - task->DecodeSlabIndex = decode_slab_index; task->IsEnvironmentMap = is_environment_map; task->TrackCompletion = track_decode; Helpers::secure_strcpy(task->Filename, sizeof(task->Filename), filename); - m_pending_texture_decodes.value.fetch_add(1, std::memory_order_release); - if (!Helpers::ThreadPoolHelper::Submit(task, &RenderResourceManager::RunTextureDecodeTask)) + if (!m_queued_texture_decodes.push(task)) { if (task->TrackCompletion) PublishTextureDecodeCompletion(tex_handle, false); + if (!existing.Valid()) + m_device->DestroyTexture(tex_handle); + m_texture_task_slab.Free(task); + ZENGINE_CORE_ERROR("[RRM] Texture decode queue full — rejecting {}", filename) + return existing; + } + + return tex_handle; + } + + void RenderResourceManager::DispatchQueuedTextureDecodes() + { + for (;;) + { + uint8_t decode_slab_index = UINT8_MAX; + if (!TryAcquireTextureDecodeSlab(&decode_slab_index)) + return; + + TextureDecodeTask* task = nullptr; + if (!m_queued_texture_decodes.pop(task)) + { + ReleaseTextureDecodeSlab(decode_slab_index); + return; + } + + task->DecodeSlabIndex = decode_slab_index; + m_pending_texture_decodes.value.fetch_add(1, std::memory_order_release); + if (Helpers::ThreadPoolHelper::Submit(task, &RenderResourceManager::RunTextureDecodeTask)) + continue; + + if (task->TrackCompletion) + PublishTextureDecodeCompletion(task->Texture, false); ReleaseTextureDecodeSlab(task->DecodeSlabIndex); CompleteTextureDecodeTask(task); ZENGINE_CORE_ERROR("[RRM] Texture decode rejected because the thread pool is shutting down") + return; } + } - return tex_handle; + void RenderResourceManager::DiscardQueuedTextureDecodes() + { + TextureDecodeTask* task = nullptr; + while (m_queued_texture_decodes.pop(task)) + m_texture_task_slab.Free(task); } void RenderResourceManager::RunTextureDecodeTask(void* context) diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.h b/ZEngine/ZEngine/Rendering/RenderResourceManager.h index 18ac14b0..e8d505d3 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.h +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.h @@ -102,7 +102,7 @@ namespace ZEngine::Rendering void BeginFrame(uint32_t frame_index); /// @brief Closes this frame's deferred upload batch, if anything joined it. - /// @details Called by AppRenderPipeline::EndFrame, before SubmitAsyncUploads. + /// @details Called by AppRenderPipeline::EndFrame before Present(). void EndFrame(); /// @brief Ingest a texture file, uploading (existing invalid) or reloading it in @@ -195,8 +195,7 @@ namespace ZEngine::Rendering /// @param frame_index Render frame index to upload against. void CompleteDeferrals(uint8_t frame_index); - /// @brief Submit all pending async upload jobs (texture uploads and mesh batch - /// uploads) to their resolved GPU queues. + /// @brief Submit all pending asynchronous texture uploads to their resolved GPU queues. /// @details Called from AppRenderPipeline::EndFrame. Processes m_async_uploads. void SubmitAsyncUploads(); @@ -464,6 +463,7 @@ namespace ZEngine::Rendering static constexpr uint32_t MAX_UUID_MAP = 4096; static constexpr uint32_t MAX_PENDING = 1024; static constexpr uint32_t MAX_TEXTURE_DEFERRALS = 8192; + static constexpr uint32_t MAX_TEXTURE_TASKS = 4096; // Per-frame-index batch state: the m_batch_timeline value last signalled for that // frame index's batch, and the staging buffers still owned by it. Retired @@ -524,6 +524,8 @@ namespace ZEngine::Rendering Core::Containers::MPSCQueue Completions = {}; }; + using TextureDecodeTaskQueue = Core::Containers::MPSCQueue; + /// @brief Append one mesh asset's vertex/index data to the global buffers. /// @details Shared by DoUploadMesh (new slot) and FlushPendingSwaps (reuse slot). /// Returns a zero-VtxCount MeshSlot on failure. @@ -567,13 +569,18 @@ namespace ZEngine::Rendering /// Called once per frame from BeginFrame. void RetireBatchStagings(); + /// @brief Start queued texture decodes while a bounded decode slab is available. + /// @details Render-thread only. Queued requests preserve their texture handles; + /// they wait here rather than being replaced with the fallback texture. + void DispatchQueuedTextureDecodes(); + void DiscardQueuedTextureDecodes(); + void ResetGeometryBuffersInternal(); /// @brief Re-pack all Resident mesh regions from offset 0, eliminating fragmentation holes. /// @details Resets the pool and re-uploads every Resident mesh's CPU asset data into /// fresh batch regions using the existing AppendMeshData path. Slot regions are - /// updated in place before RenderScene runs. The batch is submitted by - /// EndFrame/SubmitAsyncUploads; Present()'s m_batch_timeline wait ensures the - /// render submission sees the new data before drawing. + /// updated in place before RenderScene runs. EndFrame submits the batch, and + /// Present() waits on m_batch_timeline before drawing the new data. void RunCompaction(); void InitUploadPool(); void InitGlobalBuffers(); @@ -597,8 +604,7 @@ namespace ZEngine::Rendering // Dedicated command buffer manager for geometry/font-atlas uploads, separate from // Device->CommandBufferMgr. Regular pool slot 0 serves the two remaining // synchronous callers (UploadFontAtlas, UpdateBuffer's ring path); the instant - // pool serves BeginBatchUpload/EndBatchUpload, whose submission is deferred to - // SubmitAsyncUploads instead of blocked on. + // pool serves BeginBatchUpload/EndBatchUpload, submitted without a CPU wait. Hardwares::CommandBufferManagerPtr m_upload_cmd_mgr = {}; // The only two remaining synchronous, fence-blocking uploads (UploadFontAtlas and // UpdateBuffer's ring path) are both render-thread-only and always fully block @@ -642,6 +648,11 @@ namespace ZEngine::Rendering PaddedAtomic m_pending_texture_decodes = {}; PaddedAtomic m_accept_texture_decodes = {}; + // SubmitTextureFile can be called by importer threads. The render thread drains + // this queue whenever a decode slab becomes available, preserving backpressure + // without dropping texture requests from large imports. + TextureDecodeTaskQueue m_queued_texture_decodes = {}; + TextureDecodeTracker m_texture_decode_tracker = {}; Rendering::Textures::TextureHandle m_fallback_cubemap = {}; EnvironmentLightingResources m_fallback_environment_lighting = {}; From d61b5cbb91549f238a817df5b1805a6a75b8e562 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 28 Sep 2026 14:55:49 +0900 Subject: [PATCH 09/27] fix(rendering): synchronize geometry uploads with graph work --- ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp | 10 ++++ ZEngine/ZEngine/Hardwares/DeviceSwapchain.h | 6 ++ .../Rendering/RenderResourceManager.cpp | 59 +++++++++++++++---- .../ZEngine/Rendering/RenderResourceManager.h | 11 ++-- .../Rendering/Renderers/RenderGraph.cpp | 29 +++++++++ 5 files changed, 98 insertions(+), 17 deletions(-) diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index a939b60a..b9136157 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -224,6 +224,9 @@ namespace ZEngine::Hardwares void DeviceSwapchain::Dispose() { + DirectGraphicsTimeline = nullptr; + DirectGraphicsTimelineValue = 0; + // Destroy frame-context and swapchain-image Vulkan objects. // All destructors use Device->DeferFree — the second PendingFree.Drain() // at the end of VulkanDevice::Deinitialize() drains them before vkDestroyDevice. @@ -276,6 +279,13 @@ namespace ZEngine::Hardwares } } + // Frame fences do not cover independently submitted uploads. Drain every + // queue before clearing their frame waits or resetting texture timeline + // counters; this is especially important after an OUT_OF_DATE acquire, + // where Present() deliberately submits no fence-backed work. + if (!Device->IsDeviceLost.load(std::memory_order_acquire)) + Device->QueueWaitAll(); + for (int i = 0; i < FrameContextPoolSizeFactor; ++i) { FrameContexts[i + FrameContextOffset].Acquired->SetState(Primitives::SemaphoreState::Idle); diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h index 91595ec8..9bf9ef78 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h @@ -78,6 +78,12 @@ namespace ZEngine::Hardwares VkSwapchainKHR SwapchainHandle = VK_NULL_HANDLE; FrameContextPtr CurrentFrame = nullptr; Rendering::Primitives::Semaphore* RenderTimeline = nullptr; + // Latest graphics work submitted directly by RenderGraph rather than through + // Present(). A later upload must wait on this semaphore directly: relaying it + // through RenderTimeline preserves execution order but loses the draw access + // scope needed for a subsequent buffer write. + Rendering::Primitives::Semaphore* DirectGraphicsTimeline = nullptr; + uint64_t DirectGraphicsTimelineValue = 0; Rendering::Renderers::RenderPasses::Attachment* SwapchainAttachment = nullptr; Core::Containers::Array FrameContexts = {}; Core::Containers::Array SwapchainImages = {}; diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index d790bf73..ff061ff4 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -690,11 +690,8 @@ namespace ZEngine::Rendering if (frame.LastSignal != 0) m_batch_timeline->Wait(frame.LastSignal, UINT64_MAX); - // RetireBatchStagings (called earlier in BeginFrame) handles staging cleanup via - // the RenderTimeline gate — it always runs before this point. If stagings remain - // here it means RetireBatchStagings hasn't confirmed GPU completion yet (render - // timeline hasn't reached SafeRetireAfterRenderValue). Leave them for the next poll - // rather than freeing while the command buffer may still be tracked as in-use. + // RetireBatchStagings polls the batch timeline each frame. Any staging buffers + // still owned by this frame index are released only after its prior batch signal. m_batch_cmd->ResetState(); vkResetCommandBuffer(m_batch_cmd->GetHandle(), 0); m_batch_cmd->Begin(); @@ -714,15 +711,45 @@ namespace ZEngine::Rendering m_batch_frames[m_batch_frame_index].LastSignal = signal_value; // The global geometry buffers are bound as whole-buffer storage descriptors. - // A fresh copy therefore needs to wait for the previous render submission before + // A fresh copy therefore needs to wait for every prior graphics read before // writing, even when its byte range is newly allocated: validation cannot infer // the shader's per-draw subrange, and separate graphics queues may overlap. - auto* const swapchain = m_device->SwapchainPtr; - const uint64_t previous_render = swapchain->RenderTimelineNextValue; - auto* const render_timeline = swapchain->RenderTimeline; - constexpr auto copy_wait_stage = VK_PIPELINE_STAGE_2_TRANSFER_BIT; - constexpr auto consumer_wait_stage = VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT; - m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, copy_wait_stage, signal_value, previous_render, previous_render != 0 ? render_timeline : nullptr); + // + // RenderTimeline covers the previous frame's final Present() submission. Earlier + // graphics batches are submitted directly by RenderGraph, however. Relaying those + // batches through RenderTimeline preserves execution order but not their shader + // access scope, so the copy must wait on their direct timeline as well. + auto* const swapchain = m_device->SwapchainPtr; + constexpr auto copy_wait_stage = VK_PIPELINE_STAGE_2_TRANSFER_BIT; + // A batch can contain generic UpdateBuffer copies as well as mesh data. Its + // consumer queue is therefore not known here: it can be graphics or a + // dedicated compute queue. ALL_COMMANDS is valid for either queue and keeps + // every possible consumer behind the copy. + constexpr auto consumer_wait_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; + + VkSemaphoreSubmitInfo copy_waits[2] = {}; + uint32_t copy_wait_count = 0; + if (swapchain->RenderTimelineNextValue != 0) + { + copy_waits[copy_wait_count++] = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = swapchain->RenderTimeline->GetHandle(), + .value = swapchain->RenderTimelineNextValue, + .stageMask = copy_wait_stage, + }; + } + + if (swapchain->DirectGraphicsTimeline && swapchain->DirectGraphicsTimelineValue != 0) + { + copy_waits[copy_wait_count++] = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = swapchain->DirectGraphicsTimeline->GetHandle(), + .value = swapchain->DirectGraphicsTimelineValue, + .stageMask = copy_wait_stage, + }; + } + + m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, signal_value, copy_waits, copy_wait_count); m_device->EnqueueAsyncGPUOperation({consumer_wait_stage, signal_value, m_batch_timeline}); // Left in m_batch_frames[m_batch_frame_index] for RetireBatchStagings (or the next @@ -1700,6 +1727,14 @@ namespace ZEngine::Rendering { m_async_uploads.Clear(); m_device->AsyncGPUOperations.clear(); + + // Geometry batches are submitted immediately, unlike the cancellable + // texture jobs above. Keep their semaphore dependency across recreation: + // an OUT_OF_DATE acquire has no Present() submission to relay it to the + // next frame's graph. + if (m_batch_timeline && m_batch_next_value != 0) + m_device->EnqueueAsyncGPUOperation({VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, m_batch_next_value, m_batch_timeline}); + m_streaming_upload_tickets.clear(); } diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.h b/ZEngine/ZEngine/Rendering/RenderResourceManager.h index e8d505d3..87bcc2a1 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.h +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.h @@ -101,8 +101,9 @@ namespace ZEngine::Rendering /// @param frame_index Current swapchain frame index (0 .. FRAMES_IN_FLIGHT-1). void BeginFrame(uint32_t frame_index); - /// @brief Closes this frame's deferred upload batch, if anything joined it. - /// @details Called by AppRenderPipeline::EndFrame before Present(). + /// @brief Closes the current deferred upload batch, if anything joined it. + /// @details Called by RenderGraph after its declaration phase and by + /// AppRenderPipeline::EndFrame as an idempotent fallback. void EndFrame(); /// @brief Ingest a texture file, uploading (existing invalid) or reloading it in @@ -219,8 +220,9 @@ namespace ZEngine::Rendering /// @param thread_index Thread index within the per-frame pool. void RetireTextureSlots(uint8_t frame_index, uint8_t thread_index); - /// @brief Cancel all queued async upload jobs without submitting them. - /// @details Called on swapchain resize or recreate to discard stale uploads. + /// @brief Cancel queued texture upload jobs without submitting them. + /// @details Called on swapchain recreation. Already-submitted geometry batches + /// remain published through their timeline semaphore. void ClearAsyncUploads(); /// @brief Reset all texture timeline semaphore counters after a swapchain recreate. @@ -473,7 +475,6 @@ namespace ZEngine::Rendering struct BatchFrameState { uint64_t LastSignal = 0; - uint64_t SafeRetireAfterRenderValue = 0; // RenderTimeline value after which stagings are safe to free Core::Memory::BufferView StagingBuffers[MAX_PENDING * 2] = {}; uint32_t StagingCount = 0; }; diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp index 8ab7b17d..d8468b5f 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp @@ -1273,6 +1273,11 @@ namespace ZEngine::Rendering::Renderers SceneData = data; RenderWidth = Device && Device->SwapchainPtr ? Device->SwapchainPtr->SwapchainImageWidth : 0; RenderHeight = Device && Device->SwapchainPtr ? Device->SwapchainPtr->SwapchainImageHeight : 0; + if (Device && Device->SwapchainPtr) + { + Device->SwapchainPtr->DirectGraphicsTimeline = nullptr; + Device->SwapchainPtr->DirectGraphicsTimelineValue = 0; + } // RenderGraph has a bounded number of virtual passes/resources per frame. // Persistent pipelines, framebuffers, and transient images stay in the @@ -1606,6 +1611,18 @@ namespace ZEngine::Rendering::Renderers { Register({.Scene = SceneData, .FrameIndex = static_cast(Device->SwapchainPtr->CurrentFrame->Index), .RenderWidth = RenderWidth, .RenderHeight = RenderHeight}); Compile(); + + // Register() can append deferred copies (notably the UI upload), and scene + // uploads were already recorded before Execute(). Close that single batch now: + // immediate graph batches below must wait for it just like the Present-owned + // final batch. Closing later would let direct graphics read new mesh data before + // the copy submission is visible. + if (Device->RRM) + { + static_cast(Device->RRM)->EndFrame(); + Device->SwapchainPtr->CollectAsyncGPUOperations(); + } + if (!m_compile_valid) { CancelUnsubmittedReadbacks(); @@ -2269,6 +2286,11 @@ namespace ZEngine::Rendering::Renderers batch_timelines[batch_index] = timeline; batch_signal_values[batch_index] = signal_value; + if (batch.Queue == Rendering::QueueType::GRAPHIC_QUEUE) + { + Device->SwapchainPtr->DirectGraphicsTimeline = timeline; + Device->SwapchainPtr->DirectGraphicsTimelineValue = signal_value; + } AcknowledgeStreamingAcquires(batch.FirstPassOrder, batch.PassCount); SubmitReadbacks(batch.FirstPassOrder, batch.PassCount, timeline, signal_value); // RenderGraph and DeviceSwapchain run on the render thread. Appending @@ -2377,6 +2399,13 @@ namespace ZEngine::Rendering::Renderers void RenderGraph::Dispose() { + constexpr uint32_t graphics_timeline_index = static_cast(Rendering::QueueType::GRAPHIC_QUEUE); + if (Device && Device->SwapchainPtr && Device->SwapchainPtr->DirectGraphicsTimeline == QueueTimelines[graphics_timeline_index]) + { + Device->SwapchainPtr->DirectGraphicsTimeline = nullptr; + Device->SwapchainPtr->DirectGraphicsTimelineValue = 0; + } + ReadbackRing.Dispose(); DisposeQueryPools(); DisposeTimestampFrames(); From a2fdafdbedc950a632b780d9d3851ede31c49e86 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Tue, 29 Sep 2026 00:53:47 +0900 Subject: [PATCH 10/27] fix(linux): skip unsupported Wayland window icon --- ZEngine/ZEngine/Windows/GameWindow.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index bbc19ff5..11ec8a49 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -147,13 +147,21 @@ namespace ZEngine::Windows const auto* engine_context = Engine::GetContext(); const auto icon_path = engine_context && engine_context->EngineAssetsNativeRoot ? (std::filesystem::path(engine_context->EngineAssetsNativeRoot) / "Settings/Icons/AppIconBadge.png").string() : std::string{}; #if !defined(__APPLE__) - int w = 0, h = 0, channels = 0; - unsigned char* pixels = stbi_load(icon_path.c_str(), &w, &h, &channels, STBI_rgb_alpha); - if (pixels) +#if defined(__linux__) + // GLFW reports this as an error on Wayland, and this application's GLFW + // error callback is intentionally fatal. Wayland has no window-icon API; + // the compositor selects the application icon instead. + if (glfwGetPlatform() != GLFW_PLATFORM_WAYLAND) +#endif { - GLFWimage icon{w, h, pixels}; - glfwSetWindowIcon(m_native_window, 1, &icon); - stbi_image_free(pixels); + int w = 0, h = 0, channels = 0; + unsigned char* pixels = stbi_load(icon_path.c_str(), &w, &h, &channels, STBI_rgb_alpha); + if (pixels) + { + GLFWimage icon{w, h, pixels}; + glfwSetWindowIcon(m_native_window, 1, &icon); + stbi_image_free(pixels); + } } #else ZEngineSetDockIcon(icon_path.c_str()); From 0b715fef3207be8322134355a9b16780d15b1b64 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Tue, 29 Sep 2026 07:45:43 +0900 Subject: [PATCH 11/27] fix(vulkan): serialize universal queue families --- ZEngine/ZEngine/Hardwares/VulkanDevice.cpp | 81 +++++++++++++--------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp index e8f33d60..5a1512cc 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp @@ -517,38 +517,32 @@ namespace ZEngine::Hardwares if (ComputeFamilyIndex == invalid_queue_family) ComputeFamilyIndex = GraphicFamilyIndex; - HasSeperateTransfertQueueFamily = GraphicFamilyIndex != TransferFamilyIndex; - HasSeparateComputeQueueFamily = GraphicFamilyIndex != ComputeFamilyIndex; - HasSeparateTransferQueue = HasSeperateTransfertQueueFamily; - HasSeparateComputeQueue = HasSeparateComputeQueueFamily; - GraphicQueueIndex = 0; - TransferQueueIndex = 0; - ComputeQueueIndex = 0; - - // A family can expose multiple distinct queue handles. Prefer a dedicated - // family, then consume additional graphics-family queues for compute and - // transfer work without introducing a queue-family ownership transfer. - uint32_t next_graphics_queue_index = 1; - const uint32_t graphics_queue_count = physical_device_queue_family_collection[GraphicFamilyIndex].queueCount; - if (!HasSeparateComputeQueue && ComputeFamilyIndex == GraphicFamilyIndex && next_graphics_queue_index < graphics_queue_count) - { - ComputeQueueIndex = next_graphics_queue_index++; - HasSeparateComputeQueue = true; - } - if (!HasSeparateTransferQueue && TransferFamilyIndex == GraphicFamilyIndex && next_graphics_queue_index < graphics_queue_count) - { - TransferQueueIndex = next_graphics_queue_index++; - HasSeparateTransferQueue = true; + GraphicQueueIndex = 0; + TransferQueueIndex = 0; + ComputeQueueIndex = 0; + + // Do not turn additional handles from the graphics/present family into + // asynchronous lanes. They have no dedicated capabilities, and treating them + // as separate queues complicates graph synchronization without proving that + // the hardware can overlap their work. A universal graphics/compute/transfer + // family therefore always uses one graphics queue. + // + // Compute and transfer can share one non-graphics family. Use two handles + // only when the family actually exposes two queues. With one handle, retain + // the compute lane and route transfer work through graphics instead of + // modelling one VkQueue as two independent scheduler lanes. + if (ComputeFamilyIndex == TransferFamilyIndex && ComputeFamilyIndex != GraphicFamilyIndex) + { + if (physical_device_queue_family_collection[ComputeFamilyIndex].queueCount > 1) + TransferQueueIndex = 1; + else + TransferFamilyIndex = GraphicFamilyIndex; } - // Some devices expose one non-graphics family for both compute and transfer. - // Request separate handles when that family has enough queues; otherwise both - // roles intentionally serialize on its single queue. - if (ComputeFamilyIndex == TransferFamilyIndex && ComputeFamilyIndex != GraphicFamilyIndex && physical_device_queue_family_collection[ComputeFamilyIndex].queueCount > 1) - { - ComputeQueueIndex = 0; - TransferQueueIndex = 1; - } + HasSeperateTransfertQueueFamily = GraphicFamilyIndex != TransferFamilyIndex; + HasSeparateComputeQueueFamily = GraphicFamilyIndex != ComputeFamilyIndex; + HasSeparateTransferQueue = HasSeperateTransfertQueueFamily; + HasSeparateComputeQueue = HasSeparateComputeQueueFamily; QueueTimestampValidBits[static_cast(QueueType::GRAPHIC_QUEUE)] = physical_device_queue_family_collection[GraphicFamilyIndex].timestampValidBits; QueueTimestampValidBits[static_cast(QueueType::TRANSFER_QUEUE)] = physical_device_queue_family_collection[TransferFamilyIndex].timestampValidBits; @@ -716,6 +710,8 @@ namespace ZEngine::Hardwares m_queue_map.insert(Rendering::QueueType::COMPUTE_QUEUE, std::move(compute_queue)); } + ZENGINE_CORE_INFO("[GPU] Queue topology: graphics={}:{}; compute={}:{} (async={}); transfer={}:{} (async={})", GraphicFamilyIndex, GraphicQueueIndex, ComputeFamilyIndex, ComputeQueueIndex, HasSeparateComputeQueue, TransferFamilyIndex, TransferQueueIndex, HasSeparateTransferQueue) + /* Surface format selection */ uint32_t format_count = 0; Array surface_formats = {}; @@ -1266,9 +1262,28 @@ namespace ZEngine::Hardwares void VulkanDevice::QueueWaitAll() { - QueueWait(Rendering::QueueType::TRANSFER_QUEUE); - QueueWait(Rendering::QueueType::COMPUTE_QUEUE); - QueueWait(Rendering::QueueType::GRAPHIC_QUEUE); + constexpr QueueType queue_types[] = {QueueType::TRANSFER_QUEUE, QueueType::COMPUTE_QUEUE, QueueType::GRAPHIC_QUEUE}; + VkQueue waited_queues[3] = {}; + uint32_t waited_count = 0; + + for (QueueType type : queue_types) + { + const VkQueue queue = GetQueue(type).Handle; + bool already_waited = false; + for (uint32_t index = 0; index < waited_count; ++index) + { + if (waited_queues[index] == queue) + { + already_waited = true; + break; + } + } + if (already_waited) + continue; + + ZENGINE_VALIDATE_ASSERT(vkQueueWaitIdle(queue) == VK_SUCCESS, "Failed to wait on queue") + waited_queues[waited_count++] = queue; + } } void VulkanDevice::BeginDebugLabel(VkCommandBuffer command_buffer, cstring name) const From f7b73a7491f47992c9b5f0d8886c20b08fe00ce6 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Tue, 29 Sep 2026 08:32:56 +0900 Subject: [PATCH 12/27] chore(vulkan): remove obsolete queue state --- .../Hardwares/CommandBufferManager.cpp | 58 ------------------- .../ZEngine/Hardwares/CommandBufferManager.h | 1 - ZEngine/ZEngine/Hardwares/VulkanDevice.cpp | 3 +- ZEngine/ZEngine/Hardwares/VulkanDevice.h | 1 - 4 files changed, 1 insertion(+), 62 deletions(-) diff --git a/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp b/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp index bf5123eb..be055f15 100644 --- a/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp +++ b/ZEngine/ZEngine/Hardwares/CommandBufferManager.cpp @@ -389,64 +389,6 @@ namespace ZEngine::Hardwares ZENGINE_CORE_ERROR("[!] Enqueued Command Buffer overflow detected") } - void CommandBufferManager::IncreaseBuffers() - { - // TotalPoolCount = Device->SwapchainImageCount * TotalThreadCount; - // TotalCommandBufferCount = TotalPoolCount * MaxBufferPerPool; - - // if (TotalCommandBufferCount > EnqueuedCommandBuffers.size()) - // { - // auto size = EnqueuedCommandBuffers.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // EnqueuedCommandBuffers.push(nullptr); - // } - // } - - // if (TotalPoolCount > CommandPools.size()) - // { - // auto size = CommandPools.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // CommandPools.push(ZPushStructCtorArgs(Device->Arena, Rendering::Pools::CommandPool, Device, Rendering::QueueType::GRAPHIC_QUEUE)); - // } - // } - - // if (TotalCommandBufferCount > CommandBuffers.size()) - // { - // auto size = CommandBuffers.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // int pool_index = GetPoolFromIndex(Rendering::QueueType::GRAPHIC_QUEUE, i); - // auto& pool = CommandPools[pool_index]; - // CommandBuffers.push(ZPushStructCtorArgs( - // Device->Arena, - // CommandBuffer, - // Device, - // pool->Handle, - // pool->QueueType, - // /*(i % MaxBufferPerPool) == 0 ? false : true */ false)); - // } - // } - - // if (Device->HasSeperateTransfertQueueFamily) - // { - // auto size = TransferCommandPools.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // TransferCommandPools.push(ZPushStructCtorArgs(Device->Arena, Rendering::Pools::CommandPool, Device, Rendering::QueueType::TRANSFER_QUEUE)); - // } - - // size = TransferCommandBuffers.size(); - // for (uint32_t i = size; i < TotalCommandBufferCount; ++i) - // { - // int pool_index = GetPoolFromIndex(Rendering::QueueType::TRANSFER_QUEUE, i); - // auto& pool = TransferCommandPools[pool_index]; - // TransferCommandBuffers.push(ZPushStructCtorArgs(Device->Arena, CommandBuffer, Device, pool->Handle, pool->QueueType, true)); - // } - // } - } - bool CommandBufferManager::IsInitialized() const { return m_is_initialized; diff --git a/ZEngine/ZEngine/Hardwares/CommandBufferManager.h b/ZEngine/ZEngine/Hardwares/CommandBufferManager.h index 6116fe98..88cd5a75 100644 --- a/ZEngine/ZEngine/Hardwares/CommandBufferManager.h +++ b/ZEngine/ZEngine/Hardwares/CommandBufferManager.h @@ -42,7 +42,6 @@ namespace ZEngine::Hardwares /// @brief Returns the instant-submit command pool for a frame, thread, and queue role. Rendering::Pools::CommandPool* GetInstantCommandPool(Rendering::QueueType type, uint8_t frame_index, uint8_t thread_index); void ResetPool(uint8_t frame_index, uint8_t thread_index); - void IncreaseBuffers(); void EnqueueBuffer(CommandBufferPtr const buffer); void EndEnqueuedBuffers(); void ResetEnqueuedBufferIndex(); diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp index 5a1512cc..a74870fd 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp @@ -540,9 +540,8 @@ namespace ZEngine::Hardwares } HasSeperateTransfertQueueFamily = GraphicFamilyIndex != TransferFamilyIndex; - HasSeparateComputeQueueFamily = GraphicFamilyIndex != ComputeFamilyIndex; HasSeparateTransferQueue = HasSeperateTransfertQueueFamily; - HasSeparateComputeQueue = HasSeparateComputeQueueFamily; + HasSeparateComputeQueue = GraphicFamilyIndex != ComputeFamilyIndex; QueueTimestampValidBits[static_cast(QueueType::GRAPHIC_QUEUE)] = physical_device_queue_family_collection[GraphicFamilyIndex].timestampValidBits; QueueTimestampValidBits[static_cast(QueueType::TRANSFER_QUEUE)] = physical_device_queue_family_collection[TransferFamilyIndex].timestampValidBits; diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.h b/ZEngine/ZEngine/Hardwares/VulkanDevice.h index b4aeda7c..74a96271 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.h +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.h @@ -294,7 +294,6 @@ namespace ZEngine::Hardwares struct VulkanDevice { bool HasSeperateTransfertQueueFamily = false; - bool HasSeparateComputeQueueFamily = false; /// @brief True when transfer work has a distinct VkQueue handle. bool HasSeparateTransferQueue = false; /// @brief True when compute work has a distinct VkQueue handle. From 37e973d311e1967375f1529acc634c0858d867d4 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Tue, 29 Sep 2026 08:32:59 +0900 Subject: [PATCH 13/27] fix(vfs): retain FSEvents run loop until shutdown --- ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm b/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm index f3a58013..1ba1079f 100644 --- a/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm +++ b/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm @@ -264,6 +264,11 @@ { std::lock_guard lock(m_watch_mutex); m_run_loop = CFRunLoopGetCurrent(); + // The final watch can be removed while the thread is running, + // leaving the run loop with no sources. CFRunLoopRun then returns + // before StopThread joins us, so keep an owned reference until the + // joining thread has finished using it. + CFRetain(m_run_loop); RebuildStream(); } // Signal that FSEventStreamStart has been called and the run loop @@ -299,6 +304,8 @@ m_thread.join(); } std::lock_guard lock(m_watch_mutex); + if (m_run_loop) + CFRelease(m_run_loop); m_run_loop = nullptr; } From ef53662602cc03cf36c1db83043a0d6656c59ecc Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Wed, 30 Sep 2026 21:46:44 +0900 Subject: [PATCH 14/27] chore(vulkan): trace atmosphere submissions --- .../Applications/AppRenderPipeline.cpp | 1 + ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp | 51 ++++++++++++- ZEngine/ZEngine/Hardwares/DeviceSwapchain.h | 4 + .../Rendering/Renderers/RenderGraph.cpp | 75 ++++++++++++++++++- 4 files changed, 127 insertions(+), 4 deletions(-) diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp index 866ad1e3..be2e0e28 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp @@ -136,6 +136,7 @@ namespace ZEngine::Applications if (Device->RRM) static_cast(Device->RRM)->BeginFrame(swapchain->CurrentFrame->Index); swapchain->FrameAsyncOperations.clear(); + swapchain->TraceSubmission = false; swapchain->CollectAsyncGPUOperations(); Managers::AssetManager::FlushTextureReleases(); diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index b9136157..6430e65a 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -255,6 +255,20 @@ namespace ZEngine::Hardwares void DeviceSwapchain::AcquireNextImage(uint32_t frame_context_idx) { +#ifndef NDEBUG + auto trace_fence_wait = [this](cstring reason, const FrameContext& frame) { + if (!TraceSubmission) + return; + + uint64_t render_completed = 0; + vkGetSemaphoreCounterValue(Device->LogicalDevice, RenderTimeline->GetHandle(), &render_completed); + uint64_t direct_completed = 0; + if (DirectGraphicsTimeline) + vkGetSemaphoreCounterValue(Device->LogicalDevice, DirectGraphicsTimeline->GetHandle(), &direct_completed); + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire wait={} frame={} render_completed={} render_submitted={} direct_timeline={} direct_completed={} direct_submitted={}", reason, frame.Index, render_completed, RenderTimelineNextValue, static_cast(DirectGraphicsTimeline ? DirectGraphicsTimeline->GetHandle() : VK_NULL_HANDLE), direct_completed, DirectGraphicsTimelineValue) + ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); + }; +#endif if (Recreation != RecreationState::None) { // Every submitted render command buffer is covered by its frame fence. @@ -335,7 +349,12 @@ namespace ZEngine::Hardwares FrameContext& frame = FrameContexts[frame_context_idx + FrameContextOffset]; if (frame.Fence->GetState() == Rendering::Primitives::FenceState::Submitted) + { +#ifndef NDEBUG + trace_fence_wait("frame", frame); +#endif frame.Fence->Wait(UINT64_MAX); + } frame.Fence->Reset(); frame.Acquired->SetState(Primitives::SemaphoreState::Idle); @@ -362,7 +381,12 @@ namespace ZEngine::Hardwares } if (ImageInFlights[image_idx] != nullptr && !ImageInFlights[image_idx]->IsSignaled()) + { +#ifndef NDEBUG + trace_fence_wait("image", frame); +#endif ImageInFlights[image_idx]->Wait(UINT64_MAX); + } RenderCompletes[image_idx]->SetState(Rendering::Primitives::SemaphoreState::Idle); @@ -514,7 +538,24 @@ namespace ZEngine::Hardwares }); } - uint64_t work_complete_value = ++RenderTimelineNextValue; + uint64_t work_complete_value = ++RenderTimelineNextValue; + +#ifndef NDEBUG + if (TraceSubmission) + { + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame={} image={} command_buffers={} waits={} signal_render_value={}", CurrentFrame->Index, CurrentFrame->ImageIndex, cmd_infos.size(), wait_sem_infos.size(), work_complete_value) + for (uint32_t wait_index = 0; wait_index < wait_sem_infos.size(); ++wait_index) + { + const VkSemaphoreSubmitInfo& wait = wait_sem_infos[wait_index]; + uint64_t completed = 0; + const VkResult result = wait.value == 0 ? VK_SUCCESS : vkGetSemaphoreCounterValue(Device->LogicalDevice, wait.semaphore, &completed); + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present wait={} semaphore={} value={} completed={} query_result={} stages={}", wait_index, static_cast(wait.semaphore), wait.value, completed, static_cast(result), static_cast(wait.stageMask)) + } + // The normal file sink is asynchronous. Persist the synchronous ring + // buffer before submitting, so a GPU stall cannot lose this frame. + ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); + } +#endif VkSemaphoreSubmitInfo work_complete_signal = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, @@ -555,6 +596,14 @@ namespace ZEngine::Hardwares } ZENGINE_VALIDATE_ASSERT(submit == VK_SUCCESS, "Failed to submit queue") +#ifndef NDEBUG + if (TraceSubmission) + { + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present accepted frame={} render_value={}", CurrentFrame->Index, work_complete_value) + ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); + } +#endif + // The graphics command buffers are now owned by Vulkan. Deliver callbacks // before presentation: a later WSI error cannot undo this submission. for (const RenderWorkSubmissionCallback& callback : RenderWorkSubmittedCallbacks) diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h index 9bf9ef78..c0be3228 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h @@ -99,6 +99,10 @@ namespace ZEngine::Hardwares // vkQueueSubmit2 has accepted this frame's graphics command buffers. Core::Containers::Array RenderWorkSubmittedCallbacks = {}; + // Debug-only, opt-in render-graph diagnostic state. It is reset at the + // beginning of every frame and set only for a traced graph submission. + bool TraceSubmission = false; + // Returns false when the frame was aborted (OUT_OF_DATE at acquire or // zero-size surface). Callers must skip all rendering work for that frame. bool IsFrameValid() const diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp index d8468b5f..f6eeaa6a 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include using namespace ZEngine::Core::Containers; using namespace ZEngine::Helpers; @@ -22,6 +24,19 @@ namespace ZEngine::Rendering::Renderers namespace { +#ifndef NDEBUG + bool IsAtmosphereSubmissionTraceEnabled() + { + const char* const value = std::getenv("ZENGINE_TRACE_ATMOSPHERE_SUBMISSIONS"); + return value && value[0] == '1' && value[1] == '\0'; + } +#endif + + bool IsAtmosphereViewPass(cstring name) + { + return name && (std::strcmp(name, "Sky View LUT Pass") == 0 || std::strcmp(name, "Aerial Perspective Pass") == 0 || std::strcmp(name, "Sky Composite Pass") == 0); + } + // Register() callbacks may decline a pass after making declarations. Keep a // snapshot of every declaration-owned mutation so a declined callback is // indistinguishable from one that was never registered. @@ -485,7 +500,7 @@ namespace ZEngine::Rendering::Renderers } } // namespace - void BuildQueueBatches(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, Array& out_batches) + void BuildQueueBatches(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, bool isolate_atmosphere_view_passes, Array& out_batches) { out_batches.clear(); @@ -500,7 +515,9 @@ namespace ZEngine::Rendering::Renderers if ((pass.Queue == Rendering::QueueType::TRANSFER_QUEUE && !has_separate_transfer_queue) || (pass.Queue == Rendering::QueueType::COMPUTE_QUEUE && !has_separate_compute_queue)) pass.Queue = Rendering::QueueType::GRAPHIC_QUEUE; - if (!out_batches.empty() && out_batches.back().Queue == pass.Queue && out_batches.back().FirstPassOrder + out_batches.back().PassCount == order_index) + const bool isolate_current_pass = isolate_atmosphere_view_passes && IsAtmosphereViewPass(pass.Name); + const bool isolate_previous_pass = !out_batches.empty() && isolate_atmosphere_view_passes && IsAtmosphereViewPass(passes[order[out_batches.back().FirstPassOrder + out_batches.back().PassCount - 1]].Name); + if (!isolate_current_pass && !isolate_previous_pass && !out_batches.empty() && out_batches.back().Queue == pass.Queue && out_batches.back().FirstPassOrder + out_batches.back().PassCount == order_index) { ++out_batches.back().PassCount; continue; @@ -1612,6 +1629,21 @@ namespace ZEngine::Rendering::Renderers Register({.Scene = SceneData, .FrameIndex = static_cast(Device->SwapchainPtr->CurrentFrame->Index), .RenderWidth = RenderWidth, .RenderHeight = RenderHeight}); Compile(); +#ifndef NDEBUG + { + bool has_atmosphere_view_passes = false; + for (const RGPass& pass : Passes) + { + if (pass.IsActive() && IsAtmosphereViewPass(pass.Name)) + { + has_atmosphere_view_passes = true; + break; + } + } + Device->SwapchainPtr->TraceSubmission = IsAtmosphereSubmissionTraceEnabled() && has_atmosphere_view_passes; + } +#endif + // Register() can append deferred copies (notably the UI upload), and scene // uploads were already recorded before Execute(). Close that single batch now: // immediate graph batches below must wait for it just like the Present-owned @@ -2238,6 +2270,15 @@ namespace ZEngine::Rendering::Renderers if (retain_for_overlay) { +#ifndef NDEBUG + if (Device->SwapchainPtr->TraceSubmission) + { + const RGPass& first_pass = Passes[SortedPassIndices[batch.FirstPassOrder]]; + const RGPass& last_pass = Passes[SortedPassIndices[batch.FirstPassOrder + batch.PassCount - 1]]; + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph final batch={} queue={} passes={} first_pass='{}' last_pass='{}'", batch_index, QueueName(batch.Queue), batch.PassCount, first_pass.Name, last_pass.Name) + ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); + } +#endif for (uint32_t order_index = batch.FirstPassOrder; order_index < batch.FirstPassOrder + batch.PassCount && order_index < SortedPassIndices.size(); ++order_index) { const RGPass& pass = Passes[SortedPassIndices[order_index]]; @@ -2278,11 +2319,35 @@ namespace ZEngine::Rendering::Renderers ZENGINE_VALIDATE_ASSERT(timeline_index < QueueTimelineCount, "Invalid render graph queue type") auto* const timeline = QueueTimelines[timeline_index]; const uint64_t signal_value = ++QueueTimelineValues[timeline_index]; +#ifndef NDEBUG + if (Device->SwapchainPtr->TraceSubmission) + { + const RGPass& first_pass = Passes[SortedPassIndices[batch.FirstPassOrder]]; + const RGPass& last_pass = Passes[SortedPassIndices[batch.FirstPassOrder + batch.PassCount - 1]]; + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} queue={} passes={} first_pass='{}' last_pass='{}' waits={} signal_timeline={} signal_value={}", batch_index, QueueName(batch.Queue), batch.PassCount, first_pass.Name, last_pass.Name, wait_infos.size(), static_cast(timeline), signal_value) + for (const VkSemaphoreSubmitInfo& wait : wait_infos) + { + uint64_t completed = 0; + const VkResult result = vkGetSemaphoreCounterValue(Device->LogicalDevice, wait.semaphore, &completed); + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} waits timeline={} value={} completed={} query_result={} stages={}", batch_index, static_cast(wait.semaphore), wait.value, completed, static_cast(result), static_cast(wait.stageMask)) + } + // The normal file sink is asynchronous. Persist the synchronous ring + // buffer before submitting, so a GPU stall cannot lose this batch. + ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); + } +#endif if (!Device->QueueSubmit(target, timeline, signal_value, wait_infos.data(), static_cast(wait_infos.size()))) { graph_submission_failed = true; break; } +#ifndef NDEBUG + if (Device->SwapchainPtr->TraceSubmission) + { + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} accepted", batch_index) + ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); + } +#endif batch_timelines[batch_index] = timeline; batch_signal_values[batch_index] = signal_value; @@ -4216,7 +4281,11 @@ namespace ZEngine::Rendering::Renderers if (!pass.QueryWrites.empty()) pass.RequestedQueue = Rendering::QueueType::GRAPHIC_QUEUE; } - BuildQueueBatches(Passes, SortedPassIndices, Device->HasSeparateTransferQueue, Device->HasSeparateComputeQueue, QueueBatches); + bool isolate_atmosphere_view_passes = false; +#ifndef NDEBUG + isolate_atmosphere_view_passes = IsAtmosphereSubmissionTraceEnabled(); +#endif + BuildQueueBatches(Passes, SortedPassIndices, Device->HasSeparateTransferQueue, Device->HasSeparateComputeQueue, isolate_atmosphere_view_passes, QueueBatches); auto scratch = ZGetScratch(Device->Arena); BuildQueueDependencies(scratch.Arena, Passes, SortedPassIndices, QueueBatches, PassDependencies, QueueDependencies); From 87305440aca90fd5d11f35d9b1d0ae081b8247d6 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Wed, 30 Sep 2026 22:06:59 +0900 Subject: [PATCH 15/27] fix(render-graph): preserve queue batching API --- ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp index f6eeaa6a..e2158b17 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp @@ -500,7 +500,7 @@ namespace ZEngine::Rendering::Renderers } } // namespace - void BuildQueueBatches(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, bool isolate_atmosphere_view_passes, Array& out_batches) + static void BuildQueueBatchesForTrace(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, bool isolate_atmosphere_view_passes, Array& out_batches) { out_batches.clear(); @@ -530,6 +530,11 @@ namespace ZEngine::Rendering::Renderers } } + void BuildQueueBatches(ArrayView passes, ArrayView order, bool has_separate_transfer_queue, bool has_separate_compute_queue, Array& out_batches) + { + BuildQueueBatchesForTrace(passes, order, has_separate_transfer_queue, has_separate_compute_queue, false, out_batches); + } + void BuildQueueDependencies(Core::Memory::ArenaAllocator* scratch_arena, ArrayView passes, ArrayView order, ArrayView batches, ArrayView pass_dependencies, Array& out_dependencies) { out_dependencies.clear(); @@ -4285,7 +4290,7 @@ namespace ZEngine::Rendering::Renderers #ifndef NDEBUG isolate_atmosphere_view_passes = IsAtmosphereSubmissionTraceEnabled(); #endif - BuildQueueBatches(Passes, SortedPassIndices, Device->HasSeparateTransferQueue, Device->HasSeparateComputeQueue, isolate_atmosphere_view_passes, QueueBatches); + BuildQueueBatchesForTrace(Passes, SortedPassIndices, Device->HasSeparateTransferQueue, Device->HasSeparateComputeQueue, isolate_atmosphere_view_passes, QueueBatches); auto scratch = ZGetScratch(Device->Arena); BuildQueueDependencies(scratch.Arena, Passes, SortedPassIndices, QueueBatches, PassDependencies, QueueDependencies); From 1181d778886f56bc308b4bfe2e1feddc93d0069e Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Thu, 1 Oct 2026 01:19:12 +0900 Subject: [PATCH 16/27] fix(vulkan): keep submission tracing non-blocking --- ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp | 5 ----- ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp | 5 ----- 2 files changed, 10 deletions(-) diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index 6430e65a..1f786df2 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -266,7 +266,6 @@ namespace ZEngine::Hardwares if (DirectGraphicsTimeline) vkGetSemaphoreCounterValue(Device->LogicalDevice, DirectGraphicsTimeline->GetHandle(), &direct_completed); ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire wait={} frame={} render_completed={} render_submitted={} direct_timeline={} direct_completed={} direct_submitted={}", reason, frame.Index, render_completed, RenderTimelineNextValue, static_cast(DirectGraphicsTimeline ? DirectGraphicsTimeline->GetHandle() : VK_NULL_HANDLE), direct_completed, DirectGraphicsTimelineValue) - ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); }; #endif if (Recreation != RecreationState::None) @@ -551,9 +550,6 @@ namespace ZEngine::Hardwares const VkResult result = wait.value == 0 ? VK_SUCCESS : vkGetSemaphoreCounterValue(Device->LogicalDevice, wait.semaphore, &completed); ZENGINE_CORE_INFO("[VulkanSubmitTrace] present wait={} semaphore={} value={} completed={} query_result={} stages={}", wait_index, static_cast(wait.semaphore), wait.value, completed, static_cast(result), static_cast(wait.stageMask)) } - // The normal file sink is asynchronous. Persist the synchronous ring - // buffer before submitting, so a GPU stall cannot lose this frame. - ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); } #endif @@ -600,7 +596,6 @@ namespace ZEngine::Hardwares if (TraceSubmission) { ZENGINE_CORE_INFO("[VulkanSubmitTrace] present accepted frame={} render_value={}", CurrentFrame->Index, work_complete_value) - ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); } #endif diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp index e2158b17..22d00359 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp @@ -2281,7 +2281,6 @@ namespace ZEngine::Rendering::Renderers const RGPass& first_pass = Passes[SortedPassIndices[batch.FirstPassOrder]]; const RGPass& last_pass = Passes[SortedPassIndices[batch.FirstPassOrder + batch.PassCount - 1]]; ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph final batch={} queue={} passes={} first_pass='{}' last_pass='{}'", batch_index, QueueName(batch.Queue), batch.PassCount, first_pass.Name, last_pass.Name) - ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); } #endif for (uint32_t order_index = batch.FirstPassOrder; order_index < batch.FirstPassOrder + batch.PassCount && order_index < SortedPassIndices.size(); ++order_index) @@ -2336,9 +2335,6 @@ namespace ZEngine::Rendering::Renderers const VkResult result = vkGetSemaphoreCounterValue(Device->LogicalDevice, wait.semaphore, &completed); ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} waits timeline={} value={} completed={} query_result={} stages={}", batch_index, static_cast(wait.semaphore), wait.value, completed, static_cast(result), static_cast(wait.stageMask)) } - // The normal file sink is asynchronous. Persist the synchronous ring - // buffer before submitting, so a GPU stall cannot lose this batch. - ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); } #endif if (!Device->QueueSubmit(target, timeline, signal_value, wait_infos.data(), static_cast(wait_infos.size()))) @@ -2350,7 +2346,6 @@ namespace ZEngine::Rendering::Renderers if (Device->SwapchainPtr->TraceSubmission) { ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} accepted", batch_index) - ZEngine::Logging::Logger::FlushRingBufferToCrashLog(); } #endif From 7e89938a99ce8d162f8b3addb69240bfb2ef3d08 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Thu, 1 Oct 2026 01:22:49 +0900 Subject: [PATCH 17/27] chore(vulkan): trace submission boundaries --- ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp | 89 ++++++++++++++++++- .../Rendering/Renderers/RenderGraph.cpp | 1 + 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index 1f786df2..df8aa9fe 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -270,11 +270,25 @@ namespace ZEngine::Hardwares #endif if (Recreation != RecreationState::None) { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation begin state={}", static_cast(Recreation)) +#endif // Every submitted render command buffer is covered by its frame fence. for (uint32_t i = 0; i < ImageInFlights.size(); ++i) { if (ImageInFlights[i] != nullptr) + { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation image-fence wait begin image={}", i) +#endif ImageInFlights[i]->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation image-fence wait complete image={}", i) +#endif + } } // Defense-in-depth: ImageInFlights is sized SwapchainImageCount, @@ -287,7 +301,15 @@ namespace ZEngine::Hardwares { if (FrameContexts[i].Fence->GetState() == Rendering::Primitives::FenceState::Submitted) { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation frame-fence wait begin frame={}", i) +#endif FrameContexts[i].Fence->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation frame-fence wait complete frame={}", i) +#endif FrameContexts[i].Fence->Reset(); } } @@ -297,7 +319,17 @@ namespace ZEngine::Hardwares // counters; this is especially important after an OUT_OF_DATE acquire, // where Present() deliberately submits no fence-backed work. if (!Device->IsDeviceLost.load(std::memory_order_acquire)) + { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation queue-wait-all begin") +#endif Device->QueueWaitAll(); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation queue-wait-all complete") +#endif + } for (int i = 0; i < FrameContextPoolSizeFactor; ++i) { @@ -342,6 +374,11 @@ namespace ZEngine::Hardwares Recreation = RecreationState::None; ZENGINE_CORE_WARN("Swapchain recreated: {}x{}", SwapchainImageWidth, SwapchainImageHeight) +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] swapchain recreation complete") +#endif + if (OnSwapchainResized) OnSwapchainResized(SwapchainImageWidth, SwapchainImageHeight, OnSwapchainResizedCtx); } @@ -353,12 +390,24 @@ namespace ZEngine::Hardwares trace_fence_wait("frame", frame); #endif frame.Fence->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire frame-fence wait complete frame={}", frame.Index) +#endif } frame.Fence->Reset(); frame.Acquired->SetState(Primitives::SemaphoreState::Idle); - uint32_t image_idx = 0; + uint32_t image_idx = 0; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire call frame={} semaphore={}", frame.Index, static_cast(frame.Acquired->GetHandle())) +#endif VkResult acquire_image_result = vkAcquireNextImageKHR(Device->LogicalDevice, SwapchainHandle, UINT64_MAX, frame.Acquired->GetHandle(), VK_NULL_HANDLE, &image_idx); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire returned frame={} result={} image={}", frame.Index, static_cast(acquire_image_result), image_idx) +#endif frame.Acquired->SetState(Primitives::SemaphoreState::Submitted); Device->TickMemory(); @@ -385,6 +434,10 @@ namespace ZEngine::Hardwares trace_fence_wait("image", frame); #endif ImageInFlights[image_idx]->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire image-fence wait complete frame={} image={}", frame.Index, image_idx) +#endif } RenderCompletes[image_idx]->SetState(Rendering::Primitives::SemaphoreState::Idle); @@ -481,7 +534,17 @@ namespace ZEngine::Hardwares if (render_complete->GetState() == Rendering::Primitives::SemaphoreState::Submitted) render_complete->SetState(Rendering::Primitives::SemaphoreState::Idle); if (CurrentFrame->Fence->GetState() == Rendering::Primitives::FenceState::Submitted) + { +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-fence wait begin frame={}", CurrentFrame->Index) +#endif CurrentFrame->Fence->Wait(UINT64_MAX); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-fence wait complete frame={}", CurrentFrame->Index) +#endif + } QueueView queue = Device->GetQueue(Rendering::QueueType::GRAPHIC_QUEUE); @@ -581,9 +644,25 @@ namespace ZEngine::Hardwares .pSignalSemaphoreInfos = work_signal_infos, }; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-heap flush begin frame={}", CurrentFrame->Index) +#endif Device->FrameHeaps[CurrentFrame->Index].Flush(&Device->GpuMem); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-heap flush complete frame={}", CurrentFrame->Index) +#endif +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present submit begin frame={} render_value={} queue={}", CurrentFrame->Index, work_complete_value, static_cast(queue.Handle)) +#endif auto submit = vkQueueSubmit2(queue.Handle, 1, &submit_info_1, CurrentFrame->Fence->GetHandle()); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present submit returned frame={} result={}", CurrentFrame->Index, static_cast(submit)) +#endif if (Device->CheckDeviceLost(submit, "Present: render work submit")) { ZReleaseScratch(scratch); @@ -625,7 +704,15 @@ namespace ZEngine::Hardwares .pSwapchains = swapchains, .pImageIndices = frames, }; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present WSI begin frame={} image={} queue={}", CurrentFrame->Index, CurrentFrame->ImageIndex, static_cast(queue.Handle)) +#endif VkResult present_result = vkQueuePresentKHR(queue.Handle, &present_info); +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] present WSI returned frame={} image={} result={}", CurrentFrame->Index, CurrentFrame->ImageIndex, static_cast(present_result)) +#endif IdleFrameCount.value.fetch_add(1, std::memory_order_acq_rel); diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp index 22d00359..adaaaedd 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp @@ -2335,6 +2335,7 @@ namespace ZEngine::Rendering::Renderers const VkResult result = vkGetSemaphoreCounterValue(Device->LogicalDevice, wait.semaphore, &completed); ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph batch={} waits timeline={} value={} completed={} query_result={} stages={}", batch_index, static_cast(wait.semaphore), wait.value, completed, static_cast(result), static_cast(wait.stageMask)) } + ZENGINE_CORE_INFO("[VulkanSubmitTrace] graph submit begin batch={} queue={} signal_value={}", batch_index, QueueName(batch.Queue), signal_value) } #endif if (!Device->QueueSubmit(target, timeline, signal_value, wait_infos.data(), static_cast(wait_infos.size()))) From 435eb1b460e5a7caaca848bf87db6ab2a6935960 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Thu, 1 Oct 2026 01:37:05 +0900 Subject: [PATCH 18/27] fix(logging): preserve configured flush interval --- ZEngine/ZEngine/Logging/Logger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZEngine/ZEngine/Logging/Logger.cpp b/ZEngine/ZEngine/Logging/Logger.cpp index c41ec1f5..d86ea3ee 100644 --- a/ZEngine/ZEngine/Logging/Logger.cpp +++ b/ZEngine/ZEngine/Logging/Logger.cpp @@ -79,7 +79,7 @@ namespace ZEngine::Logging } spdlog::init_thread_pool(8192, 2); - spdlog::flush_every(std::chrono::duration_cast(configuration.PeriodicFlush)); + spdlog::flush_every(configuration.PeriodicFlush); spdlog::flush_on(spdlog::level::critical); s_sink = std::make_shared((log_directory_path / configuration.LogFilename).string(), 1024 * 1024, 5, false); From 078740edc6e4ff33974ad00c5a7cd2ece602deeb Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Thu, 1 Oct 2026 01:53:08 +0900 Subject: [PATCH 19/27] fix(swapchain): prevent reset fence self-wait --- ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp | 39 ++++++++++++++++--- ZEngine/ZEngine/Hardwares/DeviceSwapchain.h | 5 +++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index df8aa9fe..511ff71d 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -395,6 +395,11 @@ namespace ZEngine::Hardwares ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire frame-fence wait complete frame={}", frame.Index) #endif } + + // A reset fence must not remain associated with any swapchain image. + // Besides the normal completed submission, this clears a stale reference + // left by an aborted frame before it can become an unsignaled self-wait. + ClearImageInFlightReferences(frame.Fence, frame.Index); frame.Fence->Reset(); frame.Acquired->SetState(Primitives::SemaphoreState::Idle); @@ -428,12 +433,15 @@ namespace ZEngine::Hardwares return; } - if (ImageInFlights[image_idx] != nullptr && !ImageInFlights[image_idx]->IsSignaled()) + auto* const image_fence = ImageInFlights[image_idx]; + const bool current_frame_owns_image = image_fence == frame.Fence; + ZENGINE_VALIDATE_ASSERT(!current_frame_owns_image, "An acquired image cannot wait on the frame fence being reset") + if (!current_frame_owns_image && image_fence != nullptr && !image_fence->IsSignaled()) { #ifndef NDEBUG trace_fence_wait("image", frame); #endif - ImageInFlights[image_idx]->Wait(UINT64_MAX); + image_fence->Wait(UINT64_MAX); #ifndef NDEBUG if (TraceSubmission) ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire image-fence wait complete frame={} image={}", frame.Index, image_idx) @@ -442,9 +450,8 @@ namespace ZEngine::Hardwares RenderCompletes[image_idx]->SetState(Rendering::Primitives::SemaphoreState::Idle); - ImageInFlights[image_idx] = frame.Fence; - frame.ImageIndex = image_idx; - CurrentFrame = &frame; + frame.ImageIndex = image_idx; + CurrentFrame = &frame; // SUBOPTIMAL: image is valid; Present() schedules recreation after vkQueuePresentKHR. } @@ -544,6 +551,8 @@ namespace ZEngine::Hardwares if (TraceSubmission) ZENGINE_CORE_INFO("[VulkanSubmitTrace] present frame-fence wait complete frame={}", CurrentFrame->Index) #endif + ClearImageInFlightReferences(CurrentFrame->Fence, CurrentFrame->Index); + CurrentFrame->Fence->Reset(); } QueueView queue = Device->GetQueue(Rendering::QueueType::GRAPHIC_QUEUE); @@ -671,6 +680,11 @@ namespace ZEngine::Hardwares } ZENGINE_VALIDATE_ASSERT(submit == VK_SUCCESS, "Failed to submit queue") + // Associate an image only with a fence that Vulkan accepted for + // submission. Aborted frames therefore leave no unsignaled association. + CurrentFrame->Fence->SetState(Rendering::Primitives::FenceState::Submitted); + ImageInFlights[CurrentFrame->ImageIndex] = CurrentFrame->Fence; + #ifndef NDEBUG if (TraceSubmission) { @@ -688,7 +702,6 @@ namespace ZEngine::Hardwares ZReleaseScratch(scratch); Device->CommandBufferMgr->ResetEnqueuedBufferIndex(); - CurrentFrame->Fence->SetState(Rendering::Primitives::FenceState::Submitted); render_complete->SetState(Rendering::Primitives::SemaphoreState::Submitted); @@ -741,4 +754,18 @@ namespace ZEngine::Hardwares if (fn) RenderWorkSubmittedCallbacks.push({.Function = fn, .Cancel = cancel_fn, .Context = context}); } + + void DeviceSwapchain::ClearImageInFlightReferences(Rendering::Primitives::Fence* fence, uint32_t frame_index) + { + for (uint32_t image_index = 0; image_index < ImageInFlights.size(); ++image_index) + { + if (ImageInFlights[image_index] != fence) + continue; +#ifndef NDEBUG + if (TraceSubmission) + ZENGINE_CORE_INFO("[VulkanSubmitTrace] cleared frame-fence reference frame={} image={}", frame_index, image_index) +#endif + ImageInFlights[image_index] = nullptr; + } + } } // namespace ZEngine::Hardwares diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h index c0be3228..e8715127 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h @@ -90,6 +90,8 @@ namespace ZEngine::Hardwares Core::Containers::Array SwapchainImageViews = {}; Core::Containers::Array SwapchainFramebuffers = {}; Core::Containers::Array SwapchainImageLayouts = {}; + // Fence from the accepted graphics submission that last rendered each + // swapchain image. Reset fences are never retained here. Core::Containers::Array ImageInFlights = {}; Core::Containers::Array RenderCompletes = {}; // Render-thread-owned snapshot of asynchronous GPU work relevant to the @@ -132,6 +134,9 @@ namespace ZEngine::Hardwares Recreation = state; } #endif + + private: + void ClearImageInFlightReferences(Rendering::Primitives::Fence* fence, uint32_t frame_index); }; ZDEFINE_PTR(DeviceSwapchain); } // namespace ZEngine::Hardwares From db7666ff7874496b4c6256418b94aeb252418917 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Thu, 1 Oct 2026 02:17:22 +0900 Subject: [PATCH 20/27] fix(ui): honor Linux content scale --- Tetragrama/Layers/ZUILayer.cpp | 8 +-- .../Applications/AppRenderPipeline.cpp | 64 ++++++++++--------- ZEngine/ZEngine/UI/ZUIContext.h | 8 ++- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/Tetragrama/Layers/ZUILayer.cpp b/Tetragrama/Layers/ZUILayer.cpp index 2ecc7d85..bf8d3b2b 100644 --- a/Tetragrama/Layers/ZUILayer.cpp +++ b/Tetragrama/Layers/ZUILayer.cpp @@ -298,10 +298,10 @@ namespace Tetragrama::Layers { return false; } - // GLFW cursor callback reports in logical screen coords (same space as - // glfwGetWindowSize / ScreenW). No division needed on any platform. - m_ctx->MousePos[0] = (float) e.GetPosX(); - m_ctx->MousePos[1] = (float) e.GetPosY(); + // GLFW cursor positions are in window coordinates. Convert them into + // the DPI-independent coordinate space used by ZUI layout and rendering. + m_ctx->MousePos[0] = (float) e.GetPosX() * m_ctx->InputScale[0]; + m_ctx->MousePos[1] = (float) e.GetPosY() * m_ctx->InputScale[1]; return false; } diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp index be2e0e28..b9d6591a 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include using namespace ZEngine::Core::Containers; @@ -319,9 +320,8 @@ namespace ZEngine::Applications { if (ZUICtx) { - // Use the logical window size (glfwGetWindowSize) not the physical swapchain - // size. Mouse positions from GLFW cursor callbacks are also in logical pixels, - // so panel positions and hit-testing must use the same coordinate space. + // ZUI uses DPI-independent coordinates. The render pass projects them into + // the physical swapchain, while input is converted to the same UI space. if (Device->CurrentWindow) { auto* native = static_cast(Device->CurrentWindow->GetNativeWindow()); @@ -335,46 +335,48 @@ namespace ZEngine::Applications content_scale = 1.f; } - // Exact ImGui approach (imgui_impl_glfw.cpp GetWindowSizeAndFramebufferScale): - // ScreenW/H = glfwGetWindowSize → logical screen coords (1512 on Retina) - // UIScale = glfwGetFramebufferSize / glfwGetWindowSize → physical/logical ratio (2.0 on Retina) - // Cursor from GLFW callback is already in logical space — no division needed. - // This is identical on all platforms; macOS just happens to have UIScale=2 on Retina. + // GLFW's content scale is the UI-metric scale. On Wayland and macOS it + // normally matches framebuffer/window, while X11 can report a scaled + // desktop with a 1:1 framebuffer. Build one DPI-independent ZUI space + // from the larger factor so text and controls remain readable on all + // three platforms. + int win_w = (int) Device->CurrentWindow->GetWidth(); + int win_h = (int) Device->CurrentWindow->GetHeight(); + int fb_w = win_w; + int fb_h = win_h; if (native) { - int win_w = 0, win_h = 0, fb_w = 0, fb_h = 0; glfwGetWindowSize(native, &win_w, &win_h); glfwGetFramebufferSize(native, &fb_w, &fb_h); - if (win_w > 0 && win_h > 0) - { - ZUICtx->ScreenW = (uint32_t) win_w; - ZUICtx->ScreenH = (uint32_t) win_h; - float s = (fb_w > 0) ? (float) fb_w / (float) win_w : 1.f; - ZUICtx->UIScale = (s > 0.5f) ? s : 1.f; - } - else - { - ZUICtx->ScreenW = Device->CurrentWindow->GetWidth(); - ZUICtx->ScreenH = Device->CurrentWindow->GetHeight(); - ZUICtx->UIScale = content_scale; - } - } - else - { - ZUICtx->ScreenW = Device->CurrentWindow->GetWidth(); - ZUICtx->ScreenH = Device->CurrentWindow->GetHeight(); - ZUICtx->UIScale = content_scale; } + + win_w = win_w > 0 ? win_w : 1; + win_h = win_h > 0 ? win_h : 1; + fb_w = fb_w > 0 ? fb_w : win_w; + fb_h = fb_h > 0 ? fb_h : win_h; + + const float framebuffer_scale_x = (float) fb_w / (float) win_w; + const float framebuffer_scale_y = (float) fb_h / (float) win_h; + const float ui_scale = fmaxf(0.5f, fmaxf(content_scale, fmaxf(framebuffer_scale_x, framebuffer_scale_y))); + + ZUICtx->ScreenW = (uint32_t) fmaxf(1.f, roundf((float) fb_w / ui_scale)); + ZUICtx->ScreenH = (uint32_t) fmaxf(1.f, roundf((float) fb_h / ui_scale)); + ZUICtx->UIScale = (float) fb_w / (float) ZUICtx->ScreenW; + ZUICtx->InputScale[0] = (float) ZUICtx->ScreenW / (float) win_w; + ZUICtx->InputScale[1] = (float) ZUICtx->ScreenH / (float) win_h; if (!ZUICtx->UIScaleLogged) { - ZENGINE_CORE_INFO("[ZUI] UIScale={:.2f} Screen={}x{} (logical) ContentScale={:.2f}", ZUICtx->UIScale, ZUICtx->ScreenW, ZUICtx->ScreenH, content_scale); + ZENGINE_CORE_INFO("[ZUI] UIScale={:.2f} Screen={}x{} ContentScale={:.2f} InputScale={:.2f}x{:.2f}", ZUICtx->UIScale, ZUICtx->ScreenW, ZUICtx->ScreenH, content_scale, ZUICtx->InputScale[0], ZUICtx->InputScale[1]); ZUICtx->UIScaleLogged = true; } } else { - ZUICtx->ScreenW = Device->SwapchainPtr->SwapchainImageWidth; - ZUICtx->ScreenH = Device->SwapchainPtr->SwapchainImageHeight; + ZUICtx->ScreenW = Device->SwapchainPtr->SwapchainImageWidth; + ZUICtx->ScreenH = Device->SwapchainPtr->SwapchainImageHeight; + ZUICtx->UIScale = 1.f; + ZUICtx->InputScale[0] = 1.f; + ZUICtx->InputScale[1] = 1.f; } ZEngine::UI::ZUIBeginFrame(ZUICtx, dt); } diff --git a/ZEngine/ZEngine/UI/ZUIContext.h b/ZEngine/ZEngine/UI/ZUIContext.h index 4e113e57..12e7265a 100644 --- a/ZEngine/ZEngine/UI/ZUIContext.h +++ b/ZEngine/ZEngine/UI/ZUIContext.h @@ -386,9 +386,13 @@ namespace ZEngine::UI // current swapchain dimensions — set by AppRenderPipeline::BeginOverlayFrame each frame uint32_t ScreenW = 1280; uint32_t ScreenH = 720; - // display content scale (glfwGetWindowContentScale); 1.0=standard, 2.0=Retina. - // Widgets multiply logical pixel sizes by this to stay readable at any DPI. + // Physical framebuffer pixels per ZUI coordinate. This incorporates the GLFW + // content scale on platforms where the framebuffer/window ratio remains 1:1. float UIScale = 1.f; + // Converts GLFW cursor coordinates into ZUI coordinates. GLFW reports + // positions in window coordinates, which may be larger than ZUI's + // DPI-independent coordinate space. + float InputScale[2] = {1.f, 1.f}; // guard against per-frame ContentScale log spam — log only once bool UIScaleLogged = false; From 8e8d2b8115e8c6ea2e9609bafc29893d6e2448b1 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Sat, 3 Oct 2026 21:59:47 +0900 Subject: [PATCH 21/27] fix(ui): align framebuffer scaling and camera input Snapshot framebuffer dimensions and recreate the swapchain on resize or DPI changes. Derive UI clipping and pixel snapping from the actual render extent. Bound image acquisition and skip invalid frames safely. Convert camera viewport bounds and extent into GLFW window coordinates. Add regression coverage for resize, acquisition failures, and scaled camera input. --- Resources/Shaders/zui_draw.vert | 18 +- Tetragrama/Panels/ViewportPanel.cpp | 8 +- .../Applications/AppRenderPipeline.cpp | 14 +- .../ZEngine/Applications/AppRenderPipeline.h | 3 + .../ZEngine/Controllers/ICameraController.h | 31 ++- ZEngine/ZEngine/Engine.cpp | 13 +- ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp | 103 +++++--- ZEngine/ZEngine/Hardwares/DeviceSwapchain.h | 32 ++- .../ZEngine/Rendering/Renderers/ZUIPass.cpp | 64 +++-- ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h | 27 ++- ZEngine/ZEngine/UI/ZUIContext.h | 4 +- ZEngine/ZEngine/Windows/GameWindow.cpp | 13 +- .../AppRenderPipelineMailbox_test.cpp | 6 + .../Rendering/FlyCameraController_test.cpp | 82 +++++++ .../tests/Rendering/SwapchainExtent_test.cpp | 224 ++++++++++++++++++ .../tests/Rendering/ZUIRenderScale_test.cpp | 107 +++++++++ 16 files changed, 640 insertions(+), 109 deletions(-) create mode 100644 ZEngine/tests/Rendering/SwapchainExtent_test.cpp create mode 100644 ZEngine/tests/Rendering/ZUIRenderScale_test.cpp diff --git a/Resources/Shaders/zui_draw.vert b/Resources/Shaders/zui_draw.vert index cc8fd949..8b942ffb 100644 --- a/Resources/Shaders/zui_draw.vert +++ b/Resources/Shaders/zui_draw.vert @@ -6,10 +6,10 @@ layout(location = 2) in vec4 aColor; // RGBA8 UNORM — hardware unpacks packed layout(push_constant) uniform PC { - vec2 uScale; - vec2 uTranslate; - uint uTexIdx; - float uFbScale; // UIScale = fb/win; snaps vertices to nearest physical pixel + vec2 uScale; + vec2 uTranslate; + vec2 uFbScale; // Actual render-target pixels per logical UI unit, per axis. + uint uTexIdx; } pc; @@ -21,12 +21,12 @@ layout(location = 0) out struct void main() { - Out.Color = aColor; - Out.TexData = vec4(aUV, float(pc.uTexIdx), 0.0); + Out.Color = aColor; + Out.TexData = vec4(aUV, float(pc.uTexIdx), 0.0); // Snap to the nearest physical pixel before projecting. // Prevents sub-pixel drift that blurs glyph quads on non-Retina displays. - float fs = max(pc.uFbScale, 1.0); - vec2 snapped = round(aPos * fs) / fs; - gl_Position = vec4(snapped * pc.uScale + pc.uTranslate, 0.0, 1.0); + vec2 fs = max(pc.uFbScale, vec2(0.0001)); + vec2 snapped = round(aPos * fs) / fs; + gl_Position = vec4(snapped * pc.uScale + pc.uTranslate, 0.0, 1.0); } diff --git a/Tetragrama/Panels/ViewportPanel.cpp b/Tetragrama/Panels/ViewportPanel.cpp index b6f26e59..cf9cc0ef 100644 --- a/Tetragrama/Panels/ViewportPanel.cpp +++ b/Tetragrama/Panels/ViewportPanel.cpp @@ -163,13 +163,11 @@ namespace Tetragrama::Panels ZUISignal img_sig = ZUISignalFromBox(ctx, img); ZUIPopBox(ctx); - // Feed the viewport rect to the camera controller every frame so it can - // self-gate on cursor position without depending on the ZUI hit-test chain. + // The camera consumes raw GLFW cursor positions, not UI-space positions. + // Convert both the hover bounds and the extent used by camera rays/panning. if (app->CameraController) { - app->CameraController->SetViewportRect(rect[0], rect[1], rect[2], rect[3]); - if (sw > 0.0f && sh > 0.0f) - app->CameraController->SetViewport(sw, sh); + app->CameraController->SetViewportFromUI(rect, ctx->InputScale); } // Keep ViewportHovered for ZUI-level concerns (drag-drop, scroll routing). diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp index b9d6591a..87f17ca6 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp @@ -133,6 +133,8 @@ namespace ZEngine::Applications auto swapchain = Device->SwapchainPtr; swapchain->AcquireNextImage(CurrentFrameContextIndex); + if (!swapchain->IsFrameValid()) + return false; if (Device->RRM) static_cast(Device->RRM)->BeginFrame(swapchain->CurrentFrame->Index); @@ -161,6 +163,11 @@ namespace ZEngine::Applications void AppRenderPipeline::EndFrame() { + if (!Device->SwapchainPtr->IsFrameValid()) + { + Device->SwapchainPtr->Present(); + return; + } if (Device->RRM) static_cast(Device->RRM)->EndFrame(); @@ -320,6 +327,9 @@ namespace ZEngine::Applications { if (ZUICtx) { + const uint32_t previous_screen_w = ZUICtx->ScreenW; + const uint32_t previous_screen_h = ZUICtx->ScreenH; + const float previous_scale = ZUICtx->UIScale; // ZUI uses DPI-independent coordinates. The render pass projects them into // the physical swapchain, while input is converted to the same UI space. if (Device->CurrentWindow) @@ -364,9 +374,9 @@ namespace ZEngine::Applications ZUICtx->UIScale = (float) fb_w / (float) ZUICtx->ScreenW; ZUICtx->InputScale[0] = (float) ZUICtx->ScreenW / (float) win_w; ZUICtx->InputScale[1] = (float) ZUICtx->ScreenH / (float) win_h; - if (!ZUICtx->UIScaleLogged) + if (!ZUICtx->UIScaleLogged || previous_screen_w != ZUICtx->ScreenW || previous_screen_h != ZUICtx->ScreenH || previous_scale != ZUICtx->UIScale) { - ZENGINE_CORE_INFO("[ZUI] UIScale={:.2f} Screen={}x{} ContentScale={:.2f} InputScale={:.2f}x{:.2f}", ZUICtx->UIScale, ZUICtx->ScreenW, ZUICtx->ScreenH, content_scale, ZUICtx->InputScale[0], ZUICtx->InputScale[1]); + ZENGINE_CORE_INFO("[ZUI] UIScale={:.2f} Screen={}x{} Window={}x{} Framebuffer={}x{} ContentScale={:.2f} InputScale={:.2f}x{:.2f}", ZUICtx->UIScale, ZUICtx->ScreenW, ZUICtx->ScreenH, win_w, win_h, fb_w, fb_h, content_scale, ZUICtx->InputScale[0], ZUICtx->InputScale[1]); ZUICtx->UIScaleLogged = true; } } diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.h b/ZEngine/ZEngine/Applications/AppRenderPipeline.h index bd7acdf1..4f140d7e 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.h +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.h @@ -32,6 +32,9 @@ namespace ZEngine::Applications uint32_t RenderTargetW = 0; uint32_t RenderTargetH = 0; uint64_t ResizeSequence = 0; + // Physical window size, captured on the main thread with platform events. + uint32_t FramebufferW = 0; + uint32_t FramebufferH = 0; bool RenderOverlay = false; }; diff --git a/ZEngine/ZEngine/Controllers/ICameraController.h b/ZEngine/ZEngine/Controllers/ICameraController.h index c9ae71ab..3548ffb0 100644 --- a/ZEngine/ZEngine/Controllers/ICameraController.h +++ b/ZEngine/ZEngine/Controllers/ICameraController.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace ZEngine::Controllers { @@ -22,19 +23,33 @@ namespace ZEngine::Controllers virtual void SetViewportOrigin(float x, float y) = 0; /// @brief Notify the controller of the viewport's screen rect each frame. /// Used for self-contained hover detection — no ZUI hit-test dependency. - /// @param x0 Left edge in logical pixels. - /// @param y0 Top edge in logical pixels. - /// @param x1 Right edge in logical pixels. - /// @param y1 Bottom edge in logical pixels. + /// @param x0 Left edge in GLFW window coordinates. + /// @param y0 Top edge in GLFW window coordinates. + /// @param x1 Right edge in GLFW window coordinates. + /// @param y1 Bottom edge in GLFW window coordinates. virtual void SetViewportRect(float x0, float y0, float x1, float y1) = 0; + /// @brief Converts UI bounds and extent to the same window coordinates as cursor input. + /// @param input_scale UI coordinates per GLFW window coordinate, independently per axis. + void SetViewportFromUI(const float rect[4], const float input_scale[2]) + { + const float sx = std::isfinite(input_scale[0]) && input_scale[0] > 0.f ? input_scale[0] : 1.f; + const float sy = std::isfinite(input_scale[1]) && input_scale[1] > 0.f ? input_scale[1] : 1.f; + const float x0 = rect[0] / sx; + const float y0 = rect[1] / sy; + const float x1 = rect[2] / sx; + const float y1 = rect[3] / sy; + SetViewportRect(x0, y0, x1, y1); + if (x1 > x0 && y1 > y0) + SetViewport(x1 - x0, y1 - y0); + } /// @brief Inform the controller that the UI owns one or both input channels. /// @param pointer_captured True while a non-viewport UI control owns pointer input. /// @param keyboard_captured True while a UI control, popup, or modal owns keyboard input. - virtual void SetInputCapture(bool pointer_captured, bool keyboard_captured) {} - virtual void ResumeEventProcessing() = 0; - virtual void PauseEventProcessing() = 0; + virtual void SetInputCapture(bool pointer_captured, bool keyboard_captured) {} + virtual void ResumeEventProcessing() = 0; + virtual void PauseEventProcessing() = 0; - CameraControllerType GetControllerType() const + CameraControllerType GetControllerType() const { return m_controller_type; } diff --git a/ZEngine/ZEngine/Engine.cpp b/ZEngine/ZEngine/Engine.cpp index 72019594..5684881a 100644 --- a/ZEngine/ZEngine/Engine.cpp +++ b/ZEngine/ZEngine/Engine.cpp @@ -373,7 +373,13 @@ namespace ZEngine static_cast(g_engine_ctx->VFS)->Tick(); if (window->IsMinimized()) + { + // Publish a zero-size snapshot so the render thread pauses too; + // retaining the last visible extent is not sufficient on Wayland. + g_engine_ctx->App->RenderPipeline->PublishFrameState({}); + frame_cap.WaitForFrameBudget(); continue; + } // Measure raw delta float raw_dt = frame_timer.End(); @@ -443,6 +449,8 @@ namespace ZEngine Applications::RenderFrameState state = {}; g_engine_ctx->App->PrepareScene(state); + state.FramebufferW = window->GetWidth(); + state.FramebufferH = window->GetHeight(); if (state.Scene) { // Scene mutation happens on this thread. Send an immutable @@ -491,7 +499,7 @@ namespace ZEngine while (true) { - if (g_engine_ctx->RequestTerminate.value.load(std::memory_order_acquire)) + if (g_engine_ctx->RequestTerminate.value.load(std::memory_order_acquire) || g_engine_ctx->CloseRequested.value.load(std::memory_order_acquire)) { break; } @@ -531,6 +539,7 @@ namespace ZEngine applied_resize_sequence = state.ResizeSequence; } + pipeline->Device->SwapchainPtr->UpdateFramebufferExtent(state.FramebufferW, state.FramebufferH); const bool frame_valid = pipeline->BeginFrame(); if (frame_valid && state.Scene) { @@ -538,6 +547,8 @@ namespace ZEngine pipeline->RenderScene(state.Camera, state.Scene, state.Sky, state.CelestialLight, state.SkyRevision, overlay); } pipeline->EndFrame(); + if (!frame_valid) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); // Update SmoothedDeltaTime with the render thread's smoothed frame time. // End() is called after EndFrame() so the vsync wait is included in the sample. diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index 511ff71d..011b841f 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -1,4 +1,5 @@ +#include #include #include #include @@ -6,6 +7,7 @@ #include #include #include +#include using namespace ZEngine::Core::Containers; using namespace ZEngine::Rendering; @@ -19,6 +21,7 @@ namespace ZEngine::Hardwares device->Arena->CreateSubArena(ZMega(3), &Arena, "VulkanDevice/Swapchain"); Device = device; + FramebufferExtent = {Device->CurrentWindow->GetWidth(), Device->CurrentWindow->GetHeight()}; BufferredFrameCount = buffered_frame_size; FrameContextPoolSize = BufferredFrameCount * FrameContextPoolSizeFactor; @@ -53,25 +56,42 @@ namespace ZEngine::Hardwares Create(); } + void DeviceSwapchain::UpdateFramebufferExtent(uint32_t width, uint32_t height) + { + if (FramebufferExtent.width == width && FramebufferExtent.height == height) + return; + + FramebufferExtent = {width, height}; + // Wayland may keep returning VK_SUCCESS after a resize, so do not rely + // exclusively on OUT_OF_DATE/SUBOPTIMAL to recreate the swapchain. + Recreation = RecreationState::Pending; + } + + VkExtent2D DeviceSwapchain::ResolveExtent(const VkSurfaceCapabilitiesKHR& capabilities, VkExtent2D framebuffer_extent) + { + if (framebuffer_extent.width == 0 || framebuffer_extent.height == 0) + return {}; + if (capabilities.currentExtent.width != std::numeric_limits::max()) + return capabilities.currentExtent; + + return { + std::clamp(framebuffer_extent.width, capabilities.minImageExtent.width, capabilities.maxImageExtent.width), + std::clamp(framebuffer_extent.height, capabilities.minImageExtent.height, capabilities.maxImageExtent.height), + }; + } + void DeviceSwapchain::Create() { VkSurfaceCapabilitiesKHR capabilities{}; vkGetPhysicalDeviceSurfaceCapabilitiesKHR(Device->PhysicalDevice, Device->Surface, &capabilities); - if (capabilities.currentExtent.width != std::numeric_limits::max()) - { - SwapchainImageWidth = capabilities.currentExtent.width; - SwapchainImageHeight = capabilities.currentExtent.height; - } - else - { - // Surface does not report a concrete extent (Wayland, headless) — ask the window. - SwapchainImageWidth = Device->CurrentWindow->GetWidth(); - SwapchainImageHeight = Device->CurrentWindow->GetHeight(); - } + const VkExtent2D extent = ResolveExtent(capabilities, FramebufferExtent); + SwapchainImageWidth = extent.width; + SwapchainImageHeight = extent.height; // {0,0} extent is a spec violation; destroy stale swapchain and retry next frame. if (SwapchainImageWidth == 0 || SwapchainImageHeight == 0) { + Recreation = RecreationState::Pending; if (SwapchainHandle != VK_NULL_HANDLE) { vkDestroySwapchainKHR(Device->LogicalDevice, SwapchainHandle, nullptr); @@ -185,6 +205,8 @@ namespace ZEngine::Hardwares if (previous_image_count != SwapchainImageCount) ++SwapchainImageCountChangeCount; + ZENGINE_CORE_INFO("Swapchain created: {}x{} (framebuffer={}x{})", SwapchainImageWidth, SwapchainImageHeight, FramebufferExtent.width, FramebufferExtent.height) + if (old_swapchain != VK_NULL_HANDLE) { ZENGINE_DESTROY_VULKAN_HANDLE(Device->LogicalDevice, vkDestroySwapchainKHR, old_swapchain, nullptr) @@ -255,6 +277,14 @@ namespace ZEngine::Hardwares void DeviceSwapchain::AcquireNextImage(uint32_t frame_context_idx) { + // Do not reset fences or command pools while a minimized/zero-size + // window cannot acquire an image. Restore recreates through the normal path. + if (FramebufferExtent.width == 0 || FramebufferExtent.height == 0) + { + CurrentFrame = nullptr; + Recreation = RecreationState::Pending; + return; + } #ifndef NDEBUG auto trace_fence_wait = [this](cstring reason, const FrameContext& frame) { if (!TraceSubmission) @@ -408,30 +438,23 @@ namespace ZEngine::Hardwares if (TraceSubmission) ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire call frame={} semaphore={}", frame.Index, static_cast(frame.Acquired->GetHandle())) #endif - VkResult acquire_image_result = vkAcquireNextImageKHR(Device->LogicalDevice, SwapchainHandle, UINT64_MAX, frame.Acquired->GetHandle(), VK_NULL_HANDLE, &image_idx); + VkResult acquire_image_result = vkAcquireNextImageKHR(Device->LogicalDevice, SwapchainHandle, ImageAcquireTimeoutNs, frame.Acquired->GetHandle(), VK_NULL_HANDLE, &image_idx); #ifndef NDEBUG if (TraceSubmission) ZENGINE_CORE_INFO("[VulkanSubmitTrace] acquire returned frame={} result={} image={}", frame.Index, static_cast(acquire_image_result), image_idx) #endif - frame.Acquired->SetState(Primitives::SemaphoreState::Submitted); - Device->TickMemory(); - - if (acquire_image_result == VK_ERROR_OUT_OF_DATE_KHR) - { - // Semaphore not signalled (spec) — image_idx invalid, skip all GPU work. - frame.ImageIndex = std::numeric_limits::max(); - CurrentFrame = &frame; - Recreation = RecreationState::FrameAborted; - return; - } - - if (Device->CheckDeviceLost(acquire_image_result, "AcquireNextImage")) + if (!ApplyAcquireResult(frame, acquire_image_result, image_idx)) { - frame.ImageIndex = std::numeric_limits::max(); - CurrentFrame = &frame; - Recreation = RecreationState::FrameAborted; + // Timeout/not-ready signal no semaphore and need no recreation. The + // next render-loop iteration can consume a resize/minimize/close request. + if (acquire_image_result != VK_TIMEOUT && acquire_image_result != VK_NOT_READY && acquire_image_result != VK_ERROR_OUT_OF_DATE_KHR && !Device->CheckDeviceLost(acquire_image_result, "AcquireNextImage")) + { + ZENGINE_CORE_ERROR("vkAcquireNextImageKHR did not yield a valid image: result={} image={} image_count={}", static_cast(acquire_image_result), image_idx, SwapchainImageCount) + Engine::RequestClose(); + } return; } + frame.Acquired->SetState(Primitives::SemaphoreState::Submitted); auto* const image_fence = ImageInFlights[image_idx]; const bool current_frame_owns_image = image_fence == frame.Fence; @@ -449,10 +472,26 @@ namespace ZEngine::Hardwares } RenderCompletes[image_idx]->SetState(Rendering::Primitives::SemaphoreState::Idle); + // Tick only after a valid acquisition and the image fence wait. On a + // failed acquire (especially DEVICE_LOST), no further Vulkan work is safe. + Device->TickMemory(); + } - frame.ImageIndex = image_idx; + bool DeviceSwapchain::ApplyAcquireResult(FrameContext& frame, VkResult result, uint32_t image_index) + { CurrentFrame = &frame; - // SUBOPTIMAL: image is valid; Present() schedules recreation after vkQueuePresentKHR. + frame.ImageIndex = std::numeric_limits::max(); + if ((result == VK_SUCCESS || result == VK_SUBOPTIMAL_KHR) && image_index < SwapchainImageCount) + { + frame.ImageIndex = image_index; + if (result == VK_SUBOPTIMAL_KHR) + Recreation = RecreationState::Pending; + return true; + } + + if (result != VK_TIMEOUT && result != VK_NOT_READY) + Recreation = RecreationState::FrameAborted; + return false; } void DeviceSwapchain::CollectAsyncGPUOperations() @@ -476,9 +515,9 @@ namespace ZEngine::Hardwares RenderWorkSubmittedCallbacks.clear(); }; - if (Recreation == RecreationState::FrameAborted) + if (!IsFrameValid()) { - // OOD at acquire: semaphore not signalled, no GPU work submitted. + // Failed/zero-size acquire: no GPU work was recorded for this frame. IdleFrameCount.value.fetch_add(1, std::memory_order_acq_rel); Device->CommandBufferMgr->ResetEnqueuedBufferIndex(); discard_submission_callbacks(); diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h index e8715127..a5330723 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.h @@ -13,8 +13,8 @@ namespace ZEngine::Hardwares struct VulkanDevice; // None: normal. Pending: recreate at start of next AcquireNextImage (set by - // SUBOPTIMAL/OOD at present or zero-size surface). FrameAborted: OOD at acquire — - // semaphore not signalled, no GPU work submitted, BeginFrame returns false. + // framebuffer resize, SUBOPTIMAL at acquire/present, OOD at present or zero-size + // surface). FrameAborted: failed acquire — no image, no GPU work submitted. enum class RecreationState : uint8_t { None = 0, @@ -57,6 +57,10 @@ namespace ZEngine::Hardwares struct DeviceSwapchain { + // Leave the render thread able to consume resize/minimize/close requests + // even when the compositor stops making presentation progress. + static constexpr uint64_t ImageAcquireTimeoutNs = 100'000'000; + Core::Memory::ArenaAllocator Arena = {}; VulkanDevice* Device = nullptr; RecreationState Recreation = RecreationState::None; @@ -69,6 +73,8 @@ namespace ZEngine::Hardwares uint32_t SwapchainImageWidth = std::numeric_limits::max(); uint32_t SwapchainImageHeight = std::numeric_limits::max(); + // Render-thread-owned copy of the main thread's physical framebuffer size. + VkExtent2D FramebufferExtent = {}; uint32_t FrameContextOffset = 0; uint32_t FrameContextPoolSize = 0; const uint32_t FrameContextPoolSizeFactor = 4; @@ -105,25 +111,29 @@ namespace ZEngine::Hardwares // beginning of every frame and set only for a traced graph submission. bool TraceSubmission = false; - // Returns false when the frame was aborted (OUT_OF_DATE at acquire or + // Returns false when no image was acquired (timeout, failed acquire, or // zero-size surface). Callers must skip all rendering work for that frame. bool IsFrameValid() const { return Recreation != RecreationState::FrameAborted && CurrentFrame != nullptr && CurrentFrame->ImageIndex != std::numeric_limits::max(); } - void Initialize(VulkanDevice* const device, uint32_t buffered_frame_size); - void Create(); - void Clear(); - void Dispose(); + void Initialize(VulkanDevice* const device, uint32_t buffered_frame_size); + void UpdateFramebufferExtent(uint32_t width, uint32_t height); + static VkExtent2D ResolveExtent(const VkSurfaceCapabilitiesKHR& capabilities, VkExtent2D framebuffer_extent); + void Create(); + void Clear(); + void Dispose(); - void AcquireNextImage(uint32_t frame_context_idx); - void CollectAsyncGPUOperations(); + void AcquireNextImage(uint32_t frame_context_idx); + /// @brief Updates the frame's acquired-image state; only success/suboptimal have an image. + bool ApplyAcquireResult(FrameContext& frame, VkResult result, uint32_t image_index); + void CollectAsyncGPUOperations(); /// @brief Delivers `fn` after this frame's graphics work has been accepted by Vulkan. /// @details The caller owns `context` until delivery or cancellation. `cancel_fn` /// releases any CPU-side frame state when the graphics submission is rejected. - void EnqueueRenderWorkSubmittedCallback(RenderWorkSubmittedFn fn, void* context, RenderWorkCancelledFn cancel_fn = nullptr); - void Present(); + void EnqueueRenderWorkSubmittedCallback(RenderWorkSubmittedFn fn, void* context, RenderWorkCancelledFn cancel_fn = nullptr); + void Present(); #if !defined(NDEBUG) // Test-only: inject a recreation state without going through the Vulkan diff --git a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp index 47a91082..965b1a7f 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.cpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include using namespace ZEngine::Core::Memory; @@ -14,6 +16,29 @@ using namespace ZEngine::Rendering::Specifications; namespace ZEngine::Rendering::Renderers { + static_assert(offsetof(ZUIDrawPushConstant, FbScale) == 16); + static_assert(offsetof(ZUIDrawPushConstant, TexIdx) == 24); + static_assert(sizeof(ZUIDrawPushConstant) == 28); + + VkRect2D ZUIRenderPayload::GetScissorRect(const UI::ZUIDrawListCmd& cmd, VkExtent2D render_extent) const + { + if (!(Scale[0] > 0.f) || !(Scale[1] > 0.f) || !(cmd.ClipW > 0.f) || !(cmd.ClipH > 0.f)) + return {}; + + const float scale_x = 0.5f * (float) render_extent.width * Scale[0]; + const float scale_y = 0.5f * (float) render_extent.height * Scale[1]; + // Round outward and clamp both endpoints before converting to unsigned + // extents. This also handles partially off-screen and empty clip regions. + const float x0 = std::clamp(floorf(cmd.ClipX * scale_x), 0.f, (float) render_extent.width); + const float y0 = std::clamp(floorf(cmd.ClipY * scale_y), 0.f, (float) render_extent.height); + const float x1 = std::clamp(ceilf((cmd.ClipX + cmd.ClipW) * scale_x), x0, (float) render_extent.width); + const float y1 = std::clamp(ceilf((cmd.ClipY + cmd.ClipH) * scale_y), y0, (float) render_extent.height); + return { + { (int32_t) x0, (int32_t) y0}, + {(uint32_t) (x1 - x0), (uint32_t) (y1 - y0)} + }; + } + // Initialize / Deinitialize void ZUIPass::Initialize(Hardwares::VulkanDevicePtr device) @@ -142,14 +167,13 @@ namespace ZEngine::Rendering::Renderers const uint32_t kMaxIdx = 131072; const uint32_t max_boxes = ctx->MaxBoxesPerFrame; - float fb_w = ctx->ScreenW > 0 ? (float) ctx->ScreenW : (float) Device->SwapchainPtr->SwapchainImageWidth; - float fb_h = ctx->ScreenH > 0 ? (float) ctx->ScreenH : (float) Device->SwapchainPtr->SwapchainImageHeight; + const float screen_w = ctx->ScreenW > 0 ? (float) ctx->ScreenW : 1.f; + const float screen_h = ctx->ScreenH > 0 ? (float) ctx->ScreenH : 1.f; - out->Scale[0] = 2.f / fb_w; - out->Scale[1] = 2.f / fb_h; + out->Scale[0] = 2.f / screen_w; + out->Scale[1] = 2.f / screen_h; out->Translate[0] = -1.f; out->Translate[1] = -1.f; - out->FramebufferScale = ctx->UIScale > 0.f ? ctx->UIScale : 1.f; uint32_t atlas_idx = ctx->Atlas ? static_cast(ctx->Atlas->Handle.Index) : 0; float wu = ctx->Atlas ? ctx->Atlas->WhiteU : 0.f; @@ -157,7 +181,7 @@ namespace ZEngine::Rendering::Renderers // Init draw list into payload_arena ZUIDrawListInit(&ctx->DrawList, payload_arena, kMaxVtx, kMaxIdx, wu, wv, atlas_idx); - ZUIDrawListPushClipRect(&ctx->DrawList, 0.f, 0.f, fb_w, fb_h, false); + ZUIDrawListPushClipRect(&ctx->DrawList, 0.f, 0.f, screen_w, screen_h, false); // Helpers @@ -797,13 +821,20 @@ namespace ZEngine::Rendering::Renderers const uint32_t frame_index = device->SwapchainPtr->CurrentFrame->Index; const uint32_t fi = frame_index % FRAMES_IN_FLIGHT; { - auto* gp = static_cast(pass); - command_buffer->SetViewport(gp->GetRenderAreaWidth(), gp->GetRenderAreaHeight()); + auto* gp = static_cast(pass); + const VkExtent2D render_extent = {gp->GetRenderAreaWidth(), gp->GetRenderAreaHeight()}; + command_buffer->SetViewport(render_extent.width, render_extent.height); command_buffer->BindPipeline(gp->Pipeline); command_buffer->BindVertexBuffer(VtxBHandles[fi]); command_buffer->BindIndexBuffer(IdxBHandles[fi], VK_INDEX_TYPE_UINT16); - float fs = payload.FramebufferScale; + ZUIDrawPushConstant pc = {}; + pc.Scale[0] = payload.Scale[0]; + pc.Scale[1] = payload.Scale[1]; + pc.Translate[0] = payload.Translate[0]; + pc.Translate[1] = payload.Translate[1]; + pc.FbScale[0] = 0.5f * (float) render_extent.width * payload.Scale[0]; + pc.FbScale[1] = 0.5f * (float) render_extent.height * payload.Scale[1]; for (uint32_t i = 0; i < payload.CmdCount; ++i) { @@ -818,16 +849,11 @@ namespace ZEngine::Rendering::Renderers continue; } - // Logical → physical pixel scissor - command_buffer->SetScissor((uint32_t) (cmd.ClipW * fs), (uint32_t) (cmd.ClipH * fs), (int32_t) (cmd.ClipX * fs), (int32_t) (cmd.ClipY * fs)); - - ZUIDrawPushConstant pc = {}; - pc.Scale[0] = payload.Scale[0]; - pc.Scale[1] = payload.Scale[1]; - pc.Translate[0] = payload.Translate[0]; - pc.Translate[1] = payload.Translate[1]; - pc.TexIdx = cmd.TexIdx; - pc.FbScale = payload.FramebufferScale; + const VkRect2D scissor = payload.GetScissorRect(cmd, render_extent); + if (scissor.extent.width == 0 || scissor.extent.height == 0) + continue; + command_buffer->SetScissor(scissor.extent.width, scissor.extent.height, scissor.offset.x, scissor.offset.y); + pc.TexIdx = cmd.TexIdx; command_buffer->PushConstants(VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(ZUIDrawPushConstant), &pc); command_buffer->BindDescriptorSets(frame_index); diff --git a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h index 6ea0f396..a4beeadb 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h +++ b/ZEngine/ZEngine/Rendering/Renderers/ZUIPass.h @@ -15,18 +15,19 @@ namespace ZEngine::Rendering::Renderers /// @brief Frame-local draw data consumed by ZUIPass. struct ZUIRenderPayload { - UI::ZUIDrawVtx* Vtx = nullptr; - uint32_t VtxCount = 0; - uint16_t* Idx = nullptr; - uint32_t IdxCount = 0; - UI::ZUIDrawListCmd* Cmds = nullptr; - uint32_t CmdCount = 0; - /// @brief Physical-to-logical scale used for scissor rectangles. - float FramebufferScale = 1.f; - /// @brief NDC scale: 2 / framebuffer extent. - float Scale[2] = {}; + UI::ZUIDrawVtx* Vtx = nullptr; + uint32_t VtxCount = 0; + uint16_t* Idx = nullptr; + uint32_t IdxCount = 0; + UI::ZUIDrawListCmd* Cmds = nullptr; + uint32_t CmdCount = 0; + /// @brief NDC scale: 2 / logical UI extent. + float Scale[2] = {}; /// @brief NDC origin offset. - float Translate[2] = {}; + float Translate[2] = {}; + + /// @brief Projects a logical clip rectangle into the actual render target. + VkRect2D GetScissorRect(const UI::ZUIDrawListCmd& cmd, VkExtent2D render_extent) const; }; /// @brief Push-constant layout shared with zui_draw.vert. @@ -34,9 +35,9 @@ namespace ZEngine::Rendering::Renderers { float Scale[2] = {}; float Translate[2] = {}; + /// @brief Logical-to-physical scale consumed by uFbScale. + float FbScale[2] = {1.f, 1.f}; uint32_t TexIdx = 0; - /// @brief Physical-to-logical scale consumed by uFbScale. - float FbScale = 1.f; }; /// @brief Final graphics pass that uploads and draws the ZUI draw list. diff --git a/ZEngine/ZEngine/UI/ZUIContext.h b/ZEngine/ZEngine/UI/ZUIContext.h index 12e7265a..5efe8965 100644 --- a/ZEngine/ZEngine/UI/ZUIContext.h +++ b/ZEngine/ZEngine/UI/ZUIContext.h @@ -383,7 +383,7 @@ namespace ZEngine::UI // Vector draw list — populated by PreparePayload each frame (FrameArena-backed) ZUIDrawList DrawList = {}; - // current swapchain dimensions — set by AppRenderPipeline::BeginOverlayFrame each frame + // Logical UI dimensions — set by AppRenderPipeline::BeginOverlayFrame each frame. uint32_t ScreenW = 1280; uint32_t ScreenH = 720; // Physical framebuffer pixels per ZUI coordinate. This incorporates the GLFW @@ -393,7 +393,7 @@ namespace ZEngine::UI // positions in window coordinates, which may be larger than ZUI's // DPI-independent coordinate space. float InputScale[2] = {1.f, 1.f}; - // guard against per-frame ContentScale log spam — log only once + // Log dimensions initially and on changes, never on every frame. bool UIScaleLogged = false; // drag-and-drop — source is set by ZUIBeginDragSource while a box is held+moving; diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index 11ec8a49..fa77510c 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -168,13 +168,12 @@ namespace ZEngine::Windows #endif } - int window_width = 0, window_height = 0; - glfwGetWindowSize(m_native_window, &window_width, &window_height); - if ((window_width > 0) && (window_height > 0) && (m_property.Width != window_width) && (m_property.Height != window_height)) - { - m_property.SetWidth(window_width); - m_property.SetHeight(window_height); - } + // GetWidth/GetHeight describe physical framebuffer pixels, including before + // the first resize callback (Retina and Wayland can already be scaled). + int framebuffer_width = 0, framebuffer_height = 0; + glfwGetFramebufferSize(m_native_window, &framebuffer_width, &framebuffer_height); + m_property.SetWidth(static_cast(framebuffer_width)); + m_property.SetHeight(static_cast(framebuffer_height)); uint32_t count = 0; const char** extensions_layer_names = glfwGetRequiredInstanceExtensions(&count); diff --git a/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp b/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp index 6ba2a5ed..9824f66f 100644 --- a/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp +++ b/ZEngine/tests/Rendering/AppRenderPipelineMailbox_test.cpp @@ -12,6 +12,8 @@ TEST(AppRenderPipelineMailboxTest, RenderStateProvidesLatestCoherentCameraSnapsh first.ResizeSequence = 1; first.RenderTargetW = 1280; first.RenderTargetH = 720; + first.FramebufferW = 1500; + first.FramebufferH = 800; first.SkyRevision = 4; first.Sky.EnvironmentIntensity = 1.0f; pipeline.PublishFrameState(first); @@ -21,6 +23,8 @@ TEST(AppRenderPipelineMailboxTest, RenderStateProvidesLatestCoherentCameraSnapsh latest.ResizeSequence = 2; latest.RenderTargetW = 1920; latest.RenderTargetH = 1080; + latest.FramebufferW = 2478; + latest.FramebufferH = 1514; latest.SkyRevision = 5; latest.Sky.EnvironmentIntensity = 2.0f; pipeline.PublishFrameState(latest); @@ -31,6 +35,8 @@ TEST(AppRenderPipelineMailboxTest, RenderStateProvidesLatestCoherentCameraSnapsh EXPECT_EQ(read.ResizeSequence, 2u); EXPECT_EQ(read.RenderTargetW, 1920u); EXPECT_EQ(read.RenderTargetH, 1080u); + EXPECT_EQ(read.FramebufferW, 2478u); + EXPECT_EQ(read.FramebufferH, 1514u); EXPECT_EQ(read.SkyRevision, 5u); EXPECT_FLOAT_EQ(read.Sky.EnvironmentIntensity, 2.0f); } diff --git a/ZEngine/tests/Rendering/FlyCameraController_test.cpp b/ZEngine/tests/Rendering/FlyCameraController_test.cpp index 752d0906..59a7f2bf 100644 --- a/ZEngine/tests/Rendering/FlyCameraController_test.cpp +++ b/ZEngine/tests/Rendering/FlyCameraController_test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using namespace ZEngine; using namespace ZEngine::Controllers; @@ -31,6 +32,16 @@ namespace { return m_input_enabled; } + + bool IsHovered() const + { + return m_state == CamState::Hover; + } + + float ViewportBound(uint32_t index) const + { + return m_vp[index]; + } }; struct FlyCameraControllerTest : ::testing::Test @@ -91,3 +102,74 @@ TEST_F(FlyCameraControllerTest, PauseAndResumeControlCameraInputLifecycle) Controller.ResumeEventProcessing(); EXPECT_TRUE(Controller.IsInputEnabled()); } + +TEST_F(FlyCameraControllerTest, ScaledUiViewportUsesWindowCoordinatesForHoverAndRays) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + const float input_scale[2] = {0.8f, 0.75f}; + Controller.SetViewportFromUI(rect, input_scale); + EXPECT_FLOAT_EQ(Controller.ViewportBound(0), 125.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(1), 200.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(2), 1125.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(3), 1000.f); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 1000.f / 800.f); + + // This raw cursor lies inside the displayed viewport, but outside the old + // unconverted UI bounds. It must remain available to camera navigation. + Input.AccumulateCursorPosition(1000, 600); + Controller.Update(Core::TimeStep(0.f)); + EXPECT_TRUE(Controller.IsHovered()); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportX, 875.f); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportY, 400.f); + + const auto ray = Camera.GetRayFromViewport(500.f, 400.f); + const auto forward = Camera.GetForward(); + EXPECT_NEAR(ray.Direction.x, forward.x, 0.0001f); + EXPECT_NEAR(ray.Direction.y, forward.y, 0.0001f); + EXPECT_NEAR(ray.Direction.z, forward.z, 0.0001f); + + Input.AccumulateCursorPosition(1200, 600); + Controller.Update(Core::TimeStep(0.f)); + EXPECT_FALSE(Controller.IsHovered()); +} + +TEST_F(FlyCameraControllerTest, NativeWaylandAndRetinaInputNeedsNoExtraDpiScaling) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + const float input_scale[2] = {1.f, 1.f}; + Controller.SetViewportFromUI(rect, input_scale); + for (uint32_t index = 0; index < 4; ++index) + EXPECT_FLOAT_EQ(Controller.ViewportBound(index), rect[index]); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 800.f / 600.f); + + Input.AccumulateCursorPosition(500, 450); + Controller.Update(Core::TimeStep(0.f)); + EXPECT_TRUE(Controller.IsHovered()); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportX, 400.f); + EXPECT_FLOAT_EQ(Camera.Input.MouseViewportY, 300.f); +} + +TEST_F(FlyCameraControllerTest, DpiTransitionUpdatesBoundsAndExtentWithoutStaleScale) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + const float scaled[2] = {0.8f, 0.75f}; + const float native[2] = {1.f, 1.f}; + Controller.SetViewportFromUI(rect, scaled); + Controller.SetViewportFromUI(rect, native); + EXPECT_FLOAT_EQ(Controller.ViewportBound(0), 100.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(2), 900.f); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 800.f / 600.f); +} + +TEST_F(FlyCameraControllerTest, InvalidInputScalesDoNotDivideByZeroOrProduceNan) +{ + const float rect[4] = {100.f, 150.f, 900.f, 750.f}; + for (float invalid_scale : {0.f, -1.f, std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity()}) + { + const float scales[2] = {invalid_scale, invalid_scale}; + Controller.SetViewportFromUI(rect, scales); + EXPECT_FLOAT_EQ(Controller.ViewportBound(0), 100.f); + EXPECT_FLOAT_EQ(Controller.ViewportBound(3), 750.f); + EXPECT_FLOAT_EQ(Camera.AspectRatio, 800.f / 600.f); + } +} diff --git a/ZEngine/tests/Rendering/SwapchainExtent_test.cpp b/ZEngine/tests/Rendering/SwapchainExtent_test.cpp new file mode 100644 index 00000000..1784807b --- /dev/null +++ b/ZEngine/tests/Rendering/SwapchainExtent_test.cpp @@ -0,0 +1,224 @@ +#include +#include +#include +#include +#include + +using namespace ZEngine::Hardwares; + +namespace +{ + VkSurfaceCapabilitiesKHR VariableExtentCapabilities() + { + VkSurfaceCapabilitiesKHR caps{}; + caps.currentExtent = {std::numeric_limits::max(), std::numeric_limits::max()}; + caps.minImageExtent = {64, 64}; + caps.maxImageExtent = {4096, 4096}; + return caps; + } +} // namespace + +TEST(SwapchainExtentTest, WaylandUsesPhysicalFramebufferWithinSurfaceLimits) +{ + const auto caps = VariableExtentCapabilities(); + auto extent = DeviceSwapchain::ResolveExtent(caps, {2478, 1514}); + EXPECT_EQ(extent.width, 2478u); + EXPECT_EQ(extent.height, 1514u); + + extent = DeviceSwapchain::ResolveExtent(caps, {32, 8192}); + EXPECT_EQ(extent.width, 64u); + EXPECT_EQ(extent.height, 4096u); +} + +TEST(SwapchainExtentTest, FixedSurfaceExtentRemainsAuthoritative) +{ + auto caps = VariableExtentCapabilities(); + caps.currentExtent = {1920, 1080}; + const auto extent = DeviceSwapchain::ResolveExtent(caps, {2478, 1514}); + EXPECT_EQ(extent.width, 1920u); + EXPECT_EQ(extent.height, 1080u); +} + +TEST(SwapchainExtentTest, ZeroSizeIsDeferredRatherThanClampedToMinimum) +{ + auto caps = VariableExtentCapabilities(); + for (const VkExtent2D framebuffer : { + VkExtent2D{ 0, 0}, + VkExtent2D{ 0, 1514}, + VkExtent2D{2478, 0} + }) + { + const auto extent = DeviceSwapchain::ResolveExtent(caps, framebuffer); + EXPECT_EQ(extent.width, 0u); + EXPECT_EQ(extent.height, 0u); + } + caps.currentExtent = {0, 0}; + const auto extent = DeviceSwapchain::ResolveExtent(caps, {2478, 1514}); + EXPECT_EQ(extent.width, 0u); + EXPECT_EQ(extent.height, 0u); +} + +TEST(SwapchainExtentTest, ResizeSchedulesRecreationWithoutVulkanError) +{ + DeviceSwapchain swapchain{}; + swapchain.FramebufferExtent = {1500, 800}; + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + EXPECT_EQ(swapchain.FramebufferExtent.width, 2478u); + EXPECT_EQ(swapchain.FramebufferExtent.height, 1514u); + + swapchain.Recreation = RecreationState::None; + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::None); + + swapchain.UpdateFramebufferExtent(2480, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + swapchain.Recreation = RecreationState::None; + swapchain.UpdateFramebufferExtent(2480, 1516); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + + // A DPI-only framebuffer change still requests recreation. + swapchain.UpdateFramebufferExtent(3098, 1893); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); +} + +TEST(SwapchainExtentTest, ZeroSizeSkipsAcquisitionAndRestoreSchedulesRecreation) +{ + DeviceSwapchain swapchain{}; + swapchain.FramebufferExtent = {2478, 1514}; + swapchain.UpdateFramebufferExtent(0, 0); + // No device or GPU objects are needed: zero size must not touch Vulkan/fences. + swapchain.AcquireNextImage(0); + EXPECT_FALSE(swapchain.IsFrameValid()); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); + EXPECT_EQ(swapchain.FramebufferExtent.width, 2478u); + EXPECT_EQ(swapchain.FramebufferExtent.height, 1514u); +} + +TEST(SwapchainExtentTest, UnchangedExtentDoesNotClearAnAbortedAcquire) +{ + DeviceSwapchain swapchain{}; + swapchain.FramebufferExtent = {2478, 1514}; + swapchain.Recreation = RecreationState::FrameAborted; + swapchain.UpdateFramebufferExtent(2478, 1514); + EXPECT_EQ(swapchain.Recreation, RecreationState::FrameAborted); +} + +TEST(SwapchainExtentTest, InvalidFrameSkipsRecordingAndPresentation) +{ + VulkanDevice device{}; + CommandBufferManager command_buffers{}; + DeviceSwapchain swapchain{}; + ZEngine::Applications::AppRenderPipeline pipeline{}; + device.CommandBufferMgr = &command_buffers; + device.SwapchainPtr = &swapchain; + swapchain.Device = &device; + pipeline.Device = &device; + + // A zero-size framebuffer must not dereference GPU resources, including + // command pools, the render timeline, or CurrentCmdBuf. + EXPECT_FALSE(pipeline.BeginFrame()); + command_buffers.EnqueuedCommandBufferIndex = 1; + pipeline.EndFrame(); + EXPECT_EQ(command_buffers.EnqueuedCommandBufferIndex, 0u); + EXPECT_EQ(pipeline.CurrentCmdBuf, nullptr); + + // A concrete surface can also report zero size during recreation. In that + // case CurrentFrame exists, but ImageIndex is invalid and state is Pending. + FrameContext frame{}; + swapchain.CurrentFrame = &frame; + swapchain.Recreation = RecreationState::Pending; + command_buffers.EnqueuedCommandBufferIndex = 1; + pipeline.EndFrame(); + EXPECT_EQ(command_buffers.EnqueuedCommandBufferIndex, 0u); +} + +TEST(SwapchainExtentTest, AcquisitionWaitIsBounded) +{ + EXPECT_GT(DeviceSwapchain::ImageAcquireTimeoutNs, 0u); + EXPECT_LE(DeviceSwapchain::ImageAcquireTimeoutNs, 100'000'000u); +} + +TEST(SwapchainExtentTest, TimeoutAndNotReadyInvalidateTheOldImageWithoutRecreation) +{ + for (VkResult result : {VK_TIMEOUT, VK_NOT_READY}) + { + DeviceSwapchain swapchain{}; + FrameContext frame{}; + frame.ImageIndex = 2; + swapchain.RenderTimelineNextValue = 17; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, result, 0)); + EXPECT_FALSE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, std::numeric_limits::max()); + EXPECT_EQ(swapchain.Recreation, RecreationState::None); + EXPECT_EQ(swapchain.RenderTimelineNextValue, 17u); + + EXPECT_TRUE(swapchain.ApplyAcquireResult(frame, VK_SUCCESS, 1)); + EXPECT_TRUE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, 1u); + EXPECT_EQ(swapchain.Recreation, RecreationState::None); + } +} + +TEST(SwapchainExtentTest, SuboptimalAcquisitionHasAnImageAndSchedulesRecreation) +{ + DeviceSwapchain swapchain{}; + FrameContext frame{}; + EXPECT_TRUE(swapchain.ApplyAcquireResult(frame, VK_SUBOPTIMAL_KHR, 2)); + EXPECT_TRUE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, 2u); + EXPECT_EQ(swapchain.Recreation, RecreationState::Pending); +} + +TEST(SwapchainExtentTest, FailedAcquisitionNeverExposesAnImage) +{ + for (VkResult result : {VK_ERROR_OUT_OF_DATE_KHR, VK_ERROR_DEVICE_LOST, VK_ERROR_SURFACE_LOST_KHR, VK_ERROR_OUT_OF_HOST_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) + { + DeviceSwapchain swapchain{}; + FrameContext frame{}; + frame.ImageIndex = 2; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, result, 0)); + EXPECT_FALSE(swapchain.IsFrameValid()); + EXPECT_EQ(frame.ImageIndex, std::numeric_limits::max()); + EXPECT_EQ(swapchain.Recreation, RecreationState::FrameAborted); + } +} + +TEST(SwapchainExtentTest, SuccessWithAnInvalidImageIndexIsRejected) +{ + DeviceSwapchain swapchain{}; + FrameContext frame{}; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, VK_SUCCESS, swapchain.SwapchainImageCount)); + EXPECT_FALSE(swapchain.IsFrameValid()); +} + +TEST(SwapchainExtentTest, TimeoutSkipsGpuSubmissionAndCancelsPendingCallbacks) +{ + ZEngine::Core::Memory::MemoryManager memory{}; + memory.Initialize(ZMega(4ULL), {}); + VulkanDevice device{}; + CommandBufferManager command_buffers{}; + DeviceSwapchain swapchain{}; + ZEngine::Applications::AppRenderPipeline pipeline{}; + device.CommandBufferMgr = &command_buffers; + device.SwapchainPtr = &swapchain; + swapchain.Device = &device; + pipeline.Device = &device; + swapchain.RenderWorkSubmittedCallbacks.init(&memory.MainArena, 1); + uint32_t cancelled = 0; + swapchain.EnqueueRenderWorkSubmittedCallback([](void*, ZEngine::Rendering::Primitives::Semaphore*, uint64_t) { ADD_FAILURE() << "A timeout must not submit GPU work"; }, &cancelled, [](void* context) { ++*static_cast(context); }); + + FrameContext frame{}; + frame.ImageIndex = 2; + EXPECT_FALSE(swapchain.ApplyAcquireResult(frame, VK_TIMEOUT, 0)); + command_buffers.EnqueuedCommandBufferIndex = 1; + pipeline.EndFrame(); // No command buffer, semaphore, fence or GPU is needed. + EXPECT_EQ(command_buffers.EnqueuedCommandBufferIndex, 0u); + EXPECT_EQ(cancelled, 1u); + EXPECT_EQ(swapchain.RenderWorkSubmittedCallbacks.size(), 0u); + EXPECT_EQ(swapchain.RenderTimelineNextValue, 0u); + memory.Shutdown(); +} diff --git a/ZEngine/tests/Rendering/ZUIRenderScale_test.cpp b/ZEngine/tests/Rendering/ZUIRenderScale_test.cpp new file mode 100644 index 00000000..f3be4ec9 --- /dev/null +++ b/ZEngine/tests/Rendering/ZUIRenderScale_test.cpp @@ -0,0 +1,107 @@ +#include +#include + +using namespace ZEngine::Rendering::Renderers; +using namespace ZEngine::UI; + +namespace +{ + ZUIRenderPayload Payload(float width, float height) + { + ZUIRenderPayload payload{}; + payload.Scale[0] = 2.f / width; + payload.Scale[1] = 2.f / height; + payload.Translate[0] = payload.Translate[1] = -1.f; + return payload; + } + + ZUIDrawListCmd Clip(float x, float y, float width, float height) + { + ZUIDrawListCmd cmd{}; + cmd.ClipX = x; + cmd.ClipY = y; + cmd.ClipW = width; + cmd.ClipH = height; + return cmd; + } +} // namespace + +TEST(ZUIRenderScaleTest, ClipUsesActualSwapchainRatherThanWindowDpi) +{ + const auto payload = Payload(1982, 1211); + const auto cmd = Clip(280, 50, 100, 20); + // Regression: window framebuffer=2478x1514, but old swapchain=1500x800. + // The old 1.25 multiplier produced x=350; projected geometry starts at x=212. + const auto rect = payload.GetScissorRect(cmd, {1500, 800}); + EXPECT_EQ(rect.offset.x, 211); + EXPECT_EQ(rect.offset.y, 33); + EXPECT_EQ(rect.extent.width, 77u); + EXPECT_EQ(rect.extent.height, 14u); +} + +TEST(ZUIRenderScaleTest, FractionalDpiRoundsOutwardPerAxis) +{ + const auto rect = Payload(1000, 800).GetScissorRect(Clip(10.2f, 20.2f, 30.1f, 40.1f), {1250, 1001}); + EXPECT_EQ(rect.offset.x, 12); + EXPECT_EQ(rect.offset.y, 25); + EXPECT_EQ(rect.extent.width, 39u); + EXPECT_EQ(rect.extent.height, 51u); +} + +TEST(ZUIRenderScaleTest, NativeAndRetinaScissorsStayAligned) +{ + const auto payload = Payload(1500, 800); + const auto cmd = Clip(280, 50, 100, 20); + const auto native = payload.GetScissorRect(cmd, {1500, 800}); + EXPECT_EQ(native.offset.x, 280); + EXPECT_EQ(native.offset.y, 50); + EXPECT_EQ(native.extent.width, 100u); + EXPECT_EQ(native.extent.height, 20u); + + const auto retina = payload.GetScissorRect(cmd, {3000, 1600}); + EXPECT_EQ(retina.offset.x, 560); + EXPECT_EQ(retina.offset.y, 100); + EXPECT_EQ(retina.extent.width, 200u); + EXPECT_EQ(retina.extent.height, 40u); +} + +TEST(ZUIRenderScaleTest, FullScreenClipCoversRoundedFractionalFramebuffer) +{ + const auto rect = Payload(1982, 1211).GetScissorRect(Clip(0, 0, 1982, 1211), {2478, 1514}); + EXPECT_EQ(rect.offset.x, 0); + EXPECT_EQ(rect.offset.y, 0); + EXPECT_EQ(rect.extent.width, 2478u); + EXPECT_EQ(rect.extent.height, 1514u); +} + +TEST(ZUIRenderScaleTest, ClampsPartiallyAndFullyOffscreenClips) +{ + const auto payload = Payload(100, 100); + auto rect = payload.GetScissorRect(Clip(-10, -20, 50, 60), {200, 200}); + EXPECT_EQ(rect.offset.x, 0); + EXPECT_EQ(rect.offset.y, 0); + EXPECT_EQ(rect.extent.width, 80u); + EXPECT_EQ(rect.extent.height, 80u); + + rect = payload.GetScissorRect(Clip(90, 90, 50, 60), {200, 200}); + EXPECT_EQ(rect.offset.x, 180); + EXPECT_EQ(rect.offset.y, 180); + EXPECT_EQ(rect.extent.width, 20u); + EXPECT_EQ(rect.extent.height, 20u); + + rect = payload.GetScissorRect(Clip(110, 110, 10, 10), {200, 200}); + EXPECT_EQ(rect.extent.width, 0u); + EXPECT_EQ(rect.extent.height, 0u); + rect = payload.GetScissorRect(Clip(-50, -50, 10, 10), {200, 200}); + EXPECT_EQ(rect.extent.width, 0u); + EXPECT_EQ(rect.extent.height, 0u); +} + +TEST(ZUIRenderScaleTest, EmptyOrUninitializedPayloadDoesNotDraw) +{ + const auto payload = Payload(100, 100); + EXPECT_EQ(payload.GetScissorRect(Clip(0, 0, 0, 10), {200, 200}).extent.width, 0u); + EXPECT_EQ(payload.GetScissorRect(Clip(0, 0, 10, -1), {200, 200}).extent.height, 0u); + EXPECT_EQ(payload.GetScissorRect(Clip(0, 0, 10, 10), {0, 0}).extent.width, 0u); + EXPECT_EQ(ZUIRenderPayload{}.GetScissorRect(Clip(0, 0, 10, 10), {200, 200}).extent.width, 0u); +} From fcb499eaaeff93fae73a974728aa0ebb6adcd8a8 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Sun, 4 Oct 2026 00:11:52 +0900 Subject: [PATCH 22/27] fix(tests): avoid Windows swapchain test stack overflow --- .../tests/Rendering/SwapchainExtent_test.cpp | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ZEngine/tests/Rendering/SwapchainExtent_test.cpp b/ZEngine/tests/Rendering/SwapchainExtent_test.cpp index 1784807b..ede4f07c 100644 --- a/ZEngine/tests/Rendering/SwapchainExtent_test.cpp +++ b/ZEngine/tests/Rendering/SwapchainExtent_test.cpp @@ -3,6 +3,7 @@ #include #include #include +#include using namespace ZEngine::Hardwares; @@ -109,14 +110,15 @@ TEST(SwapchainExtentTest, UnchangedExtentDoesNotClearAnAbortedAcquire) TEST(SwapchainExtentTest, InvalidFrameSkipsRecordingAndPresentation) { - VulkanDevice device{}; + // Device-owned queues exceed the default 1 MB Windows stack. + auto device = std::make_unique(); CommandBufferManager command_buffers{}; DeviceSwapchain swapchain{}; ZEngine::Applications::AppRenderPipeline pipeline{}; - device.CommandBufferMgr = &command_buffers; - device.SwapchainPtr = &swapchain; - swapchain.Device = &device; - pipeline.Device = &device; + device->CommandBufferMgr = &command_buffers; + device->SwapchainPtr = &swapchain; + swapchain.Device = device.get(); + pipeline.Device = device.get(); // A zero-size framebuffer must not dereference GPU resources, including // command pools, the render timeline, or CurrentCmdBuf. @@ -199,14 +201,14 @@ TEST(SwapchainExtentTest, TimeoutSkipsGpuSubmissionAndCancelsPendingCallbacks) { ZEngine::Core::Memory::MemoryManager memory{}; memory.Initialize(ZMega(4ULL), {}); - VulkanDevice device{}; + auto device = std::make_unique(); CommandBufferManager command_buffers{}; DeviceSwapchain swapchain{}; ZEngine::Applications::AppRenderPipeline pipeline{}; - device.CommandBufferMgr = &command_buffers; - device.SwapchainPtr = &swapchain; - swapchain.Device = &device; - pipeline.Device = &device; + device->CommandBufferMgr = &command_buffers; + device->SwapchainPtr = &swapchain; + swapchain.Device = device.get(); + pipeline.Device = device.get(); swapchain.RenderWorkSubmittedCallbacks.init(&memory.MainArena, 1); uint32_t cancelled = 0; swapchain.EnqueueRenderWorkSubmittedCallback([](void*, ZEngine::Rendering::Primitives::Semaphore*, uint64_t) { ADD_FAILURE() << "A timeout must not submit GPU work"; }, &cancelled, [](void* context) { ++*static_cast(context); }); From e1d45a2a0feaca209dd78f0b41147931b057429a Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Sun, 4 Oct 2026 01:10:51 +0900 Subject: [PATCH 23/27] fix(rendering): submit upload batches before frame-slot changes --- ZEngine/ZEngine/Hardwares/VulkanDevice.h | 3 + .../Rendering/RenderResourceManager.cpp | 14 +- .../ZEngine/Rendering/RenderResourceManager.h | 7 +- ZEngine/tests/Rendering/GpuAllocatorTest.cpp | 195 +++++++++++++++++- 4 files changed, 209 insertions(+), 10 deletions(-) diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.h b/ZEngine/ZEngine/Hardwares/VulkanDevice.h index 74a96271..3f22ff96 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.h +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.h @@ -1,6 +1,8 @@ #pragma once #include +struct RRMUploadBatchTestHelper; + namespace ZEngine::Rendering::Renderers::Pipelines { struct IPipeline; @@ -477,6 +479,7 @@ namespace ZEngine::Hardwares private: friend struct CommandBuffer; + friend struct ::RRMUploadBatchTestHelper; VulkanLayer m_layer = {}; Core::Containers::UnorderedHashMap m_queue_map = {}; diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index ff061ff4..ffd9c9cb 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -317,6 +317,11 @@ namespace ZEngine::Rendering void RenderResourceManager::BeginFrame(uint32_t frame_index) { + // Initialization may leave a builtin-geometry batch open for slot 0. + // A resize or skipped acquire can make the first rendered slot different. + // Submit its copies before changing slots or retiring their staging data. + if (m_batch_mode && m_batch_frame_index != frame_index) + EndBatchUpload(); m_active_frame_index = static_cast(frame_index); RetireBatchStagings(); DispatchQueuedTextureDecodes(); @@ -556,8 +561,8 @@ namespace ZEngine::Rendering ZENGINE_VALIDATE_ASSERT(m_builtin_idx_cursor + idx_bytes <= BUILTIN_IDX_CAPACITY, "RRM::RegisterBuiltinGeometry: builtin index buffer out of space") // Called pre-render-thread (single-threaded init) — the batch this opens stays - // open until the first real frame's RRM::EndFrame, where any mesh uploads from - // that frame join it too. Builtin data goes into the pinned builtin buffers, not + // open until the first real frame joins it or submits it before switching + // frame slots. Builtin data goes into the pinned builtin buffers, not // the streaming global buffers, so a ResetGeometryBuffers never corrupts it. EnsureBatchOpen(static_cast(m_active_frame_index)); AppendToGlobalBuffer(m_builtin_vertex_buf, vtx_data, vtx_bytes, m_builtin_vtx_cursor, m_active_frame_index); @@ -656,6 +661,9 @@ namespace ZEngine::Rendering for (uint32_t i = 0; i < m_batch_frames.size(); ++i) { + // LastSignal covers the previous submission, not newly recorded copies. + if (m_batch_mode && i == m_batch_frame_index) + continue; BatchFrameState& frame = m_batch_frames[i]; if (frame.StagingCount == 0) continue; @@ -670,6 +678,8 @@ namespace ZEngine::Rendering void RenderResourceManager::EnsureBatchOpen(uint8_t frame_index) { + if (m_batch_mode && m_batch_frame_index != frame_index) + EndBatchUpload(); if (!m_batch_mode) BeginBatchUpload(frame_index); } diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.h b/ZEngine/ZEngine/Rendering/RenderResourceManager.h index 87bcc2a1..dfae9603 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.h +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.h @@ -24,6 +24,7 @@ // Forward declaration for the test-only helper that accesses RRM private members. struct RRMTestHelper; struct RRMDecodeTestHelper; +struct RRMUploadBatchTestHelper; namespace ZEngine::Hardwares { @@ -561,9 +562,8 @@ namespace ZEngine::Rendering void BeginBatchUpload(uint8_t frame_index); void EndBatchUpload(); - /// @brief Open the batch if not already open this frame. Idempotent — every join - /// point (mesh uploads, hot-reload swaps, UpdateBuffer staging, builtin - /// registration) calls this before recording so only the first one opens it. + /// @brief Reuse this slot's open batch, or submit the previous slot's batch + /// before opening a new one. Every upload joins through this method. void EnsureBatchOpen(uint8_t frame_index); /// @brief Free frame-index batch stagings once m_batch_timeline reaches LastSignal. @@ -598,6 +598,7 @@ namespace ZEngine::Rendering friend class GeometryStreamingManager; friend struct ::RRMTestHelper; // test-only — grants slot state access to streaming manager tests friend struct ::RRMDecodeTestHelper; + friend struct ::RRMUploadBatchTestHelper; Hardwares::VulkanDevice* m_device = nullptr; Core::VFS::AssetRegistry* m_registry = nullptr; diff --git a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp index d103e5e3..33f7e7a4 100644 --- a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp +++ b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp @@ -1,10 +1,13 @@ #include #include +#include #include #include +#include #include #include #include +#include using namespace ZEngine::Core::Memory; using namespace ZEngine::Logging; @@ -64,11 +67,15 @@ namespace device_extension_count = 1; #endif - VkDeviceCreateInfo device_info = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; - device_info.queueCreateInfoCount = 1; - device_info.pQueueCreateInfos = &queue_info; - device_info.enabledExtensionCount = device_extension_count; - device_info.ppEnabledExtensionNames = device_extension_count > 0 ? device_extensions : nullptr; + VkDeviceCreateInfo device_info = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + device_info.queueCreateInfoCount = 1; + device_info.pQueueCreateInfos = &queue_info; + device_info.enabledExtensionCount = device_extension_count; + device_info.ppEnabledExtensionNames = device_extension_count > 0 ? device_extensions : nullptr; + + VkPhysicalDeviceVulkan13Features features13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES, .synchronization2 = VK_TRUE}; + VkPhysicalDeviceVulkan12Features features12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, .pNext = &features13, .timelineSemaphore = VK_TRUE}; + device_info.pNext = &features12; return vkCreateDevice(PhysicalDevice, &device_info, nullptr, &Device) == VK_SUCCESS; } @@ -301,3 +308,181 @@ TEST_F(GpuAllocatorTest, StagingRingSharesDeclaredPool) allocator().FreeBuffer(one_shot); } + +// Reuse the headless Vulkan fixture to exercise real upload submissions without +// creating a window, swapchain images, or the rest of the resource manager. +struct RRMUploadBatchTestHelper +{ + static void Initialize(ZEngine::Rendering::RenderResourceManager& manager, ZEngine::Hardwares::VulkanDevice& device, ZEngine::Hardwares::CommandBufferManager& commands, ZEngine::Rendering::Primitives::Semaphore* timeline, ArenaAllocator& arena, VkQueue queue) + { + device.m_queue_map.init(&arena, 2); + device.m_queue_map.insert(ZEngine::Rendering::QueueType::GRAPHIC_QUEUE, queue); + manager.m_device = &device; + manager.m_upload_cmd_mgr = &commands; + manager.m_batch_timeline = timeline; + manager.m_batch_frames.init(&arena, 3, 3); + for (auto& frame : manager.m_batch_frames) + frame = {}; + } + + static void Append(ZEngine::Rendering::RenderResourceManager& manager, BufferView& target, uint8_t frame_index, uint32_t value, VkDeviceSize offset) + { + manager.EnsureBatchOpen(frame_index); + manager.AppendToGlobalBuffer(target, &value, sizeof(value), offset, frame_index); + } + + static bool IsOpen(const ZEngine::Rendering::RenderResourceManager& manager) + { + return manager.m_batch_mode; + } + + static uint32_t StagingCount(const ZEngine::Rendering::RenderResourceManager& manager, uint32_t frame_index) + { + return manager.m_batch_frames[frame_index].StagingCount; + } + + static uint64_t LastSignal(const ZEngine::Rendering::RenderResourceManager& manager, uint32_t frame_index) + { + return manager.m_batch_frames[frame_index].LastSignal; + } +}; + +class RenderResourceManagerBatchTest : public GpuAllocatorTest +{ +protected: + MemoryManager Memory{}; + std::unique_ptr Device = std::make_unique(); + std::unique_ptr Manager = std::make_unique(); + ZEngine::Hardwares::DeviceSwapchain Swapchain{}; + ZEngine::Hardwares::CommandBufferManager UploadCommands{}; + std::array, 3> Commands{}; + std::unique_ptr Timeline{}; + VkCommandPool Pool = VK_NULL_HANDLE; + BufferView Target{}; + bool Initialized = false; + + void SetUp() override + { + ASSERT_NE(s_allocator, nullptr); + Memory.Initialize(ZMega(4), {}); + auto& arena = Memory.MainArena; + Device->Arena = &arena; + Device->LogicalDevice = s_vk->Device; + Device->GraphicFamilyIndex = 0; + Device->SwapchainPtr = &Swapchain; + Device->GpuMem.Allocator = s_allocator->Allocator; + + VkQueue queue = VK_NULL_HANDLE; + vkGetDeviceQueue(s_vk->Device, 0, 0, &queue); + VkCommandPoolCreateInfo pool_info = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = 0}; + ASSERT_EQ(vkCreateCommandPool(s_vk->Device, &pool_info, nullptr, &Pool), VK_SUCCESS); + UploadCommands.Device = Device.get(); + UploadCommands.TotalThreadCount = 1; + const uint32_t stride = UploadCommands.MaxBufferPerPool * UploadCommands.MaxBufferPerPool; + UploadCommands.InstantGraphicsCommandBuffers.init(&arena, 3 * stride, 3 * stride); + for (uint32_t frame = 0; frame < Commands.size(); ++frame) + { + Commands[frame] = std::make_unique(Device.get(), Pool, ZEngine::Rendering::QueueType::GRAPHIC_QUEUE, true); + UploadCommands.InstantGraphicsCommandBuffers[frame * stride] = Commands[frame].get(); + } + Timeline = std::make_unique(Device.get(), true); + RRMUploadBatchTestHelper::Initialize(*Manager, *Device, UploadCommands, Timeline.get(), arena, queue); + Target = allocator().AllocateBuffer(64, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "batch_test_target"); + Initialized = true; + } + + void TearDown() override + { + if (Initialized) + { + Manager->EndFrame(); + vkDeviceWaitIdle(s_vk->Device); + Manager->BeginFrame(0); // Retire all submitted staging allocations. + allocator().FreeBuffer(Target); + } + for (auto& command : Commands) + command.reset(); + Timeline.reset(); + if (Device->LogicalDevice) + Device->PendingFree.Drain(&Device->GpuMem, Device->LogicalDevice, UINT64_MAX); + if (Pool) + vkDestroyCommandPool(s_vk->Device, Pool, nullptr); + } + + void Append(uint8_t frame, uint32_t value, VkDeviceSize offset = 0) + { + RRMUploadBatchTestHelper::Append(*Manager, Target, frame, value, offset); + } + + uint32_t Read(uint64_t signal_value, VkDeviceSize offset = 0) + { + Timeline->Wait(signal_value); + uint32_t value = 0; + EXPECT_EQ(vmaCopyAllocationToMemory(allocator().Allocator, Target.Allocation, offset, &value, sizeof(value)), VK_SUCCESS); + return value; + } +}; + +TEST_F(RenderResourceManagerBatchTest, StartupResizeSubmitsInitializationBeforeSwitchingSlots) +{ + Append(0, 42); + Manager->BeginFrame(1); // First acquired slot changes after startup recreation. + ASSERT_FALSE(RRMUploadBatchTestHelper::IsOpen(*Manager)); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 1u); + + ZEngine::Hardwares::AsyncGPUOperationHandle operation{}; + ASSERT_TRUE(Device->AsyncGPUOperations.pop(operation)); + EXPECT_EQ(operation.Timeline, Timeline.get()); + EXPECT_EQ(operation.SignalValue, 1u); + EXPECT_EQ(operation.StageFlags, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT); + + Append(1, 99, sizeof(uint32_t)); + Manager->EndFrame(); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 1), 2u); + EXPECT_EQ(Read(2), 42u); + EXPECT_EQ(Read(2, sizeof(uint32_t)), 99u); +} + +TEST_F(RenderResourceManagerBatchTest, SameSlotInitializationJoinsTheFirstFrame) +{ + Append(0, 42); + Manager->BeginFrame(0); + EXPECT_TRUE(RRMUploadBatchTestHelper::IsOpen(*Manager)); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 1u); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 0u); + + Append(0, 99, sizeof(uint32_t)); + Manager->EndFrame(); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 1u); + EXPECT_EQ(Read(1), 42u); + EXPECT_EQ(Read(1, sizeof(uint32_t)), 99u); +} + +TEST_F(RenderResourceManagerBatchTest, UploadJoinSubmitsThePreviousSlotsBatch) +{ + Append(0, 42); + Append(2, 99, sizeof(uint32_t)); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 0), 1u); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 2), 1u); + Manager->EndFrame(); + EXPECT_EQ(RRMUploadBatchTestHelper::LastSignal(*Manager, 2), 2u); + EXPECT_EQ(Read(2), 42u); + EXPECT_EQ(Read(2, sizeof(uint32_t)), 99u); +} + +TEST_F(RenderResourceManagerBatchTest, PreviousSignalNeverRetiresAnOpenBatchsStaging) +{ + Append(0, 42); + Manager->EndFrame(); + EXPECT_EQ(Read(1), 42u); + Manager->BeginFrame(0); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 0u); + + Append(0, 99); + Manager->BeginFrame(0); // LastSignal=1 is complete, but the new copy is not submitted. + ASSERT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 1u); + Manager->EndFrame(); + EXPECT_EQ(Read(2), 99u); + Manager->BeginFrame(2); + EXPECT_EQ(RRMUploadBatchTestHelper::StagingCount(*Manager, 0), 0u); +} From ba4113bb318dc6294fd2fa4c4ab82bb346e313e1 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Sun, 4 Oct 2026 13:32:24 +0900 Subject: [PATCH 24/27] fix(tests): explicitly initialize upload batch frame state --- ZEngine/tests/Rendering/GpuAllocatorTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp index 33f7e7a4..72dffbae 100644 --- a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp +++ b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp @@ -322,7 +322,7 @@ struct RRMUploadBatchTestHelper manager.m_batch_timeline = timeline; manager.m_batch_frames.init(&arena, 3, 3); for (auto& frame : manager.m_batch_frames) - frame = {}; + frame = ZEngine::Rendering::RenderResourceManager::BatchFrameState{}; } static void Append(ZEngine::Rendering::RenderResourceManager& manager, BufferView& target, uint8_t frame_index, uint32_t value, VkDeviceSize offset) From e7812bc3db860015e4771bd5eadcce32552fa74e Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 5 Oct 2026 17:54:41 +0900 Subject: [PATCH 25/27] fix(importers): harden asset cooking and scratch ownership Validate model inputs and output paths, serialize shared importer scratch, and clone material strings into persistent storage. Preserve material and texture identities, publish cooked assets and metadata through checked atomic writes, and propagate persistence failures to editor completion. Add importer output, lifetime, concurrency, and VFS regression coverage. Follow-up gaps remain in editor completion scratch reclamation, metadata serialized-size limits, and scanner error handling. --- Tetragrama/Panels/AssetImporterPanel.cpp | 41 +- .../Serializers/EditorSceneSerializer.cpp | 11 +- ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp | 103 +- ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h | 2 +- ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp | 9 +- ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp | 41 + ZEngine/ZEngine/Core/VFS/VFSFileIO.h | 8 + ZEngine/ZEngine/Importers/AssetCodec.cpp | 275 ++- ZEngine/ZEngine/Importers/AssetCodec.h | 40 +- ZEngine/ZEngine/Importers/AssetTypes.h | 51 +- ZEngine/ZEngine/Importers/AssimpImporter.cpp | 224 +-- ZEngine/ZEngine/Importers/AssimpImporter.h | 23 +- ZEngine/ZEngine/Importers/FbxImporter.cpp | 186 +- ZEngine/ZEngine/Importers/FbxImporter.h | 3 +- ZEngine/ZEngine/Importers/GltfImporter.cpp | 654 +++++-- ZEngine/ZEngine/Importers/GltfImporter.h | 8 +- ZEngine/ZEngine/Importers/IAssetImporter.h | 3 +- ZEngine/ZEngine/Importers/TextureImporter.cpp | 1 + ZEngine/ZEngine/Importers/TextureImporter.h | 4 + ZEngine/ZEngine/Managers/AssetManager.cpp | 2 +- .../Rendering/AssetMaterialLifetime_test.cpp | 62 + .../Rendering/ModelImporterOutput_test.cpp | 1661 +++++++++++++++++ ZEngine/tests/VFS/vfs_disk_test.cpp | 28 + ZEngine/tests/VFS/vfs_meta_test.cpp | 56 +- 24 files changed, 2865 insertions(+), 631 deletions(-) create mode 100644 ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp create mode 100644 ZEngine/ZEngine/Core/VFS/VFSFileIO.h create mode 100644 ZEngine/tests/Rendering/AssetMaterialLifetime_test.cpp create mode 100644 ZEngine/tests/Rendering/ModelImporterOutput_test.cpp diff --git a/Tetragrama/Panels/AssetImporterPanel.cpp b/Tetragrama/Panels/AssetImporterPanel.cpp index 61ee8590..11349a51 100644 --- a/Tetragrama/Panels/AssetImporterPanel.cpp +++ b/Tetragrama/Panels/AssetImporterPanel.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -846,13 +847,23 @@ namespace Tetragrama::Panels if (material.MaterialUUID.is_nil()) continue; - auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); - ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; - meta.AssetUUID = material.MaterialUUID; + auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); + if (meta_result.Failed() && meta_result.Error() != ZEngine::Core::VFS::VFSError::NotFound && meta_result.Error() != ZEngine::Core::VFS::VFSError::Corrupted) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to read metadata for '{}' (VFS error {})", mat_path, static_cast(meta_result.Error()))); + return; + } + ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; + meta.AssetUUID = material.MaterialUUID; secure_strncpy(meta.SourcePath, sizeof(meta.SourcePath), self->m_path_buf, sizeof(meta.SourcePath) - 1); secure_strncpy(meta.ArtifactPath, sizeof(meta.ArtifactPath), mat_path, sizeof(meta.ArtifactPath) - 1); secure_strncpy(meta.ImporterName, sizeof(meta.ImporterName), "GltfImporter/AssimpImporter", sizeof(meta.ImporterName) - 1); - ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + auto write = ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + if (write.Failed()) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to persist metadata for '{}' (VFS error {})", mat_path, static_cast(write.Error()))); + return; + } } } } @@ -889,8 +900,13 @@ namespace Tetragrama::Panels auto rel = VFSPath::Parse(mesh_path); if (rel.Succeeded()) { - auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); - ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; + auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); + if (meta_result.Failed() && meta_result.Error() != ZEngine::Core::VFS::VFSError::NotFound && meta_result.Error() != ZEngine::Core::VFS::VFSError::Corrupted) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to read metadata for '{}' (VFS error {})", mesh_path, static_cast(meta_result.Error()))); + return; + } + ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; // Sync to the file's own embedded UUID — otherwise a nil // AssetUUID gets locked in forever (#755). if (has_header) @@ -898,7 +914,12 @@ namespace Tetragrama::Panels secure_strncpy(meta.SourcePath, sizeof(meta.SourcePath), self->m_path_buf, sizeof(meta.SourcePath) - 1); secure_strncpy(meta.ArtifactPath, sizeof(meta.ArtifactPath), mesh_path, sizeof(meta.ArtifactPath) - 1); secure_strncpy(meta.ImporterName, sizeof(meta.ImporterName), "GltfImporter/AssimpImporter", sizeof(meta.ImporterName) - 1); - ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + auto write = ZEngine::Core::VFS::MetaFileIO::Write(*vfs, rel.Value(), meta); + if (write.Failed()) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to persist metadata for '{}' (VFS error {})", mesh_path, static_cast(write.Error()))); + return; + } } } @@ -986,8 +1007,10 @@ namespace Tetragrama::Panels void AssetImporterPanel::CompleteImportErrorOnMainThread(void* ctx, std::string_view err) { - auto* self = reinterpret_cast(ctx); - char msg[512]; + auto* self = reinterpret_cast(ctx); + self->m_add_to_scene = false; + self->m_instance_name[0] = '\0'; + char msg[512]; snprintf(msg, sizeof(msg), "Error: %.*s", (int) err.size(), err.data()); self->PushLog(msg, kRed[0], kRed[1], kRed[2]); char fn[256] = {}; diff --git a/Tetragrama/Serializers/EditorSceneSerializer.cpp b/Tetragrama/Serializers/EditorSceneSerializer.cpp index ca6b5116..76b3e174 100644 --- a/Tetragrama/Serializers/EditorSceneSerializer.cpp +++ b/Tetragrama/Serializers/EditorSceneSerializer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -106,12 +107,16 @@ namespace Tetragrama::Serializers cook_cfg.VFS = reinterpret_cast(ZEngine::Engine::GetContext()->VFS); auto output = ZEngine::Importers::AssetCodec::SerializeMeshAssetFile(scratch.Arena, *mesh, *hierarchy, cook_cfg); - if (!output.Path.empty()) + if (output.Succeeded()) { // Register the artifact path so future saves don't re-cook - ZEngine::Helpers::secure_strncpy(const_cast(rec)->Meta.ArtifactPath, MAX_FILE_PATH_COUNT, output.Path.c_str(), output.Path.size()); + ZEngine::Helpers::secure_strncpy(const_cast(rec)->Meta.ArtifactPath, MAX_FILE_PATH_COUNT, output.Value().Path.c_str(), output.Value().Path.size()); - scene->PushAssetFile(output); + scene->PushAssetFile(output.Value()); + } + else + { + ZENGINE_CORE_ERROR("Failed to cook '{}/{}' (VFS error {})", cook_cfg.OutputAssetsPath.c_str(), output_file, static_cast(output.Error())) } } ZReleaseScratch(scratch); diff --git a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp index 9f938184..c340e0e0 100644 --- a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp +++ b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp @@ -1,10 +1,11 @@ #include +#include #include #include #include #include #include -#include +#include #include namespace ZEngine::Core::VFS @@ -22,33 +23,26 @@ namespace ZEngine::Core::VFS static void CopyStr(const std::string& src, char* dst, size_t cap) { size_t n = src.size() < cap - 1 ? src.size() : cap - 1; - std::memcpy(dst, src.data(), n); + Helpers::secure_memcpy(dst, cap, src.data(), n); dst[n] = '\0'; } } // namespace // MetaFileIO - VFSPath MetaFileIO::MetaPathFor(const VFSPath& asset_path) + VFSResult MetaFileIO::MetaPathFor(const VFSPath& asset_path) { - char buf[MAX_FILE_PATH_COUNT] = {}; - const char* raw = asset_path.CStr(); - size_t len = Helpers::secure_strlen(raw); - size_t avail = MAX_FILE_PATH_COUNT - 1; - size_t copy = len < avail ? len : avail; - Helpers::secure_memcpy(buf, MAX_FILE_PATH_COUNT, raw, copy); - const char suffix[] = ".meta"; - size_t suf_len = sizeof(suffix) - 1; - if (copy + suf_len < MAX_FILE_PATH_COUNT) - Helpers::secure_memcpy(buf + copy, MAX_FILE_PATH_COUNT - copy, suffix, suf_len + 1); - return VFSPath::Parse(buf).Value(); + if (!asset_path.IsValid() || asset_path.IsRoot()) + return VFSResult::Fail(VFSError::InvalidPath); + return VFSPath::Parse((std::string(asset_path.CStr()) + ".meta").c_str()); } VFSResult MetaFileIO::Read(IVFSContext& ctx, const VFSPath& asset_path) { - VFSPath meta_path = MetaPathFor(asset_path); - - auto open_result = ctx.Open(meta_path, VFSOpenFlags::Read); + auto meta_path = MetaPathFor(asset_path); + if (meta_path.Failed()) + return VFSResult::Fail(meta_path.Error()); + auto open_result = ctx.Open(meta_path.Value(), VFSOpenFlags::Read); if (open_result.Failed()) return VFSResult::Fail(open_result.Error()); @@ -74,21 +68,23 @@ namespace ZEngine::Core::VFS if (read_result.Failed()) return VFSResult::Fail(read_result.Error()); + if (read_result.Value() != size) + return VFSResult::Fail(VFSError::Corrupted); buf[size] = '\0'; auto j = nlohmann::json::parse(reinterpret_cast(buf), nullptr, /*allow_exceptions=*/false); - if (j.is_discarded()) + if (j.is_discarded() || !j.is_object()) return VFSResult::Fail(VFSError::Corrupted); MetaFileData out{}; - if (j.contains("uuid") && j["uuid"].is_string()) - { - auto parsed = uuids::uuid::from_string(j["uuid"].get()); - if (parsed.has_value()) - out.AssetUUID = parsed.value(); - } + if (!j.contains("uuid") || !j["uuid"].is_string()) + return VFSResult::Fail(VFSError::Corrupted); + const auto parsed = uuids::uuid::from_string(j["uuid"].get()); + if (!parsed.has_value() || parsed->is_nil()) + return VFSResult::Fail(VFSError::Corrupted); + out.AssetUUID = *parsed; if (j.contains("importer") && j["importer"].is_string()) CopyStr(j["importer"].get(), out.ImporterName, sizeof(out.ImporterName)); @@ -111,7 +107,7 @@ namespace ZEngine::Core::VFS { if (out.SettingsCount >= META_MAX_SETTINGS) break; - if (!s.contains("key") || !s.contains("value")) + if (!s.contains("key") || !s.contains("value") || !s["key"].is_string() || !s["value"].is_string()) continue; auto& kv = out.Settings[out.SettingsCount++]; CopyStr(s["key"].get(), kv.Key, sizeof(kv.Key)); @@ -125,6 +121,11 @@ namespace ZEngine::Core::VFS VFSResult MetaFileIO::Write(IVFSContext& ctx, const VFSPath& asset_path, const MetaFileData& data) { + auto meta_path = MetaPathFor(asset_path); + if (meta_path.Failed()) + return VFSResult::Fail(meta_path.Error()); + if (data.AssetUUID.is_nil() || data.SettingsCount > META_MAX_SETTINGS) + return VFSResult::Fail(VFSError::Corrupted); nlohmann::json j; j["uuid"] = uuids::to_string(data.AssetUUID); j["importer"] = data.ImporterName; @@ -142,44 +143,17 @@ namespace ZEngine::Core::VFS }); } - std::string serialized = j.dump(4); - - // Build tmp path: .tmp - VFSPath meta_path = MetaPathFor(asset_path); - char tmp_buf[MAX_FILE_PATH_COUNT] = {}; - const char* meta_raw = meta_path.CStr(); - size_t meta_len = Helpers::secure_strlen(meta_raw); - size_t avail = MAX_FILE_PATH_COUNT - 1; - size_t copy = meta_len < avail ? meta_len : avail; - Helpers::secure_memcpy(tmp_buf, MAX_FILE_PATH_COUNT, meta_raw, copy); - const char tmp_suffix[] = ".tmp"; - size_t suf_len = sizeof(tmp_suffix) - 1; - if (copy + suf_len < MAX_FILE_PATH_COUNT) - Helpers::secure_memcpy(tmp_buf + copy, MAX_FILE_PATH_COUNT - copy, tmp_suffix, suf_len + 1); - VFSPath tmp_path = VFSPath::Parse(tmp_buf).Value(); - - auto open_result = ctx.Open(tmp_path, VFSOpenFlags::Write | VFSOpenFlags::Create | VFSOpenFlags::Truncate); - if (open_result.Failed()) - return VFSResult::Fail(open_result.Error()); - - IVFSFile* file = open_result.Value(); - const auto* bytes = reinterpret_cast(serialized.data()); - auto write_result = file->Write({bytes, serialized.size()}, 0); - auto flush_result = file->Flush(); - file->Close(); - ctx.Close(file); - - if (write_result.Failed()) - return VFSResult::Fail(write_result.Error()); - if (flush_result.Failed()) - return VFSResult::Fail(flush_result.Error()); - - return ctx.Rename(tmp_path, meta_path); + const std::string serialized = j.dump(4); + return WriteFileAtomically(ctx, meta_path.Value(), {reinterpret_cast(serialized.data()), serialized.size()}); } VFSResult MetaFileIO::GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash) { - auto read_result = Read(ctx, asset_path); + // Scanner and importer callers must not mint different identities for + // the same absent sidecar. Writes remain atomic; serialize this RMW. + static std::mutex identity_mutex; + std::lock_guard identity_lock(identity_mutex); + auto read_result = Read(ctx, asset_path); if (read_result.Succeeded()) { @@ -194,10 +168,15 @@ namespace ZEngine::Core::VFS existing.SourceHash = current_hash; existing.LastImportTimeNs = NowNs(); existing.Status = ImportStatus::Stale; - Write(ctx, asset_path, existing); // best-effort; ignore failure + auto write = Write(ctx, asset_path, existing); + if (write.Failed()) + return VFSResult::Fail(write.Error()); return VFSResult::Ok(existing); } + if (read_result.Error() != VFSError::NotFound && read_result.Error() != VFSError::Corrupted) + return VFSResult::Fail(read_result.Error()); + // No .meta or corrupt .meta — generate a fresh UUID. MetaFileData fresh{}; { @@ -210,7 +189,9 @@ namespace ZEngine::Core::VFS fresh.SourceHash = current_hash; fresh.LastImportTimeNs = NowNs(); fresh.Status = ImportStatus::New; - Write(ctx, asset_path, fresh); + auto write = Write(ctx, asset_path, fresh); + if (write.Failed()) + return VFSResult::Fail(write.Error()); return VFSResult::Ok(fresh); } diff --git a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h index f7bc262f..8b6e9de1 100644 --- a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h +++ b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h @@ -9,7 +9,7 @@ namespace ZEngine::Core::VFS struct MetaFileIO { // Returns the sidecar path for an asset: "/project/mesh.glb" -> "/project/mesh.glb.meta" - static VFSPath MetaPathFor(const VFSPath& asset_path); + static VFSResult MetaPathFor(const VFSPath& asset_path); // Reads the .meta sidecar from the VFS. Returns Fail if the file is absent or malformed. // Status on the returned MetaFileData is always ImportStatus::Unknown — caller sets it. diff --git a/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp b/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp index 42c3d3ec..79cd90a2 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSDiskBackend.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -328,10 +329,12 @@ namespace ZEngine::Core::VFS file->m_handle = CreateFileA(native, access, FILE_SHARE_READ, nullptr, creation, FILE_ATTRIBUTE_NORMAL, nullptr); if (file->m_handle == INVALID_HANDLE_VALUE) { + const DWORD native_error = GetLastError(); + const auto error = native_error == ERROR_FILE_NOT_FOUND || native_error == ERROR_PATH_NOT_FOUND ? VFSError::NotFound : native_error == ERROR_ACCESS_DENIED ? VFSError::PermissionDenied : VFSError::IOError; file->~VFSDiskFile(); std::lock_guard lock(m_file_pool_mutex); m_file_pool.Free(file); - return VFSResult::Fail(VFSError::NotFound); + return VFSResult::Fail(error); } LARGE_INTEGER sz; if (GetFileSizeEx(file->m_handle, &sz)) @@ -349,10 +352,12 @@ namespace ZEngine::Core::VFS file->m_fd = ::open(native, oflags, 0644); if (file->m_fd < 0) { + const int native_error = errno; + const auto error = native_error == ENOENT ? VFSError::NotFound : native_error == EACCES || native_error == EPERM ? VFSError::PermissionDenied : native_error == ENOTDIR ? VFSError::NotADirectory : native_error == EISDIR ? VFSError::NotAFile : VFSError::IOError; file->~VFSDiskFile(); std::lock_guard lock(m_file_pool_mutex); m_file_pool.Free(file); - return VFSResult::Fail(VFSError::NotFound); + return VFSResult::Fail(error); } struct stat st; if (::fstat(file->m_fd, &st) == 0) diff --git a/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp b/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp new file mode 100644 index 00000000..b2d53108 --- /dev/null +++ b/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + +namespace ZEngine::Core::VFS +{ + VFSResult WriteFileAtomically(IVFSContext& ctx, const VFSPath& path, Containers::ArrayView data) + { + if (!path.IsValid() || path.IsRoot()) + return VFSResult::Fail(VFSError::InvalidPath); + + std::random_device random; + uuids::basic_uuid_random_generator generator(random); + const auto temporary = path.Parent().Append(("." + uuids::to_string(generator()) + ".tmp").c_str()); + if (temporary.Failed()) + return VFSResult::Fail(temporary.Error()); + const auto& tmp_path = temporary.Value(); + auto fail = [&](VFSError error) { + (void) ctx.Remove(tmp_path); + return VFSResult::Fail(error); + }; + auto open = ctx.Open(tmp_path, VFSOpenFlags::Write | VFSOpenFlags::Create | VFSOpenFlags::Truncate); + if (open.Failed()) + return fail(open.Error()); + auto* file = open.Value(); + auto write = file->Write(data, 0); + auto flush = file->Flush(); + auto close = file->Close(); + ctx.Close(file); + if (write.Failed()) + return fail(write.Error()); + if (write.Value() != data.size()) + return fail(VFSError::IOError); + if (flush.Failed()) + return fail(flush.Error()); + if (close.Failed()) + return fail(close.Error()); + auto rename = ctx.Rename(tmp_path, path); + return rename.Failed() ? fail(rename.Error()) : rename; + } +} // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/VFSFileIO.h b/ZEngine/ZEngine/Core/VFS/VFSFileIO.h new file mode 100644 index 00000000..2492d6cb --- /dev/null +++ b/ZEngine/ZEngine/Core/VFS/VFSFileIO.h @@ -0,0 +1,8 @@ +#pragma once +#include + +namespace ZEngine::Core::VFS +{ + // Publish using a unique temporary sibling on the same mount/filesystem. + [[nodiscard]] VFSResult WriteFileAtomically(IVFSContext& ctx, const VFSPath& path, Containers::ArrayView data); +} // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Importers/AssetCodec.cpp b/ZEngine/ZEngine/Importers/AssetCodec.cpp index 9747b75a..0329589e 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.cpp +++ b/ZEngine/ZEngine/Importers/AssetCodec.cpp @@ -1,23 +1,29 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include #include #include #include #include +#include using namespace uuids; using namespace ZEngine::Helpers; using namespace ZEngine::Core::Containers; using namespace ZEngine::Core::Maths; +using ZEngine::Core::VFS::VFSError; using ZEngine::Core::VFS::VFSPath; +using ZEngine::Core::VFS::VFSResult; namespace ZEngine::Importers::AssetCodec { @@ -34,65 +40,139 @@ namespace ZEngine::Importers::AssetCodec } } // namespace - Core::VFS::VFSResult WriteFileAtomically(Core::VFS::IVFSContext& ctx, const VFSPath& out_path, ArrayView data) + VFSResult MakeOutputPath(const char* directory, const char* filename) { - constexpr char tmp_suffix[] = ".tmp"; - constexpr size_t suffix_len = sizeof(tmp_suffix) - 1; - if (out_path.Length() + suffix_len >= MAX_FILE_PATH_COUNT) - return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); - - char tmp_buf[MAX_FILE_PATH_COUNT] = {}; - const char* raw = out_path.CStr(); - size_t copy_len = out_path.Length(); - secure_memcpy(tmp_buf, MAX_FILE_PATH_COUNT, raw, copy_len); - secure_memcpy(tmp_buf + copy_len, MAX_FILE_PATH_COUNT - copy_len, tmp_suffix, sizeof(tmp_suffix)); - auto tmp_result = VFSPath::Parse(tmp_buf); - if (tmp_result.Failed()) - return Core::VFS::VFSResult::Fail(tmp_result.Error()); - auto tmp_path = tmp_result.Value(); - - auto open_result = ctx.Open(tmp_path, Core::VFS::VFSOpenFlags::Write | Core::VFS::VFSOpenFlags::Create | Core::VFS::VFSOpenFlags::Truncate); - if (open_result.Failed()) - return Core::VFS::VFSResult::Fail(open_result.Error()); + if (!filename || !filename[0]) + return VFSResult::Fail(VFSError::InvalidPath); + auto dir = VFSPath::Parse(directory); + if (dir.Failed()) + return VFSResult::Fail(dir.Error()); + auto path = dir.Value().Append(filename); + if (path.Failed()) + return path; + if (path.Value().Parent() != dir.Value()) + return VFSResult::Fail(VFSError::InvalidPath); + // Cooked assets also receive a .meta sidecar. + constexpr size_t metadata_suffix_size = sizeof(".meta") - 1; + if (path.Value().Length() + metadata_suffix_size >= MAX_FILE_PATH_COUNT) + return VFSResult::Fail(VFSError::InvalidPath); + // Account for the unique temporary sibling, even when the destination + // has a shorter name. Check without generating a UUID or touching VFS. + auto temporary = dir.Value().Append(".00000000-0000-0000-0000-000000000000.tmp"); + if (temporary.Failed()) + return temporary; + return path; + } - Core::VFS::IVFSFile* file = open_result.Value(); - auto w = file->Write(data, 0); - auto flush = file->Flush(); - file->Close(); - ctx.Close(file); + VFSResult ValidateImportConfiguration(const ImportConfiguration& config) + { + if (!config.VFS) + return VFSResult::Fail(VFSError::Unsupported); + auto mesh_path = MakeOutputPath(config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str()); + if (mesh_path.Failed()) + return mesh_path; + if (config.Options.ImportMaterials) + { + auto materials = MakeOutputPath(config.OutputMaterialPath.c_str(), config.AssetName.c_str()); + if (materials.Failed()) + return materials; + if (config.Options.ImportTextures) + { + auto textures = MakeOutputPath(config.OutputTextureFilesPath.c_str(), config.AssetName.c_str()); + if (textures.Failed()) + return textures; + } + } + return mesh_path; + } - auto fail = [&ctx, &tmp_path](Core::VFS::VFSError error) { - ctx.Remove(tmp_path); - return Core::VFS::VFSResult::Fail(error); - }; - if (w.Failed()) - return fail(w.Error()); - if (w.Value() != data.size()) - return fail(Core::VFS::VFSError::IOError); - if (flush.Failed()) - return fail(flush.Error()); + static VFSResult PrepareOutputPath(Core::VFS::IVFSContext* vfs, VFSResult path) + { + if (!vfs) + return VFSResult::Fail(VFSError::Unsupported); + if (path.Failed()) + return path; + const auto dir = path.Value().Parent(); + auto create = vfs->CreateDir(dir); + if (create.Failed() && create.Error() != VFSError::AlreadyExists) + return VFSResult::Fail(create.Error()); + return path; + } - return ctx.Rename(tmp_path, out_path); + VFSResult CopyTextureFiles(ArrayView textures, const ImportConfiguration& config) + { + if (textures.size() == 0) + return VFSResult::Ok(); + if (!config.VFS) + return VFSResult::Fail(VFSError::Unsupported); + auto directory = MakeOutputPath(config.OutputTextureFilesPath.c_str(), config.AssetName.c_str()); + if (directory.Failed()) + return VFSResult::Fail(directory.Error()); + auto create = config.VFS->CreateDir(directory.Value()); + if (create.Failed() && create.Error() != VFSError::AlreadyExists) + return create; + for (size_t index = 0; index < textures.size(); ++index) + { + auto& texture = textures[index]; + if (texture.Path.empty()) + return VFSResult::Fail(VFSError::InvalidPath); + const std::filesystem::path source_path(texture.Path.c_str()); + const auto source = source_path.is_absolute() ? source_path : std::filesystem::path(config.InputBaseAssetFilePath.empty() ? "" : config.InputBaseAssetFilePath.c_str()) / source_path; + std::ifstream input(source, std::ios::binary | std::ios::ate); + if (!input.is_open()) + { + ZENGINE_CORE_ERROR("Texture not found: {}", source.string()) + return VFSResult::Fail(VFSError::NotFound); + } + const auto size = input.tellg(); + if (size <= 0) + return VFSResult::Fail(VFSError::IOError); + std::vector bytes(static_cast(size)); + input.seekg(0, std::ios::beg); + if (!input.read(reinterpret_cast(bytes.data()), static_cast(size))) + return VFSResult::Fail(VFSError::IOError); + // Source basenames are not unique (e.g. two folders with albedo.png). + const auto output_name = fmt::format("{}_{}{}", source.stem().string(), index, source.extension().string()); + auto destination = MakeOutputPath(directory.Value().CStr(), output_name.c_str()); + if (destination.Failed()) + return VFSResult::Fail(destination.Error()); + auto write = Core::VFS::WriteFileAtomically(*config.VFS, destination.Value(), {bytes.data(), bytes.size()}); + if (write.Failed()) + { + ZENGINE_CORE_ERROR("Failed to copy texture '{}' to '{}' (VFS error {})", source.string(), destination.Value().CStr(), static_cast(write.Error())) + return write; + } + texture.Path.clear(); + texture.Path.append(destination.Value().CStr()); + } + return VFSResult::Ok(); } // Write data atomically via VFS: open .tmp, write, flush, close, rename to out_path. - static bool WriteVFS(Core::VFS::IVFSContext* vfs, const VFSPath& out_path, const std::string& data) + static VFSResult WriteVFS(Core::VFS::IVFSContext* vfs, const VFSPath& out_path, const std::string& data) { if (!vfs) - return false; + return VFSResult::Fail(VFSError::Unsupported); const auto* bytes = reinterpret_cast(data.data()); - return WriteFileAtomically(*vfs, out_path, {bytes, data.size()}).Succeeded(); + return Core::VFS::WriteFileAtomically(*vfs, out_path, {bytes, data.size()}); } - AssetImporterOutput SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config) + VFSResult RestoreAssetUUID(Core::VFS::IVFSContext& ctx, const VFSPath& path, uuids::uuid& id) { - AssetImporterOutput output = {}; - if (config.OutputAssetFile.empty()) - return output; + auto meta = Core::VFS::MetaFileIO::Read(ctx, path); + if (meta.Succeeded()) + id = meta.Value().AssetUUID; + else if (meta.Error() != VFSError::NotFound && meta.Error() != VFSError::Corrupted) + return VFSResult::Fail(meta.Error()); + return VFSResult::Ok(); + } - auto mesh_dir = VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - config.VFS->CreateDir(mesh_dir); + VFSResult SerializeMeshAssetFile(Core::Memory::ArenaAllocator* /*arena*/, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config) + { + auto path = PrepareOutputPath(config.VFS, MakeOutputPath(config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str())); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + const auto& mesh_path = path.Value(); std::ostringstream buf(std::ios::binary); WriteBinary(buf, ZEMESH_MAGIC); @@ -121,22 +201,75 @@ namespace ZEngine::Importers::AssetCodec WriteBinaryHashMap(buf, hierarchies.NodeMeshes); WriteBinaryHashMap(buf, hierarchies.NodeMaterials); - if (!WriteVFS(config.VFS, mesh_path, buf.str())) - return output; + auto write = WriteVFS(config.VFS, mesh_path, buf.str()); + if (write.Failed()) + return VFSResult::Fail(write.Error()); + + return VFSResult::Ok({.Type = AssetFileType::MESH, .Path = mesh_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); + } + + std::string MaterialOutputFilename(const AssetMaterial& material, size_t index) + { + // Names are optional/non-unique in source formats. The source index keeps + // duplicate and unnamed materials distinct and stable across recooks. + return fmt::format("{}_{}.zematerial", material.Name.empty() ? "material" : material.Name.c_str(), index); + } + + VFSResult MaterialOutputPath(const AssetMaterial& material, const ImportConfiguration& config, size_t index) + { + auto directory = MakeOutputPath(config.OutputMaterialPath.c_str(), config.AssetName.c_str()); + if (directory.Failed()) + return directory; + return MakeOutputPath(directory.Value().CStr(), MaterialOutputFilename(material, index).c_str()); + } - output = {.Type = AssetFileType::MESH, .Path = mesh_path.CStr(), .RootPath = config.OutputWorkingSpacePath.c_str()}; - return output; + VFSResult SynchronizeTextureMetadata(Core::Memory::ArenaAllocator* arena, ArrayView textures, ArrayView materials, const ImportConfiguration& config, const char* importer) + { + if (!config.VFS) + return VFSResult::Fail(VFSError::Unsupported); + for (size_t t = 0; t < textures.size(); ++t) + { + auto& texture = textures[t]; + auto path = VFSPath::Parse(texture.Path.c_str()); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + auto hash = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, path.Value()); + if (hash.Failed()) + return VFSResult::Fail(hash.Error()); + auto meta = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, path.Value(), importer, hash.Value()); + if (meta.Failed()) + return VFSResult::Fail(meta.Error()); + + const auto original_uuid = texture.TextureUUID; + texture.TextureUUID = meta.Value().AssetUUID; + for (size_t m = 0; m < materials.size(); ++m) + { + auto& material = materials[m]; + auto sync = [&](uuids::uuid& id, String& destination) { + if (!original_uuid.is_nil() && id == original_uuid) + { + id = texture.TextureUUID; + destination.init(arena, texture.Path.c_str()); + } + }; + sync(material.AlbedoTexUUID, material.AlbedoTexPath); + sync(material.EmissiveTexUUID, material.EmissiveTexPath); + sync(material.NormalTexUUID, material.NormalTexPath); + sync(material.OpacityTexUUID, material.OpacityTexPath); + sync(material.SpecularTexUUID, material.SpecularTexPath); + } + } + return VFSResult::Ok(); } - AssetImporterOutput SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* /*arena*/, AssetMaterial& material, const ImportConfiguration& config) + VFSResult SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* /*arena*/, AssetMaterial& material, const ImportConfiguration& config, size_t material_index) { - AssetImporterOutput output = {}; - std::string asset_mat_filename = fmt::format("{}{}", material.Name.c_str(), ".zematerial"); - auto mat_dir = VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - config.VFS->CreateDir(mat_dir); - auto mat_path = mat_dir / asset_mat_filename.c_str(); + auto path = PrepareOutputPath(config.VFS, MaterialOutputPath(material, config, material_index)); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + const auto& mat_path = path.Value(); - auto tex_obj = [](const uuids::uuid& uuid, const Core::Containers::String& path) { + auto tex_obj = [](const uuids::uuid& uuid, const Core::Containers::String& path) { nlohmann::json t; t["uuid"] = uuids::to_string(uuid); t["path"] = path.empty() ? "" : path.c_str(); @@ -158,20 +291,22 @@ namespace ZEngine::Importers::AssetCodec j["specular_color"] = nlohmann::json::array({material.SpecularColor[0], material.SpecularColor[1], material.SpecularColor[2], material.SpecularColor[3]}); j["factors"] = nlohmann::json::array({material.Factors[0], material.Factors[1], material.Factors[2], material.Factors[3]}); - if (!WriteVFS(config.VFS, mat_path, j.dump(4))) - return output; + auto write = WriteVFS(config.VFS, mat_path, j.dump(4)); + if (write.Failed()) + return VFSResult::Fail(write.Error()); - output = {.Type = AssetFileType::MATERIAL, .Path = mat_path.CStr(), .RootPath = config.OutputWorkingSpacePath.c_str()}; - return output; + return VFSResult::Ok({.Type = AssetFileType::MATERIAL, .Path = mat_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); } - AssetImporterOutput SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, ArrayView textures, const ImportConfiguration& config) + VFSResult SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* /*arena*/, ArrayView textures, const ImportConfiguration& config) { - AssetImporterOutput output = {}; - std::string asset_tex_filename = fmt::format("{}{}", config.AssetName.c_str(), ".zetextures"); - auto tex_dir = VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto tex_path = tex_dir / asset_tex_filename.c_str(); - config.VFS->CreateDir(tex_dir); + if (config.AssetName.empty()) + return VFSResult::Fail(VFSError::InvalidPath); + std::string asset_tex_filename = fmt::format("{}.zetextures", config.AssetName.c_str()); + auto path = PrepareOutputPath(config.VFS, MakeOutputPath(config.OutputAssetsPath.c_str(), asset_tex_filename.c_str())); + if (path.Failed()) + return VFSResult::Fail(path.Error()); + const auto& tex_path = path.Value(); std::ostringstream buf(std::ios::binary); WriteBinary(buf, ZETEXTURES_MAGIC); @@ -184,11 +319,11 @@ namespace ZEngine::Importers::AssetCodec WriteBinaryString(buf, textures[i].Path); } - if (!WriteVFS(config.VFS, tex_path, buf.str())) - return output; + auto write = WriteVFS(config.VFS, tex_path, buf.str()); + if (write.Failed()) + return VFSResult::Fail(write.Error()); - output = {.Type = AssetFileType::TEXTURES, .Path = tex_path.CStr(), .RootPath = config.OutputWorkingSpacePath.c_str()}; - return output; + return VFSResult::Ok({.Type = AssetFileType::TEXTURES, .Path = tex_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); } void DeserializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMesh& mesh, AssetNodeHierarchy& hierarchies) diff --git a/ZEngine/ZEngine/Importers/AssetCodec.h b/ZEngine/ZEngine/Importers/AssetCodec.h index 4292ef5c..22c4b114 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.h +++ b/ZEngine/ZEngine/Importers/AssetCodec.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,14 @@ namespace ZEngine::Importers::AssetCodec ImportOptions Options = {}; }; + // Validate before parsing a source or accessing VFS metadata; returns the mesh destination. + [[nodiscard]] Core::VFS::VFSResult ValidateImportConfiguration(const ImportConfiguration& config); + [[nodiscard]] Core::VFS::VFSResult MakeOutputPath(const char* directory, const char* filename); + [[nodiscard]] Core::VFS::VFSResult CopyTextureFiles(Core::Containers::ArrayView textures, const ImportConfiguration& config); + std::string MaterialOutputFilename(const AssetMaterial& material, size_t index); + [[nodiscard]] Core::VFS::VFSResult MaterialOutputPath(const AssetMaterial& material, const ImportConfiguration& config, size_t index); + [[nodiscard]] Core::VFS::VFSResult SynchronizeTextureMetadata(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, Core::Containers::ArrayView materials, const ImportConfiguration& config, const char* importer); + struct AssetMeshFileHeader { uint32_t MagicNumber = 0xFFFFFF; @@ -91,34 +100,33 @@ namespace ZEngine::Importers::AssetCodec uint32_t ImporterVersion = ENVIRONMENT_MAP_IMPORTER_VERSION; }; - AssetImporterOutput SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config); + [[nodiscard]] Core::VFS::VFSResult SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config); - AssetImporterOutput SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, AssetMaterial& material, const ImportConfiguration& config); + // Preserve an existing identity; only missing/corrupt sidecars permit a new one. + [[nodiscard]] Core::VFS::VFSResult RestoreAssetUUID(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, uuids::uuid& id); - AssetImporterOutput SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, const ImportConfiguration& config); + [[nodiscard]] Core::VFS::VFSResult SerializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, AssetMaterial& material, const ImportConfiguration& config, size_t material_index); - /// @brief Atomically publishes a file through the VFS using a temporary sibling and rename. - /// @details Use for generated project assets so file watchers never observe a partially written file. - Core::VFS::VFSResult WriteFileAtomically(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, Core::Containers::ArrayView data); + [[nodiscard]] Core::VFS::VFSResult SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, const ImportConfiguration& config); - [[nodiscard]] uint32_t GetEnvironmentMapFullMipCount(uint32_t face_size); - [[nodiscard]] bool IsEnvironmentMapFileHeaderValid(const EnvironmentMapFileHeader& header); - [[nodiscard]] bool DoesEnvironmentMapHeaderMatchSource(const EnvironmentMapFileHeader& header, uint64_t source_hash); + [[nodiscard]] uint32_t GetEnvironmentMapFullMipCount(uint32_t face_size); + [[nodiscard]] bool IsEnvironmentMapFileHeaderValid(const EnvironmentMapFileHeader& header); + [[nodiscard]] bool DoesEnvironmentMapHeaderMatchSource(const EnvironmentMapFileHeader& header, uint64_t source_hash); // VFS-based — writes through IVFSContext using atomic .tmp → rename protocol. // out_path: the VFS path to write (e.g. project://_cache/envmaps/.zenvmap) - Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap, const EnvironmentMapCookMetadata& metadata = {}); + Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap, const EnvironmentMapCookMetadata& metadata = {}); - void DeserializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMesh& mesh, AssetNodeHierarchy& hierarchies); + void DeserializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMesh& mesh, AssetNodeHierarchy& hierarchies); - void DeserializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMaterial& material); + void DeserializeMaterialAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMaterial& material); - void DeserializeTextureAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, Core::Containers::Array& textures); + void DeserializeTextureAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, Core::Containers::Array& textures); - bool ReadAssetMeshFileHeader(const char* asset_file, AssetMeshFileHeader& header); + bool ReadAssetMeshFileHeader(const char* asset_file, AssetMeshFileHeader& header); - bool DeserializeEnvironmentMapFile(const char* zenvmap_file, Rendering::Buffers::Bitmap& out_cubemap); + bool DeserializeEnvironmentMapFile(const char* zenvmap_file, Rendering::Buffers::Bitmap& out_cubemap); - bool ReadEnvironmentMapFileHeader(const char* zenvmap_file, EnvironmentMapFileHeader& out_header); + bool ReadEnvironmentMapFileHeader(const char* zenvmap_file, EnvironmentMapFileHeader& out_header); } // namespace ZEngine::Importers::AssetCodec diff --git a/ZEngine/ZEngine/Importers/AssetTypes.h b/ZEngine/ZEngine/Importers/AssetTypes.h index 486410de..d897840e 100644 --- a/ZEngine/ZEngine/Importers/AssetTypes.h +++ b/ZEngine/ZEngine/Importers/AssetTypes.h @@ -46,27 +46,40 @@ namespace ZEngine::Importers struct AssetMaterial { - Core::Containers::String Name = {}; - uuids::uuid MaterialUUID = {}; + Core::Containers::String Name = {}; + uuids::uuid MaterialUUID = {}; // Texture UUIDs — runtime lookup key into AssetManager - uuids::uuid AlbedoTexUUID = {}; - uuids::uuid EmissiveTexUUID = {}; - uuids::uuid NormalTexUUID = {}; - uuids::uuid OpacityTexUUID = {}; - uuids::uuid SpecularTexUUID = {}; + uuids::uuid AlbedoTexUUID = {}; + uuids::uuid EmissiveTexUUID = {}; + uuids::uuid NormalTexUUID = {}; + uuids::uuid OpacityTexUUID = {}; + uuids::uuid SpecularTexUUID = {}; // Texture paths — project-relative VFS path to the extracted image file // Stored in .zematerial so no separate .zetextures file is needed - Core::Containers::String AlbedoTexPath = {}; - Core::Containers::String EmissiveTexPath = {}; - Core::Containers::String NormalTexPath = {}; - Core::Containers::String OpacityTexPath = {}; - Core::Containers::String SpecularTexPath = {}; - float AmbientColor[4] = {0}; - float AlbedoColor[4] = {0}; - float EmissiveColor[4] = {0}; - float RoughnessColor[4] = {0}; - float SpecularColor[4] = {0}; - float Factors[4] = {0}; + Core::Containers::String AlbedoTexPath = {}; + Core::Containers::String EmissiveTexPath = {}; + Core::Containers::String NormalTexPath = {}; + Core::Containers::String OpacityTexPath = {}; + Core::Containers::String SpecularTexPath = {}; + float AmbientColor[4] = {0}; + float AlbedoColor[4] = {0}; + float EmissiveColor[4] = {0}; + float RoughnessColor[4] = {0}; + float SpecularColor[4] = {0}; + float Factors[4] = {0}; + + // Copy string storage as well as values when crossing an arena lifetime boundary. + [[nodiscard]] AssetMaterial Clone(Core::Memory::ArenaAllocator* arena) const + { + AssetMaterial copy = *this; + copy.Name.init(arena, Name.c_str()); + copy.AlbedoTexPath.init(arena, AlbedoTexPath.c_str()); + copy.EmissiveTexPath.init(arena, EmissiveTexPath.c_str()); + copy.NormalTexPath.init(arena, NormalTexPath.c_str()); + copy.OpacityTexPath.init(arena, OpacityTexPath.c_str()); + copy.SpecularTexPath.init(arena, SpecularTexPath.c_str()); + return copy; + } }; struct AssetTexture @@ -118,6 +131,8 @@ namespace ZEngine::Importers std::string RootPath = ""; }; + // Callbacks run synchronously during ImportFile; do not re-enter the same importer. + // The output view is borrowed for the duration of the callback; copy it before deferring work. using ImportCompleteCallback = void (*)(void* ctx, Core::Containers::ArrayView outputs); using ImportProgressCallback = void (*)(void* ctx, float progress); using ImportErrorCallback = void (*)(void* ctx, std::string_view message); diff --git a/ZEngine/ZEngine/Importers/AssimpImporter.cpp b/ZEngine/ZEngine/Importers/AssimpImporter.cpp index 60c1f3a3..97c75ed3 100644 --- a/ZEngine/ZEngine/Importers/AssimpImporter.cpp +++ b/ZEngine/ZEngine/Importers/AssimpImporter.cpp @@ -1,16 +1,15 @@ #include -#include #include #include #include #include +#include #include #include #include #include +#include #include -#include -#include #include using namespace ZEngine::Helpers; @@ -20,20 +19,11 @@ using namespace ZEngine::Core::Containers; using namespace ZEngine::Core::Maths; using namespace uuids; -namespace fs = std::filesystem; - namespace ZEngine::Importers { static constexpr uint32_t kDefaultFlags = aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_GenNormals | aiProcess_SortByPType; - AssimpImporter::AssimpImporter() : m_progress_handler{} - { - m_progress_handler.SetImporter(this); - } - - AssimpImporter::~AssimpImporter() {} - - void AssimpImporter::Initialize(Core::Memory::ArenaAllocator* arena) + void AssimpImporter::Initialize(Core::Memory::ArenaAllocator* arena) { arena->CreateSubArena(ZMega(128), &Arena, "ImportPipeline/AssimpImporter"); } @@ -47,10 +37,11 @@ namespace ZEngine::Importers Core::VFS::VFSResult AssimpImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { + std::lock_guard import_lock(m_import_mutex); // Resolve VFS path to native filesystem path for Assimp. // path is VFS-relative (e.g. /Assets/Lamp01.glb); Assimp needs the full native path. - char native[MAX_FILE_PATH_COUNT] = {}; - const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + char native[MAX_FILE_PATH_COUNT] = {}; + const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; if (working_space && working_space[0] != '\0') path.ResolveNative(working_space, native, sizeof(native)); else @@ -84,7 +75,6 @@ namespace ZEngine::Importers ExtractTextures(&scratch, scene, gen, materials, textures); CreateHierachy(&scratch, scene, gen, hierarchies, mesh, materials); - importer.SetProgressHandler(nullptr); importer.FreeScene(); // Ingest directly into AssetManager CPU buffers — no intermediate .zasset round-trip. @@ -104,6 +94,14 @@ namespace ZEngine::Importers void AssimpImporter::ImportFile(const char* filename, const AssetCodec::ImportConfiguration& cfg, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log) { + std::lock_guard import_lock(m_import_mutex); + auto mesh_path = AssetCodec::ValidateImportConfiguration(cfg); + if (mesh_path.Failed() || !filename || !filename[0]) + { + if (on_error) + on_error(context, fmt::format("Invalid import configuration (VFS error {})", static_cast(mesh_path.Failed() ? mesh_path.Error() : Core::VFS::VFSError::InvalidPath))); + return; + } // The caller's arena is sized only for a few short path strings (#760) — carve a // scratch sub-arena from this importer's own, generously-sized private Arena // instead, matching the pattern Import() already uses for hot-reload. @@ -133,9 +131,7 @@ namespace ZEngine::Importers flags |= aiProcess_GenSmoothNormals; Assimp::Importer importer{}; - importer.SetProgressHandler(&m_progress_handler); - - const aiScene* scene = importer.ReadFile(filename, flags); + const aiScene* scene = importer.ReadFile(filename, flags); if (!scene || (scene->mFlags & AI_SCENE_FLAGS_INCOMPLETE) || !scene->mRootNode) { @@ -163,11 +159,14 @@ namespace ZEngine::Importers // "this is an update to an existing mesh" — every re-cook would otherwise // mint a fresh random UUID and look like a brand new, unrelated asset. { - auto mesh_dir = Core::VFS::VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mesh_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - mesh.MeshUUID = meta_result.Value().AssetUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mesh_path.Value(), mesh.MeshUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read mesh metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } if (on_progress) @@ -198,51 +197,43 @@ namespace ZEngine::Importers // reference off this same materials array for each SubMesh. for (size_t m = 0; m < materials.size(); ++m) { - auto mat_dir = Core::VFS::VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - std::string mat_filename = fmt::format("{}{}", materials[m].Name.c_str(), ".zematerial"); - auto mat_path = mat_dir / mat_filename.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mat_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - materials[m].MaterialUUID = meta_result.Value().AssetUUID; + std::string mat_filename = AssetCodec::MaterialOutputFilename(materials[m], m); + auto mat_path = AssetCodec::MaterialOutputPath(materials[m], config, m); + if (mat_path.Failed()) + { + if (on_error) + on_error(context, fmt::format("Invalid material output '{}' (VFS error {})", mat_filename, static_cast(mat_path.Error()))); + Arena.Clear(); + return; + } + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mat_path.Value(), materials[m].MaterialUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read material metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } if (config.Options.ImportTextures) { ExtractTextures(arena, scene, gen, materials, textures); - CopyTextureFiles(arena, textures, config); - for (size_t m = 0; m < materials.size(); ++m) + auto copy = AssetCodec::CopyTextureFiles(ArrayView{textures}, config); + if (copy.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to copy texture files (VFS error {})", static_cast(copy.Error()))); + Arena.Clear(); + return; + } + auto metadata = AssetCodec::SynchronizeTextureMetadata(arena, ArrayView{textures}, ArrayView{materials}, config, "AssimpImporter"); + if (metadata.Failed()) { - // ExtractTextures assigned a throwaway random UUID (#755); now - // that the file is at its final path, replace it with a stable, - // content-hashed one, matched by the original UUID still held - // in the material fields. - auto sync_texture = [&](uuids::uuid& uuid_field, Core::Containers::String& path_out) { - for (size_t t = 0; t < textures.size(); ++t) - { - if (textures[t].TextureUUID == uuid_field && !textures[t].Path.empty()) - { - path_out.init(arena, textures[t].Path.c_str()); - - auto vfs_path = Core::VFS::VFSPath::Parse(textures[t].Path.c_str()); - if (vfs_path.Succeeded() && config.VFS) - { - auto hash_result = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, vfs_path.Value()); - auto meta_result = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, vfs_path.Value(), "AssimpImporter", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta_result.Succeeded()) - { - textures[t].TextureUUID = meta_result.Value().AssetUUID; - uuid_field = meta_result.Value().AssetUUID; - } - } - return; - } - } - }; - sync_texture(materials[m].AlbedoTexUUID, materials[m].AlbedoTexPath); - sync_texture(materials[m].EmissiveTexUUID, materials[m].EmissiveTexPath); - sync_texture(materials[m].NormalTexUUID, materials[m].NormalTexPath); - sync_texture(materials[m].OpacityTexUUID, materials[m].OpacityTexPath); - sync_texture(materials[m].SpecularTexUUID, materials[m].SpecularTexPath); + if (on_error) + on_error(context, fmt::format("Failed to persist texture metadata (VFS error {})", static_cast(metadata.Error()))); + Arena.Clear(); + return; } } } @@ -250,21 +241,45 @@ namespace ZEngine::Importers if (on_progress) on_progress(context, 0.7f); - Array outputs = {}; - outputs.init(arena, 100); - outputs.push(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchies, config)); + std::vector outputs; + outputs.reserve(1 + materials.size()); + auto append_output = [&](Core::VFS::VFSResult result, const char* directory, const std::string& filename) { + if (result.Failed()) + { + const auto message = fmt::format("Failed to write '{}/{}' (VFS error {})", directory, filename, static_cast(result.Error())); + ZENGINE_CORE_ERROR("[AssimpImporter] {}", message) + if (on_error) + on_error(context, message); + return false; + } + outputs.push_back(std::move(result.Value())); + return true; + }; + bool serialized = true; if (config.Options.ImportMaterials) for (size_t i = 0; i < materials.size(); ++i) - outputs.push(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config)); + if (!append_output(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config, i), config.OutputMaterialPath.c_str(), fmt::format("{}/{}", config.AssetName.c_str(), AssetCodec::MaterialOutputFilename(materials[i], i)))) + { + serialized = false; + break; + } - if (on_progress) - on_progress(context, 1.0f); - if (on_complete) - on_complete(context, ArrayView{outputs}); + // The mesh is the last published artifact: failed dependencies leave + // the previous mesh untouched, without a multi-file transaction layer. + if (serialized) + serialized = append_output(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchies, config), config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str()); + if (serialized) + { + std::rotate(outputs.begin(), outputs.end() - 1, outputs.end()); + if (on_progress) + on_progress(context, 1.0f); + if (on_complete) + on_complete(context, {outputs.data(), outputs.size()}); + } } - importer.SetProgressHandler(nullptr); importer.FreeScene(); + Arena.Clear(); } void AssimpImporter::ExtractMeshes(Core::Memory::ArenaAllocator* arena, const aiScene* scene, uuids::uuid_random_generator& generator, AssetMesh& mesh) @@ -305,7 +320,7 @@ namespace ZEngine::Importers for (int v = 0; v < ai_mesh->mNumVertices; ++v) { const aiVector3D position = ai_mesh->mVertices[v]; - const aiVector3D normal = ai_mesh->mNormals[v]; + const aiVector3D normal = ai_mesh->HasNormals() ? ai_mesh->mNormals[v] : aiVector3D{0, 1, 0}; const aiVector3D texture = ai_mesh->HasTextureCoords(0) ? ai_mesh->mTextureCoords[0][v] : aiVector3D{}; mesh.Vertices.push(position.x); @@ -569,13 +584,17 @@ namespace ZEngine::Importers n.init(arena, mesh_name.C_Str() ? mesh_name.C_Str() : ""); hierarchy.NodeMeshes[sub_node_id] = mesh; - hierarchy.NodeMaterials[sub_node_id] = material_id; hierarchy.GlobalTransforms[sub_node_id] = Identity(); hierarchy.LocalTransforms[sub_node_id] = Identity(); - auto& asset_mat = materials[material_id]; auto& sub_mesh = asset_mesh.SubMeshes[mesh]; - sub_mesh.MaterialUUID = asset_mat.MaterialUUID; + if (material_id < materials.size()) + { + hierarchy.NodeMaterials[sub_node_id] = material_id; + sub_mesh.MaterialUUID = materials[material_id].MaterialUUID; + } + else + sub_mesh.MaterialUUID = {}; } for (uint32_t child = 0; child < node->mNumChildren; ++child) @@ -584,50 +603,6 @@ namespace ZEngine::Importers } } - void AssimpImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* /*arena*/, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) - { - if (!config.VFS) - return; - - auto texture_dir = VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str(); - auto create_dir = config.VFS->CreateDir(texture_dir); - if (create_dir.Failed() && create_dir.Error() != Core::VFS::VFSError::AlreadyExists) - return; - - for (auto& tex : textures) - { - if (tex.Path.empty()) - continue; - - fs::path tex_path(tex.Path.c_str()); - fs::path src = tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path; - std::ifstream in(src, std::ios::binary | std::ios::ate); - if (!in.is_open()) - { - ZENGINE_CORE_WARN("[AssimpImporter] Texture not found: {}", src.string()) - continue; - } - - const std::streamsize byte_count = in.tellg(); - if (byte_count <= 0) - continue; - std::vector bytes(static_cast(byte_count)); - in.seekg(0, std::ios::beg); - if (!in.read(reinterpret_cast(bytes.data()), byte_count)) - continue; - - auto filename = src.filename().string(); - auto new_path = texture_dir / filename.c_str(); - if (AssetCodec::WriteFileAtomically(*config.VFS, new_path, {bytes.data(), bytes.size()}).Failed()) - { - ZENGINE_CORE_WARN("[AssimpImporter] Failed to copy texture: {}", src.string()) - continue; - } - tex.Path.clear(); - tex.Path.append(new_path.CStr()); - } - } - Mat4f AssimpImporter::ConvertToMat4(const aiMatrix4x4& m) { Mat4f mm; @@ -641,13 +616,4 @@ namespace ZEngine::Importers return mm; } - void AssimpProgressHandler::SetImporter(AssimpImporter* const importer) - { - m_importer = importer; - } - - bool AssimpProgressHandler::Update(float /*percentage*/) - { - return true; - } } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/AssimpImporter.h b/ZEngine/ZEngine/Importers/AssimpImporter.h index fe968973..7ddc515b 100644 --- a/ZEngine/ZEngine/Importers/AssimpImporter.h +++ b/ZEngine/ZEngine/Importers/AssimpImporter.h @@ -3,30 +3,15 @@ #include #include #include -#include #include #include +#include namespace ZEngine::Importers { - class AssimpImporter; - struct AssimpProgressHandler; - - struct AssimpProgressHandler : public Assimp::ProgressHandler - { - void SetImporter(AssimpImporter* const importer); - bool Update(float percentage) override; - - private: - AssimpImporter* m_importer{nullptr}; - }; - class AssimpImporter : public IAssetImporter { public: - AssimpImporter(); - virtual ~AssimpImporter(); - void Initialize(Core::Memory::ArenaAllocator* arena); Core::Memory::ArenaAllocator Arena = {}; @@ -39,12 +24,8 @@ namespace ZEngine::Importers // file-picker import path (produces cooked .zasset artifacts). void ImportFile(const char* filename, const AssetCodec::ImportConfiguration& config, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log); - void CopyTextureFiles(Core::Memory::ArenaAllocator*, Core::Containers::Array&, const AssetCodec::ImportConfiguration&); - private: - AssimpProgressHandler m_progress_handler; - - friend struct AssimpProgressHandler; + std::mutex m_import_mutex; void ExtractMeshes(Core::Memory::ArenaAllocator*, const aiScene*, uuids::uuid_random_generator&, AssetMesh&); void ExtractMaterials(Core::Memory::ArenaAllocator*, const aiScene*, uuids::uuid_random_generator&, Core::Containers::Array&, AssetNodeHierarchy&); diff --git a/ZEngine/ZEngine/Importers/FbxImporter.cpp b/ZEngine/ZEngine/Importers/FbxImporter.cpp index 739d1896..eefe0b3b 100644 --- a/ZEngine/ZEngine/Importers/FbxImporter.cpp +++ b/ZEngine/ZEngine/Importers/FbxImporter.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,8 +6,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -19,8 +17,6 @@ using namespace ZEngine::Core::Maths; using namespace ZEngine::Importers; using namespace uuids; -namespace fs = std::filesystem; - namespace ZEngine::Importers { // Vertex deduplication key — (position, normal, uv) index triple. @@ -55,8 +51,9 @@ namespace ZEngine::Importers Core::VFS::VFSResult FbxImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { - char native[MAX_FILE_PATH_COUNT] = {}; - const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + std::lock_guard import_lock(m_import_mutex); + char native[MAX_FILE_PATH_COUNT] = {}; + const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; if (working_space && working_space[0] != '\0') path.ResolveNative(working_space, native, sizeof(native)); else @@ -216,6 +213,14 @@ namespace ZEngine::Importers void FbxImporter::ImportFile(const char* filename, const AssetCodec::ImportConfiguration& cfg, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log) { + std::lock_guard import_lock(m_import_mutex); + auto mesh_path = AssetCodec::ValidateImportConfiguration(cfg); + if (mesh_path.Failed() || !filename || !filename[0]) + { + if (on_error) + on_error(context, fmt::format("Invalid import configuration (VFS error {})", static_cast(mesh_path.Failed() ? mesh_path.Error() : Core::VFS::VFSError::InvalidPath))); + return; + } // The caller's arena is sized only for a few short path strings (#760) — route // everything through this importer's own, generously-sized private Arena instead. Arena.Clear(); @@ -274,11 +279,15 @@ namespace ZEngine::Importers // update to an existing mesh" — every re-cook would otherwise mint a fresh // random UUID and look like a brand new, unrelated asset. { - auto mesh_dir = VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mesh_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - mesh.MeshUUID = meta_result.Value().AssetUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mesh_path.Value(), mesh.MeshUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read mesh metadata (VFS error {})", static_cast(identity.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } } mesh.Vertices.init(scratch.Arena, (size_t) total_tris * 3 * 8); @@ -433,56 +442,55 @@ namespace ZEngine::Importers { for (size_t m = 0; m < materials.size(); ++m) { - auto mat_dir = VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - std::string mat_filename = fmt::format("{}{}", materials[m].Name.c_str(), ".zematerial"); - auto mat_path = mat_dir / mat_filename.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mat_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) + std::string mat_filename = AssetCodec::MaterialOutputFilename(materials[m], m); + auto mat_path = AssetCodec::MaterialOutputPath(materials[m], config, m); + if (mat_path.Failed()) + { + if (on_error) + on_error(context, fmt::format("Invalid material output '{}' (VFS error {})", mat_filename, static_cast(mat_path.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } + const auto old_uuid = materials[m].MaterialUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mat_path.Value(), materials[m].MaterialUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read material metadata (VFS error {})", static_cast(identity.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } + if (old_uuid != materials[m].MaterialUUID) { - uuids::uuid old_uuid = materials[m].MaterialUUID; - uuids::uuid new_uuid = meta_result.Value().AssetUUID; - materials[m].MaterialUUID = new_uuid; for (size_t s = 0; s < mesh.SubMeshes.size(); ++s) if (mesh.SubMeshes[s].MaterialUUID == old_uuid) - mesh.SubMeshes[s].MaterialUUID = new_uuid; + mesh.SubMeshes[s].MaterialUUID = materials[m].MaterialUUID; } } } if (config.Options.ImportMaterials && config.Options.ImportTextures) { - CopyTextureFiles(arena, textures, config); + auto copy = AssetCodec::CopyTextureFiles(ArrayView{textures}, config); + if (copy.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to copy texture files (VFS error {})", static_cast(copy.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; + } - // Fbx never synced texture paths into materials (pre-existing gap), and - // push_tex assigned throwaway random UUIDs (#755) — fix both here, - // matched by the original UUID before it's replaced. - for (size_t m = 0; m < materials.size(); ++m) + auto metadata = AssetCodec::SynchronizeTextureMetadata(arena, ArrayView{textures}, ArrayView{materials}, config, "FbxImporter"); + if (metadata.Failed()) { - auto sync_texture = [&](uuids::uuid& uuid_field, Core::Containers::String& path_out) { - for (size_t t = 0; t < textures.size(); ++t) - { - if (textures[t].TextureUUID == uuid_field && !textures[t].Path.empty()) - { - path_out.init(arena, textures[t].Path.c_str()); - - auto vfs_path = VFSPath::Parse(textures[t].Path.c_str()); - if (vfs_path.Succeeded() && config.VFS) - { - auto hash_result = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, vfs_path.Value()); - auto meta_result = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, vfs_path.Value(), "FbxImporter", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta_result.Succeeded()) - { - textures[t].TextureUUID = meta_result.Value().AssetUUID; - uuid_field = meta_result.Value().AssetUUID; - } - } - return; - } - } - }; - sync_texture(materials[m].AlbedoTexUUID, materials[m].AlbedoTexPath); - sync_texture(materials[m].NormalTexUUID, materials[m].NormalTexPath); - sync_texture(materials[m].SpecularTexUUID, materials[m].SpecularTexPath); + if (on_error) + on_error(context, fmt::format("Failed to persist texture metadata (VFS error {})", static_cast(metadata.Error()))); + ufbx_free_scene(scene); + ZReleaseScratch(scratch); + return; } } @@ -504,12 +512,35 @@ namespace ZEngine::Importers hier.LocalTransforms.push(Identity()); hier.GlobalTransforms.push(Identity()); - Array outputs = {}; - outputs.init(scratch.Arena, 16); - outputs.push(AssetCodec::SerializeMeshAssetFile(scratch.Arena, mesh, hier, config)); + std::vector outputs; + outputs.reserve(1 + materials.size()); + auto append_output = [&](Core::VFS::VFSResult result, const char* directory, const std::string& filename) { + if (result.Failed()) + { + const auto message = fmt::format("Failed to write '{}/{}' (VFS error {})", directory, filename, static_cast(result.Error())); + ZENGINE_CORE_ERROR("[FbxImporter] {}", message) + if (on_error) + on_error(context, message); + return false; + } + outputs.push_back(std::move(result.Value())); + return true; + }; if (config.Options.ImportMaterials) for (size_t i = 0; i < materials.size(); ++i) - outputs.push(AssetCodec::SerializeMaterialAssetFile(scratch.Arena, materials[i], config)); + if (!append_output(AssetCodec::SerializeMaterialAssetFile(scratch.Arena, materials[i], config, i), config.OutputMaterialPath.c_str(), fmt::format("{}/{}", config.AssetName.c_str(), AssetCodec::MaterialOutputFilename(materials[i], i)))) + { + ZReleaseScratch(scratch); + return; + } + + // Dependencies first; publish the mesh last while keeping callback order. + if (!append_output(AssetCodec::SerializeMeshAssetFile(scratch.Arena, mesh, hier, config), config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str())) + { + ZReleaseScratch(scratch); + return; + } + std::rotate(outputs.begin(), outputs.end() - 1, outputs.end()); auto* mgr = Managers::AssetManager::Instance(); if (mgr) @@ -525,52 +556,9 @@ namespace ZEngine::Importers if (on_progress) on_progress(context, 1.0f); if (on_complete) - on_complete(context, ArrayView{outputs}); + on_complete(context, {outputs.data(), outputs.size()}); ZReleaseScratch(scratch); } - void FbxImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* /*arena*/, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) - { - if (textures.empty() || !config.VFS) - return; - - auto texture_dir = VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str(); - auto create_dir = config.VFS->CreateDir(texture_dir); - if (create_dir.Failed() && create_dir.Error() != Core::VFS::VFSError::AlreadyExists) - return; - - for (auto& tex : textures) - { - if (tex.Path.empty()) - continue; - - fs::path tex_path(tex.Path.c_str()); - fs::path src = tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path; - std::ifstream in(src, std::ios::binary | std::ios::ate); - if (!in.is_open()) - { - ZENGINE_CORE_WARN("[FbxImporter] Texture not found: {}", src.string()) - continue; - } - - const std::streamsize byte_count = in.tellg(); - if (byte_count <= 0) - continue; - std::vector bytes(static_cast(byte_count)); - in.seekg(0, std::ios::beg); - if (!in.read(reinterpret_cast(bytes.data()), byte_count)) - continue; - - auto filename = tex_path.filename().string(); - auto new_path = texture_dir / filename.c_str(); - if (AssetCodec::WriteFileAtomically(*config.VFS, new_path, {bytes.data(), bytes.size()}).Failed()) - { - ZENGINE_CORE_WARN("[FbxImporter] Failed to copy texture: {}", src.string()) - continue; - } - tex.Path.clear(); - tex.Path.append(new_path.CStr()); - } - } } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/FbxImporter.h b/ZEngine/ZEngine/Importers/FbxImporter.h index e548ccdd..25adde8a 100644 --- a/ZEngine/ZEngine/Importers/FbxImporter.h +++ b/ZEngine/ZEngine/Importers/FbxImporter.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace ZEngine::Importers { @@ -18,6 +19,6 @@ namespace ZEngine::Importers void ImportFile(const char* filename, const AssetCodec::ImportConfiguration& config, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log); private: - void CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config); + std::mutex m_import_mutex; }; } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/GltfImporter.cpp b/ZEngine/ZEngine/Importers/GltfImporter.cpp index 338acff2..eac778e8 100644 --- a/ZEngine/ZEngine/Importers/GltfImporter.cpp +++ b/ZEngine/ZEngine/Importers/GltfImporter.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -16,8 +15,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -78,7 +79,224 @@ namespace ZEngine::Importers node.transform); } - static void ExtractMeshes(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, AssetMesh& out) + static bool HasTriangleGeometry(const fastgltf::Primitive& primitive) + { + return primitive.type == fastgltf::PrimitiveType::Triangles && primitive.findAttribute("POSITION") != primitive.attributes.end(); + } + + static ArrayView BufferViewBytes(const fastgltf::Asset& asset, size_t index) + { + if (index >= asset.bufferViews.size()) + return {}; + const auto& view = asset.bufferViews[index]; + if (view.bufferIndex >= asset.buffers.size()) + return {}; + const auto& buffer = asset.buffers[view.bufferIndex]; + ArrayView bytes; + if (const auto* array = std::get_if(&buffer.data)) + bytes = {reinterpret_cast(array->bytes.data()), array->bytes.size()}; + else if (const auto* byte_view = std::get_if(&buffer.data)) + bytes = {reinterpret_cast(byte_view->bytes.data()), byte_view->bytes.size()}; + const size_t available = std::min(bytes.size(), buffer.byteLength); + if (!bytes.data() || view.byteOffset > available || view.byteLength > available - view.byteOffset) + return {}; + return {bytes.data() + view.byteOffset, view.byteLength}; + } + + static bool HasAccessorRange(const fastgltf::Asset& asset, size_t view_index, size_t offset, size_t count, size_t element_size, size_t stride) + { + const auto bytes = BufferViewBytes(asset, view_index); + if (!count || !element_size || stride < element_size || offset > bytes.size() || element_size > bytes.size() - offset) + return false; + // Division avoids overflow in offset + (count - 1) * stride + size. + return count - 1 <= (bytes.size() - offset - element_size) / stride; + } + + static bool HasValidAccessor(const fastgltf::Asset& asset, size_t index, fastgltf::AccessorType type) + { + if (index >= asset.accessors.size()) + return false; + const auto& accessor = asset.accessors[index]; + const auto element_size = fastgltf::getElementByteSize(accessor.type, accessor.componentType); + if (accessor.type != type || !accessor.count || !element_size || accessor.count > std::numeric_limits::max()) + return false; + if (accessor.bufferViewIndex) + { + const auto view = *accessor.bufferViewIndex; + if (view >= asset.bufferViews.size() || !HasAccessorRange(asset, view, accessor.byteOffset, accessor.count, element_size, asset.bufferViews[view].byteStride.value_or(element_size))) + return false; + } + else if (accessor.byteOffset != 0) + return false; + if (accessor.sparse) + { + const auto& sparse = *accessor.sparse; + if (sparse.count > accessor.count || (sparse.indexComponentType != fastgltf::ComponentType::UnsignedByte && sparse.indexComponentType != fastgltf::ComponentType::UnsignedShort && sparse.indexComponentType != fastgltf::ComponentType::UnsignedInt)) + return false; + const auto index_size = fastgltf::getComponentByteSize(sparse.indexComponentType); + if (!HasAccessorRange(asset, sparse.indicesBufferView, sparse.indicesByteOffset, sparse.count, index_size, index_size) || !HasAccessorRange(asset, sparse.valuesBufferView, sparse.valuesByteOffset, sparse.count, element_size, element_size)) + return false; + } + return true; + } + + static bool HasValidGltfData(const fastgltf::Asset& asset) + { + size_t vertex_count = 0, index_count = 0; + for (const auto& mesh : asset.meshes) + for (const auto& primitive : mesh.primitives) + { + if (!HasTriangleGeometry(primitive)) + continue; + const auto position = primitive.findAttribute("POSITION")->accessorIndex; + if (!HasValidAccessor(asset, position, fastgltf::AccessorType::Vec3)) + return false; + const auto count = asset.accessors[position].count; + for (const auto& attribute : primitive.attributes) + { + const auto type = attribute.name == "NORMAL" ? fastgltf::AccessorType::Vec3 : fastgltf::AccessorType::Vec2; + if (attribute.name != "NORMAL" && attribute.name != "TEXCOORD_0") + continue; + if (!HasValidAccessor(asset, attribute.accessorIndex, type) || asset.accessors[attribute.accessorIndex].count != count) + return false; + } + size_t indices = count; + if (primitive.indicesAccessor) + { + if (!HasValidAccessor(asset, *primitive.indicesAccessor, fastgltf::AccessorType::Scalar)) + return false; + const auto& accessor = asset.accessors[*primitive.indicesAccessor]; + if (accessor.componentType != fastgltf::ComponentType::UnsignedByte && accessor.componentType != fastgltf::ComponentType::UnsignedShort && accessor.componentType != fastgltf::ComponentType::UnsignedInt) + return false; + indices = accessor.count; + } + if (indices % 3 != 0 || count > std::numeric_limits::max() / 8 - vertex_count || indices > std::numeric_limits::max() - index_count) + return false; + vertex_count += count; + index_count += indices; + } + for (const auto& texture : asset.textures) + if (texture.imageIndex) + { + if (*texture.imageIndex >= asset.images.size()) + return false; + if (const auto* view = std::get_if(&asset.images[*texture.imageIndex].data); view && BufferViewBytes(asset, view->bufferViewIndex).size() == 0) + return false; + } + return true; + } + + static bool HasValidNodeGraph(const fastgltf::Asset& asset) + { + if (asset.defaultScene && *asset.defaultScene >= asset.scenes.size()) + return false; + std::vector parents(asset.nodes.size()); + for (const auto& node : asset.nodes) + { + if (node.meshIndex && *node.meshIndex >= asset.meshes.size()) + return false; + for (auto child : node.children) + if (child >= asset.nodes.size() || ++parents[child] > 1) + return false; // glTF nodes form trees, not shared-child DAGs. + } + for (const auto& scene : asset.scenes) + { + std::vector roots(asset.nodes.size()); + for (auto root : scene.nodeIndices) + { + if (root >= asset.nodes.size() || parents[root] != 0 || roots[root]) + return false; + roots[root] = true; + } + } + // Topological traversal detects cycles, including disconnected nodes, + // without recursing on untrusted scene depth. + std::vector nodes; + for (size_t i = 0; i < parents.size(); ++i) + if (parents[i] == 0) + nodes.push_back(i); + for (size_t i = 0; i < nodes.size(); ++i) + for (auto child : asset.nodes[nodes[i]].children) + if (--parents[child] == 0) + nodes.push_back(child); + return nodes.size() == asset.nodes.size(); + } + + static bool FitsScratch(const fastgltf::Asset& asset, size_t available, bool import_materials, bool import_textures, size_t& hierarchy_capacity) + { + // Upper bound every arena allocation, including hash-table rounding and + // copied strings. Keep the existing bounded scratch policy, but fail + // before custom containers can dereference a failed allocation. + auto consume = [&](size_t count, size_t width) { + if (count > available / width) + return false; + available -= count * width; + return true; + }; + using NodeMap = UnorderedHashMap; + if (!consume(1, 1024 + 3 * 16 * sizeof(NodeMap::Entry))) + return false; // minimum map capacities and allocation alignment + for (const auto& mesh : asset.meshes) + for (const auto& primitive : mesh.primitives) + if (HasTriangleGeometry(primitive)) + { + const auto vertices = asset.accessors[primitive.findAttribute("POSITION")->accessorIndex].count; + const auto indices = primitive.indicesAccessor ? asset.accessors[*primitive.indicesAccessor].count : vertices; + if (!consume(vertices, 8 * sizeof(float)) || !consume(indices, sizeof(uint32_t)) || !consume(1, sizeof(AssetSubMesh))) + return false; + } + if (import_materials) + { + if (!consume(asset.materials.size(), sizeof(AssetMaterial) + sizeof(String) + (import_textures ? 5 * MAX_FILE_PATH_COUNT : 0))) + return false; + for (const auto& material : asset.materials) + if (!consume(2, std::max(material.name.size(), size_t(8)) + 1)) + return false; + if (import_textures) + { + if (!consume(asset.textures.size(), sizeof(AssetTexture) + MAX_FILE_PATH_COUNT + 512)) + return false; + for (const auto& texture : asset.textures) + if (texture.imageIndex && !consume(1, asset.images[*texture.imageIndex].name.size() + 1)) + return false; + } + } + const size_t node_bytes = sizeof(Helpers::NodeHierarchy) + 2 * sizeof(Mat4f) + sizeof(String) + 3 * 4 * sizeof(NodeMap::Entry); + hierarchy_capacity = 0; + std::vector pending; + auto count_scene = [&](const fastgltf::Scene& scene) { + pending.assign(scene.nodeIndices.begin(), scene.nodeIndices.end()); + while (!pending.empty()) + { + const auto& node = asset.nodes[pending.back()]; + pending.pop_back(); + if (!consume(1, node_bytes) || !consume(1, std::max(node.name.size(), size_t(6)) + 1)) + return false; + ++hierarchy_capacity; + if (node.meshIndex) + { + const auto& mesh = asset.meshes[*node.meshIndex]; + for (const auto& primitive : mesh.primitives) + if (HasTriangleGeometry(primitive)) + { + if (!consume(1, node_bytes) || !consume(1, std::max(mesh.name.size(), size_t(6)) + 1)) + return false; + ++hierarchy_capacity; + } + } + pending.insert(pending.end(), node.children.begin(), node.children.end()); + } + return true; + }; + if (asset.defaultScene) + return count_scene(asset.scenes[*asset.defaultScene]); + for (const auto& scene : asset.scenes) + if (!count_scene(scene)) + return false; + return true; + } + + static bool ExtractMeshes(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, AssetMesh& out) { uint32_t total_verts = 0; uint32_t total_indices = 0; @@ -88,7 +306,7 @@ namespace ZEngine::Importers { for (const auto& prim : mesh.primitives) { - if (prim.type != fastgltf::PrimitiveType::Triangles) + if (!HasTriangleGeometry(prim)) continue; auto it = prim.findAttribute("POSITION"); if (it != prim.attributes.end()) @@ -113,7 +331,7 @@ namespace ZEngine::Importers { for (const auto& prim : mesh.primitives) { - if (prim.type != fastgltf::PrimitiveType::Triangles) + if (!HasTriangleGeometry(prim)) continue; auto pos_it = prim.findAttribute("POSITION"); @@ -165,7 +383,13 @@ namespace ZEngine::Importers { const auto& idx_acc = asset.accessors[prim.indicesAccessor.value()]; ic = (uint32_t) idx_acc.count; - fastgltf::iterateAccessorWithIndex(asset, idx_acc, [&](uint32_t idx, std::size_t) { out.Indices.push(idx); }); + bool valid_indices = true; + fastgltf::iterateAccessorWithIndex(asset, idx_acc, [&](uint32_t idx, std::size_t) { + valid_indices = valid_indices && idx < vc; + out.Indices.push(idx); + }); + if (!valid_indices) + return false; } else { @@ -189,6 +413,7 @@ namespace ZEngine::Importers index_offset += ic; } } + return true; } static void ExtractMaterials(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, uuid_random_generator& gen, Array& out) @@ -286,87 +511,90 @@ namespace ZEngine::Importers } } - static void TraverseNode(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, std::size_t node_index, AssetNodeHierarchy& hier, const Array& mats, int parent_id, int depth) + static void TraverseNodes(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, const fastgltf::Scene& scene, AssetNodeHierarchy& hier, const Array& mats, const std::vector& mesh_offsets) { - const fastgltf::Node& node = asset.nodes[node_index]; - int id = AddNode(hier, parent_id, depth); - hier.NodeNames[id] = (uint32_t) hier.Names.size(); - auto& name = hier.Names.push_use({}); - name.init(arena, node.name.empty() ? "" : node.name.c_str()); - - hier.LocalTransforms[id] = NodeLocalTransform(node); - hier.GlobalTransforms[id] = Identity(); - - if (node.meshIndex.has_value()) + struct PendingNode + { + size_t Index; + int Parent; + int Depth; + }; + std::vector pending; + for (auto root = scene.nodeIndices.rbegin(); root != scene.nodeIndices.rend(); ++root) + pending.push_back({*root, -1, 0}); + while (!pending.empty()) { - std::size_t mesh_idx = node.meshIndex.value(); - const auto& mesh = asset.meshes[mesh_idx]; - for (std::size_t p = 0; p < mesh.primitives.size(); ++p) + const auto [node_index, parent_id, depth] = pending.back(); + pending.pop_back(); + const fastgltf::Node& node = asset.nodes[node_index]; + int id = AddNode(hier, parent_id, depth); + hier.NodeNames[id] = (uint32_t) hier.Names.size(); + auto& name = hier.Names.push_use({}); + name.init(arena, node.name.empty() ? "" : node.name.c_str()); + + hier.LocalTransforms[id] = NodeLocalTransform(node); + hier.GlobalTransforms[id] = Identity(); + + if (node.meshIndex.has_value()) { - int sub_id = AddNode(hier, id, depth + 1); - hier.NodeNames[sub_id] = (uint32_t) hier.Names.size(); - auto& sub_name = hier.Names.push_use({}); - sub_name.init(arena, mesh.name.empty() ? "" : mesh.name.c_str()); - - hier.NodeMeshes[sub_id] = (uint32_t) mesh_idx; - hier.LocalTransforms[sub_id] = Identity(); - hier.GlobalTransforms[sub_id] = Identity(); - - if (mesh.primitives[p].materialIndex.has_value()) + std::size_t mesh_idx = node.meshIndex.value(); + const auto& mesh = asset.meshes[mesh_idx]; + uint32_t submesh_index = mesh_offsets[mesh_idx]; + for (std::size_t p = 0; p < mesh.primitives.size(); ++p) { - uint32_t mat_idx = (uint32_t) mesh.primitives[p].materialIndex.value(); - hier.NodeMaterials[sub_id] = mat_idx; - if (mat_idx < mats.size()) + if (!HasTriangleGeometry(mesh.primitives[p])) + continue; + int sub_id = AddNode(hier, id, depth + 1); + hier.NodeNames[sub_id] = (uint32_t) hier.Names.size(); + auto& sub_name = hier.Names.push_use({}); + sub_name.init(arena, mesh.name.empty() ? "" : mesh.name.c_str()); + + hier.NodeMeshes[sub_id] = submesh_index++; + hier.LocalTransforms[sub_id] = Identity(); + hier.GlobalTransforms[sub_id] = Identity(); + + if (mesh.primitives[p].materialIndex.has_value()) { - AssetMesh dummy{}; - // wire material UUID into the corresponding sub-mesh - // (done outside traverse in BuildHierarchy below) - (void) dummy; + uint32_t mat_idx = (uint32_t) mesh.primitives[p].materialIndex.value(); + if (mat_idx < mats.size()) + hier.NodeMaterials[sub_id] = mat_idx; } } } - } - for (std::size_t child : node.children) - TraverseNode(arena, asset, child, hier, mats, id, depth + 1); + for (auto child = node.children.rbegin(); child != node.children.rend(); ++child) + pending.push_back({*child, id, depth + 1}); + } } - static void BuildHierarchy(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, uuid_random_generator& gen, AssetNodeHierarchy& hier, AssetMesh& mesh, const Array& mats) + static void BuildHierarchy(Core::Memory::ArenaAllocator* arena, const fastgltf::Asset& asset, uuid_random_generator& gen, AssetNodeHierarchy& hier, AssetMesh& mesh, const Array& mats, size_t capacity) { hier.NodeHierarchyUUID = gen(); hier.MeshUUID = mesh.MeshUUID; - uint32_t cap = 3000; - hier.Hierarchies.init(arena, cap); - hier.LocalTransforms.init(arena, cap); - hier.GlobalTransforms.init(arena, cap); - hier.Names.init(arena, cap); - hier.NodeNames.init(arena, cap); - hier.NodeMeshes.init(arena, cap); - hier.NodeMaterials.init(arena, cap); + hier.Hierarchies.init(arena, capacity); + hier.LocalTransforms.init(arena, capacity); + hier.GlobalTransforms.init(arena, capacity); + hier.Names.init(arena, capacity); + hier.NodeNames.init(arena, 2 * capacity); + hier.NodeMeshes.init(arena, 2 * capacity); + hier.NodeMaterials.init(arena, 2 * capacity); hier.MaterialNames.init(arena, (uint32_t) mats.size(), (uint32_t) mats.size()); for (uint32_t m = 0; m < (uint32_t) mats.size(); ++m) hier.MaterialNames[m].init(arena, mats[m].Name.c_str()); - if (asset.defaultScene.has_value()) - { - for (std::size_t n : asset.scenes[asset.defaultScene.value()].nodeIndices) - TraverseNode(arena, asset, n, hier, mats, -1, 0); - } - else - { - for (const auto& scene : asset.scenes) - for (std::size_t n : scene.nodeIndices) - TraverseNode(arena, asset, n, hier, mats, -1, 0); - } - - // Wire material UUIDs into sub-meshes - uint32_t sub_idx = 0; + // Use the same primitive filtering as extraction. These offsets also map + // hierarchy nodes to flattened submeshes across multi-primitive meshes. + std::vector mesh_offsets(asset.meshes.size()); + uint32_t sub_idx = 0; for (std::size_t mi = 0; mi < asset.meshes.size(); ++mi) { const auto& fmesh = asset.meshes[mi]; + mesh_offsets[mi] = sub_idx; for (std::size_t p = 0; p < fmesh.primitives.size(); ++p) { + if (!HasTriangleGeometry(fmesh.primitives[p])) + continue; if (sub_idx >= mesh.SubMeshes.size()) break; if (fmesh.primitives[p].materialIndex.has_value()) @@ -378,6 +606,15 @@ namespace ZEngine::Importers ++sub_idx; } } + if (asset.defaultScene.has_value()) + { + TraverseNodes(arena, asset, asset.scenes[*asset.defaultScene], hier, mats, mesh_offsets); + } + else + { + for (const auto& scene : asset.scenes) + TraverseNodes(arena, asset, scene, hier, mats, mesh_offsets); + } } void GltfImporter::Initialize(Core::Memory::ArenaAllocator* arena) @@ -394,9 +631,10 @@ namespace ZEngine::Importers Core::VFS::VFSResult GltfImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { + std::lock_guard import_lock(m_import_mutex); // Resolve VFS path to native filesystem path - char native[MAX_FILE_PATH_COUNT] = {}; - const char* ws = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + char native[MAX_FILE_PATH_COUNT] = {}; + const char* ws = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; if (ws && ws[0] != '\0') path.ResolveNative(ws, native, sizeof(native)); else @@ -411,7 +649,7 @@ namespace ZEngine::Importers return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } - auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages | fastgltf::Options::GenerateMeshIndices); + auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages); if (result.error() != fastgltf::Error::None) { @@ -419,10 +657,21 @@ namespace ZEngine::Importers return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } - fastgltf::Asset& asset = result.get(); + fastgltf::Asset& asset = result.get(); + if (!HasValidGltfData(asset) || !HasValidNodeGraph(asset)) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::Corrupted); + size_t hierarchy_capacity = 0; + if (!FitsScratch(asset, ZMega(32), true, true, hierarchy_capacity)) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::OutOfMemory); Core::Memory::ArenaAllocator scratch{}; Arena.CreateSubArena(ZMega(32), &scratch, "ImportPipeline/GltfImporter/RuntimeScratch"); + if (!scratch.Allocate(ZMega(32))) + { + Arena.Clear(); + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::OutOfMemory); + } + scratch.Clear(); std::random_device rd; std::mt19937 generator(rd()); @@ -433,12 +682,16 @@ namespace ZEngine::Importers Array materials = {}; Array textures = {}; - ExtractMeshes(&scratch, asset, mesh); + if (!ExtractMeshes(&scratch, asset, mesh)) + { + Arena.Clear(); + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::Corrupted); + } mesh.MeshUUID = meta.AssetUUID; ExtractMaterials(&scratch, asset, gen, materials); ExtractTextures(&scratch, asset, gen, textures, materials); - BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials); + BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials, hierarchy_capacity); auto* mgr = Managers::AssetManager::Instance(); if (mgr) @@ -467,34 +720,33 @@ namespace ZEngine::Importers void GltfImporter::ImportFile(const char* filename, const AssetCodec::ImportConfiguration& cfg, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log) { + std::lock_guard import_lock(m_import_mutex); + auto mesh_path = AssetCodec::ValidateImportConfiguration(cfg); + if (mesh_path.Failed() || !filename || !filename[0]) + { + if (on_error) + on_error(context, fmt::format("Invalid import configuration (VFS error {})", static_cast(mesh_path.Failed() ? mesh_path.Error() : Core::VFS::VFSError::InvalidPath))); + return; + } // The caller's arena is sized only for a few short path strings (#760) — carve a // scratch sub-arena from this importer's own, generously-sized private Arena - // instead, matching the pattern Import() already uses for hot-reload. Declared - // once here (rather than down at the old ExtractMeshes call site) so the config - // copy below and the final serialized outputs share the same backing memory - // instead of each getting an independent, aliasing CreateSubArena carve-out. + // instead, matching the pattern Import() already uses for hot-reload. + // Intermediate geometry uses this scratch arena; output paths own their + // strings independently and remain valid throughout the completion callback. Core::Memory::ArenaAllocator scratch{}; Arena.CreateSubArena(ZMega(32), &scratch, "ImportPipeline/GltfImporter/EditorScratch"); - arena = &scratch; - - // Build arena-allocated config copy (same pattern as AssimpImporter::ImportFile) - AssetCodec::ImportConfiguration config = {}; - config.OutputWorkingSpacePath.init(arena, cfg.OutputWorkingSpacePath.c_str()); - config.OutputTextureFilesPath.init(arena, cfg.OutputTextureFilesPath.c_str()); - config.OutputAssetsPath.init(arena, cfg.OutputAssetsPath.c_str()); - config.OutputMaterialPath.init(arena, cfg.OutputMaterialPath.c_str()); - config.AssetName.init(arena, cfg.AssetName.c_str()); - config.OutputAssetFile.init(arena, cfg.OutputAssetFile.c_str()); - config.InputBaseAssetFilePath.init(arena, cfg.InputBaseAssetFilePath.c_str()); - config.VFS = cfg.VFS; - config.Options = cfg.Options; - - auto fs_path = std::filesystem::path(filename); - auto buf = fastgltf::GltfDataBuffer::FromPath(fs_path); + arena = &scratch; + + // Config remains borrowed throughout this synchronous call; no scratch copy. + const auto& config = cfg; + + auto fs_path = std::filesystem::path(filename); + auto buf = fastgltf::GltfDataBuffer::FromPath(fs_path); if (buf.error() != fastgltf::Error::None) { if (on_error) on_error(context, fastgltf::getErrorMessage(buf.error())); + Arena.Clear(); return; } @@ -502,19 +754,37 @@ namespace ZEngine::Importers on_progress(context, 0.1f); fastgltf::Parser parser; - auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages | fastgltf::Options::GenerateMeshIndices); + auto result = parser.loadGltf(buf.get(), fs_path.parent_path(), fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages); if (result.error() != fastgltf::Error::None) { if (on_error) on_error(context, fastgltf::getErrorMessage(result.error())); + Arena.Clear(); return; } if (on_progress) on_progress(context, 0.3f); - fastgltf::Asset& asset = result.get(); + fastgltf::Asset& asset = result.get(); + if (!HasValidGltfData(asset) || !HasValidNodeGraph(asset)) + { + if (on_error) + on_error(context, "Invalid glTF buffer range, accessor, triangle count, or node graph"); + Arena.Clear(); + return; + } + + size_t hierarchy_capacity = 0; + if (!FitsScratch(asset, scratch.m_total_size, config.Options.ImportMaterials, config.Options.ImportMaterials && config.Options.ImportTextures, hierarchy_capacity) || !scratch.Allocate(scratch.m_total_size)) + { + if (on_error) + on_error(context, "glTF exceeds the importer scratch memory budget or memory could not be committed"); + Arena.Clear(); + return; + } + scratch.Clear(); std::random_device rd; std::mt19937 gen_mt(rd()); @@ -525,7 +795,13 @@ namespace ZEngine::Importers Array materials = {}; Array textures = {}; - ExtractMeshes(&scratch, asset, mesh); + if (!ExtractMeshes(&scratch, asset, mesh)) + { + if (on_error) + on_error(context, "Invalid glTF triangle vertex index"); + Arena.Clear(); + return; + } mesh.MeshUUID = gen(); // Stabilize the mesh UUID (#762): re-importing the same destination path must @@ -533,11 +809,14 @@ namespace ZEngine::Importers // update to an existing mesh" — every re-cook would otherwise mint a fresh // random UUID and look like a brand new, unrelated asset. { - auto mesh_dir = Core::VFS::VFSPath::Parse(config.OutputAssetsPath.c_str()).Value(); - auto mesh_path = mesh_dir / config.OutputAssetFile.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mesh_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - mesh.MeshUUID = meta_result.Value().AssetUUID; + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mesh_path.Value(), mesh.MeshUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read mesh metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } // Apply per-vertex transform options @@ -575,26 +854,12 @@ namespace ZEngine::Importers // Optimize each submesh: vertex cache, overdraw, vertex fetch. for (uint32_t si = 0; si < mesh.SubMeshes.size(); ++si) { - auto& sub = mesh.SubMeshes[si]; - uint32_t* sub_idx = mesh.Indices.data() + sub.IndexOffset; - - // Cross-primitive index contamination or a malformed GLTF can leave an index - // outside this submesh's own vertex range. Rebasing such an index underflows - // (uint32_t subtraction), which meshopt::buildTriangleAdjacency asserts on. - // Skip optimization for this submesh rather than crash — indices are left - // untouched (still correct against the unmodified vertex buffer). - bool in_range = true; - for (uint32_t j = 0; j < sub.IndexCount && in_range; ++j) - if (sub_idx[j] < sub.VertexOffset || sub_idx[j] >= sub.VertexOffset + sub.VertexCount) - in_range = false; - if (!in_range) - continue; - - for (uint32_t j = 0; j < sub.IndexCount; ++j) - sub_idx[j] -= sub.VertexOffset; + auto& sub = mesh.SubMeshes[si]; + uint32_t* sub_idx = mesh.Indices.data() + sub.IndexOffset; + + // glTF indices are already local to the primitive's vertex slice. + // Extraction validated both the range and triangle count. Importers::OptimizeMeshSubmesh(mesh.Vertices.data() + sub.VertexOffset * 8, sub.VertexCount, sub_idx, sub.IndexCount); - for (uint32_t j = 0; j < sub.IndexCount; ++j) - sub_idx[j] += sub.VertexOffset; } if (config.Options.ImportMaterials) @@ -608,18 +873,29 @@ namespace ZEngine::Importers // which reads MaterialUUID by reference off this same materials array. for (size_t m = 0; m < materials.size(); ++m) { - auto mat_dir = Core::VFS::VFSPath::Parse(config.OutputMaterialPath.c_str()).Value(); - std::string mat_filename = fmt::format("{}{}", materials[m].Name.c_str(), ".zematerial"); - auto mat_path = mat_dir / mat_filename.c_str(); - auto meta_result = Core::VFS::MetaFileIO::Read(*config.VFS, mat_path); - if (meta_result.Succeeded() && !meta_result.Value().AssetUUID.is_nil()) - materials[m].MaterialUUID = meta_result.Value().AssetUUID; + std::string mat_filename = AssetCodec::MaterialOutputFilename(materials[m], m); + auto mat_path = AssetCodec::MaterialOutputPath(materials[m], config, m); + if (mat_path.Failed()) + { + if (on_error) + on_error(context, fmt::format("Invalid material output '{}' (VFS error {})", mat_filename, static_cast(mat_path.Error()))); + Arena.Clear(); + return; + } + auto identity = AssetCodec::RestoreAssetUUID(*config.VFS, mat_path.Value(), materials[m].MaterialUUID); + if (identity.Failed()) + { + if (on_error) + on_error(context, fmt::format("Failed to read material metadata (VFS error {})", static_cast(identity.Error()))); + Arena.Clear(); + return; + } } if (config.Options.ImportTextures) ExtractTextures(&scratch, asset, gen, textures, materials); } - BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials); + BuildHierarchy(&scratch, asset, gen, hierarchy, mesh, materials, hierarchy_capacity); // Extract texture image bytes to disk and record project-relative paths if (config.Options.ImportTextures && config.Options.ImportMaterials) @@ -630,21 +906,10 @@ namespace ZEngine::Importers { if (on_error) on_error(context, "Unable to create the texture output directory"); + Arena.Clear(); return; } - // Pre-resolve base pointers for every buffer — for a GLB all images share - // buffer 0 (the binary chunk). Resolving once avoids repeated get_if per texture. - std::vector buf_ptrs(asset.buffers.size(), nullptr); - for (size_t i = 0; i < asset.buffers.size(); ++i) - { - const auto& bd = asset.buffers[i].data; - if (const auto* arr = std::get_if(&bd)) - buf_ptrs[i] = reinterpret_cast(arr->bytes.data()); - else if (const auto* bvd = std::get_if(&bd)) - buf_ptrs[i] = reinterpret_cast(bvd->bytes.data()); - } - for (size_t tex_idx = 0; tex_idx < textures.size() && tex_idx < asset.textures.size(); ++tex_idx) { const auto& fgltf_tex = asset.textures[tex_idx]; @@ -674,15 +939,11 @@ namespace ZEngine::Importers } else if (const auto* bv_src = std::get_if(&img.data)) { - const auto& bv = asset.bufferViews[bv_src->bufferViewIndex]; - const uint8_t* base = bv.bufferIndex < buf_ptrs.size() ? buf_ptrs[bv.bufferIndex] : nullptr; - if (base) - { - bytes = base + bv.byteOffset; - nbytes = bv.byteLength; - if (bv_src->mimeType == fastgltf::MimeType::JPEG) - ext = ".jpg"; - } + const auto view = BufferViewBytes(asset, bv_src->bufferViewIndex); + bytes = view.data(); + nbytes = view.size(); + if (bv_src->mimeType == fastgltf::MimeType::JPEG) + ext = ".jpg"; } else if (const auto* uri_src = std::get_if(&img.data)) { @@ -701,9 +962,11 @@ namespace ZEngine::Importers if (file_sz > 0) { uri_file_buf.resize(static_cast(file_sz)); - fread(uri_file_buf.data(), 1, uri_file_buf.size(), src_file); - bytes = uri_file_buf.data(); - nbytes = uri_file_buf.size(); + if (fread(uri_file_buf.data(), 1, uri_file_buf.size(), src_file) == uri_file_buf.size()) + { + bytes = uri_file_buf.data(); + nbytes = uri_file_buf.size(); + } // Derive extension from URI path if (path_sv.size() >= 4) { @@ -748,8 +1011,10 @@ namespace ZEngine::Importers if (!bytes || nbytes == 0) { - ZENGINE_LOG_ASSET_WARN("GltfImporter: tex {} image data not accessible (unsupported source variant)", tex_idx) - continue; + if (on_error) + on_error(context, fmt::format("Unable to read texture {} image data", tex_idx)); + Arena.Clear(); + return; } // Build output filename — prefer image name, then URI filename, then tex_N @@ -769,55 +1034,35 @@ namespace ZEngine::Importers } else { - snprintf(stem_buf, sizeof(stem_buf), "tex_%zu", tex_idx); + Helpers::secure_strcpy(stem_buf, sizeof(stem_buf), "image"); } - char output_name[sizeof(stem_buf) + 5] = {}; - snprintf(output_name, sizeof(output_name), "%s%s", stem_buf, ext); - auto output_path = texture_dir / output_name; - if (AssetCodec::WriteFileAtomically(*config.VFS, output_path, {bytes, nbytes}).Succeeded()) + // Image names are optional/non-unique. The source image index + // preserves identity while textures sharing an image share a file. + const auto output_name = fmt::format("{}_{}{}", stem_buf, *fgltf_tex.imageIndex, ext); + auto output_path = AssetCodec::MakeOutputPath(texture_dir.CStr(), output_name.c_str()); + auto write = output_path.Succeeded() ? Core::VFS::WriteFileAtomically(*config.VFS, output_path.Value(), {bytes, nbytes}) : Core::VFS::VFSResult::Fail(output_path.Error()); + if (write.Succeeded()) { - ZENGINE_LOG_ASSET_INFO("GltfImporter: extracted texture '{}' ({} bytes)", output_path.CStr(), nbytes) - textures[tex_idx].Path.init(&scratch, output_path.CStr()); + ZENGINE_LOG_ASSET_INFO("GltfImporter: extracted texture '{}' ({} bytes)", output_path.Value().CStr(), nbytes) + textures[tex_idx].Path.init(&scratch, output_path.Value().CStr()); } else { - ZENGINE_LOG_ASSET_WARN("GltfImporter: unable to write extracted texture '{}'", output_path.CStr()) + if (on_error) + on_error(context, fmt::format("Failed to write '{}/{}' (VFS error {})", texture_dir.CStr(), output_name, static_cast(write.Error()))); + Arena.Clear(); + return; } } - // Propagate tex.Path → material.*TexPath by UUID match, replacing - // ExtractTextures' throwaway random UUID (#755) with a stable, - // content-hashed one. - for (size_t m = 0; m < materials.size(); ++m) + auto metadata = AssetCodec::SynchronizeTextureMetadata(arena, ArrayView{textures}, ArrayView{materials}, config, "GltfImporter"); + if (metadata.Failed()) { - auto sync_texture = [&](uuids::uuid& uuid_field, Core::Containers::String& path_out) { - for (size_t t = 0; t < textures.size(); ++t) - { - if (textures[t].TextureUUID == uuid_field && !textures[t].Path.empty()) - { - path_out.init(&scratch, textures[t].Path.c_str()); - - auto vfs_path = Core::VFS::VFSPath::Parse(textures[t].Path.c_str()); - if (vfs_path.Succeeded() && config.VFS) - { - auto hash_result = Core::VFS::MetaFileIO::ComputeHash(*config.VFS, vfs_path.Value()); - auto meta_result = Core::VFS::MetaFileIO::GetOrCreate(*config.VFS, vfs_path.Value(), "GltfImporter", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta_result.Succeeded()) - { - textures[t].TextureUUID = meta_result.Value().AssetUUID; - uuid_field = meta_result.Value().AssetUUID; - } - } - return; - } - } - }; - sync_texture(materials[m].AlbedoTexUUID, materials[m].AlbedoTexPath); - sync_texture(materials[m].EmissiveTexUUID, materials[m].EmissiveTexPath); - sync_texture(materials[m].NormalTexUUID, materials[m].NormalTexPath); - sync_texture(materials[m].OpacityTexUUID, materials[m].OpacityTexPath); - sync_texture(materials[m].SpecularTexUUID, materials[m].SpecularTexPath); + if (on_error) + on_error(context, fmt::format("Failed to persist texture metadata (VFS error {})", static_cast(metadata.Error()))); + Arena.Clear(); + return; } } @@ -825,12 +1070,37 @@ namespace ZEngine::Importers on_progress(context, 0.7f); // Serialize to disk — .zemesh + .zematerial (no .zetextures: paths are inline in material) - Array outputs = {}; - outputs.init(arena, 16); - outputs.push(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchy, config)); + // AssetImporterOutput owns std::strings and must use constructed storage. + std::vector outputs; + outputs.reserve(1 + materials.size()); + auto append_output = [&](Core::VFS::VFSResult result, const char* directory, const std::string& filename) { + if (result.Failed()) + { + const auto message = fmt::format("Failed to write '{}/{}' (VFS error {})", directory, filename, static_cast(result.Error())); + ZENGINE_CORE_ERROR("[GltfImporter] {}", message) + if (on_error) + on_error(context, message); + return false; + } + outputs.push_back(std::move(result.Value())); + return true; + }; if (config.Options.ImportMaterials) for (size_t i = 0; i < materials.size(); ++i) - outputs.push(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config)); + if (!append_output(AssetCodec::SerializeMaterialAssetFile(arena, materials[i], config, i), config.OutputMaterialPath.c_str(), fmt::format("{}/{}", config.AssetName.c_str(), AssetCodec::MaterialOutputFilename(materials[i], i)))) + { + Arena.Clear(); + return; + } + + // Publish the mesh only after all referenced dependencies were written. + // Preserve the mesh-first completion callback ordering. + if (!append_output(AssetCodec::SerializeMeshAssetFile(arena, mesh, hierarchy, config), config.OutputAssetsPath.c_str(), config.OutputAssetFile.c_str())) + { + Arena.Clear(); + return; + } + std::rotate(outputs.begin(), outputs.end() - 1, outputs.end()); auto* mgr = Managers::AssetManager::Instance(); if (mgr) @@ -847,7 +1117,7 @@ namespace ZEngine::Importers on_progress(context, 1.0f); if (on_complete) - on_complete(context, ArrayView{outputs}); + on_complete(context, {outputs.data(), outputs.size()}); Arena.Clear(); } diff --git a/ZEngine/ZEngine/Importers/GltfImporter.h b/ZEngine/ZEngine/Importers/GltfImporter.h index 885b0e30..a4386b23 100644 --- a/ZEngine/ZEngine/Importers/GltfImporter.h +++ b/ZEngine/ZEngine/Importers/GltfImporter.h @@ -3,13 +3,14 @@ #include #include #include +#include #include namespace ZEngine::Importers { // Handles GLB and GLTF import via fastgltf. - // Stateless across concurrent imports — each Import() call carves a scratch - // sub-arena from the importer's own Arena for intermediate geometry data. + // Shared scratch is serialized across runtime and editor imports to preserve + // the bounded arena budget without allowing one import to rewind another. class GltfImporter : public IAssetImporter { public: @@ -25,5 +26,8 @@ namespace ZEngine::Importers void ImportFile(const char* filename, const AssetCodec::ImportConfiguration& config, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log); Core::Memory::ArenaAllocator Arena = {}; + + private: + std::mutex m_import_mutex; }; } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/IAssetImporter.h b/ZEngine/ZEngine/Importers/IAssetImporter.h index bca04030..394ca577 100644 --- a/ZEngine/ZEngine/Importers/IAssetImporter.h +++ b/ZEngine/ZEngine/Importers/IAssetImporter.h @@ -10,7 +10,8 @@ namespace ZEngine::Importers int AddNode(AssetNodeHierarchy& hierarchy, int parent, int depth); // Lightweight per-format importer interface used by ImportCoordinator. - // Implementations must be stateless — a single instance handles concurrent imports. + // A single instance handles concurrent calls. Implementations with shared scratch + // storage must serialize its use; different importer instances remain independent. // UUID must NOT be generated inside Import(); read it from meta.AssetUUID which was // assigned by MetaFileIO at scan time and is stable across reimports. struct IAssetImporter diff --git a/ZEngine/ZEngine/Importers/TextureImporter.cpp b/ZEngine/ZEngine/Importers/TextureImporter.cpp index 1b0f21bc..d52b576a 100644 --- a/ZEngine/ZEngine/Importers/TextureImporter.cpp +++ b/ZEngine/ZEngine/Importers/TextureImporter.cpp @@ -22,6 +22,7 @@ namespace ZEngine::Importers Core::VFS::VFSResult TextureImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { + std::lock_guard import_lock(m_import_mutex); (void) ctx; // path is workspace-relative — ToNative alone only swaps separators and would diff --git a/ZEngine/ZEngine/Importers/TextureImporter.h b/ZEngine/ZEngine/Importers/TextureImporter.h index c856801a..f6060745 100644 --- a/ZEngine/ZEngine/Importers/TextureImporter.h +++ b/ZEngine/ZEngine/Importers/TextureImporter.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace ZEngine::Importers { @@ -19,5 +20,8 @@ namespace ZEngine::Importers Core::VFS::VFSResult Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) override; Core::Memory::ArenaAllocator Arena = {}; + + private: + std::mutex m_import_mutex; }; } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Managers/AssetManager.cpp b/ZEngine/ZEngine/Managers/AssetManager.cpp index b0c176a8..11d1a266 100644 --- a/ZEngine/ZEngine/Managers/AssetManager.cpp +++ b/ZEngine/ZEngine/Managers/AssetManager.cpp @@ -304,7 +304,7 @@ namespace ZEngine::Managers return; auto slot = static_cast(s_Instance->Materials.size()); - s_Instance->Materials.push(mat); + s_Instance->Materials.push(mat.Clone(s_Instance->Arena)); RegisterAsset(AssetType::MATERIAL, mat.MaterialUUID, slot); s_Instance->UUIDToMaterialSlot.insert(mat.MaterialUUID, slot); diff --git a/ZEngine/tests/Rendering/AssetMaterialLifetime_test.cpp b/ZEngine/tests/Rendering/AssetMaterialLifetime_test.cpp new file mode 100644 index 00000000..c47fdd59 --- /dev/null +++ b/ZEngine/tests/Rendering/AssetMaterialLifetime_test.cpp @@ -0,0 +1,62 @@ +#include +#include +#include +#include +#include + +using ZEngine::Core::Containers::String; +using ZEngine::Core::Memory::ArenaAllocator; +using ZEngine::Importers::AssetMaterial; + +TEST(AssetMaterialLifetimeTest, CloneOwnsEveryStringAfterSourceScratchIsReused) +{ + ZEngine::Core::Memory::MemoryManager memory; + memory.Initialize(ZMega(2), {}); + ArenaAllocator scratch; + ArenaAllocator persistent; + memory.MainArena.CreateSubArena(ZMega(1), &scratch); + memory.MainArena.CreateSubArena(ZKilo(256), &persistent); + + AssetMaterial source; + const std::array source_strings = {&source.Name, &source.AlbedoTexPath, &source.EmissiveTexPath, &source.NormalTexPath, &source.OpacityTexPath, &source.SpecularTexPath}; + std::array expected; + for (size_t i = 0; i < expected.size(); ++i) + { + expected[i] = "/textures/" + std::string(80, static_cast('a' + i)) + ".png"; + source_strings[i]->init(&scratch, expected[i].c_str()); + } + source.MaterialUUID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + source.AlbedoTexUUID = source.MaterialUUID; + source.NormalTexUUID = source.MaterialUUID; + source.AlbedoColor[2] = 0.75f; + source.Factors[0] = 0.5f; + + auto copy = source.Clone(&persistent); + const std::array copied_strings = {©.Name, ©.AlbedoTexPath, ©.EmissiveTexPath, ©.NormalTexPath, ©.OpacityTexPath, ©.SpecularTexPath}; + for (size_t i = 0; i < copied_strings.size(); ++i) + EXPECT_NE(copied_strings[i]->c_str(), source_strings[i]->c_str()); + scratch.Clear(); + ASSERT_NE(scratch.Allocate(ZKilo(4)), nullptr); // allocation zeroes and overwrites the old strings + for (size_t i = 0; i < copied_strings.size(); ++i) + EXPECT_STREQ(copied_strings[i]->c_str(), expected[i].c_str()); + EXPECT_EQ(copy.MaterialUUID, source.MaterialUUID); + EXPECT_EQ(copy.AlbedoTexUUID, source.AlbedoTexUUID); + EXPECT_EQ(copy.NormalTexUUID, source.NormalTexUUID); + EXPECT_FLOAT_EQ(copy.AlbedoColor[2], 0.75f); + EXPECT_FLOAT_EQ(copy.Factors[0], 0.5f); +} + +TEST(AssetMaterialLifetimeTest, CloneSupportsUninitializedOptionalStrings) +{ + ZEngine::Core::Memory::MemoryManager memory; + memory.Initialize(ZKilo(64), {}); + const AssetMaterial source; + const auto copy = source.Clone(&memory.MainArena); + EXPECT_TRUE(copy.Name.empty()); + EXPECT_TRUE(copy.AlbedoTexPath.empty()); + EXPECT_TRUE(copy.EmissiveTexPath.empty()); + EXPECT_TRUE(copy.NormalTexPath.empty()); + EXPECT_TRUE(copy.OpacityTexPath.empty()); + EXPECT_TRUE(copy.SpecularTexPath.empty()); + EXPECT_TRUE(copy.MaterialUUID.is_nil()); +} diff --git a/ZEngine/tests/Rendering/ModelImporterOutput_test.cpp b/ZEngine/tests/Rendering/ModelImporterOutput_test.cpp new file mode 100644 index 00000000..64d9809c --- /dev/null +++ b/ZEngine/tests/Rendering/ModelImporterOutput_test.cpp @@ -0,0 +1,1661 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ZEngine::Core::VFS; +using namespace ZEngine::Importers; +using namespace ZEngine::Importers::AssetCodec; +using ZEngine::Core::Containers::ArrayView; +using ZEngine::Core::Memory::ArenaAllocator; + +namespace +{ + enum class FailurePoint + { + None, + CreateDir, + Open, + Write, + ShortWrite, + Flush, + Close, + Rename + }; + + // Use the real in-memory VFS; inject only the operation under test. + struct FaultVFSContext : VFSContext + { + struct FileProxy : IVFSFile + { + IVFSFile* Inner = nullptr; + FailurePoint Point = FailurePoint::None; + + VFSResult Read(ArrayView buffer, uint64_t offset) override + { + return Inner->Read(buffer, offset); + } + VFSResult Write(ArrayView buffer, uint64_t offset) override + { + if (Point == FailurePoint::Write) + return VFSResult::Fail(VFSError::IOError); + if (Point == FailurePoint::ShortWrite) + return Inner->Write({buffer.data(), buffer.size() - 1}, offset); + return Inner->Write(buffer, offset); + } + VFSResult Size() const override + { + return Inner->Size(); + } + VFSResult Stat() const override + { + return Inner->Stat(); + } + VFSResult Flush() override + { + return Point == FailurePoint::Flush ? VFSResult::Fail(VFSError::IOError) : Inner->Flush(); + } + const VFSPath& Path() const override + { + return Inner->Path(); + } + VFSResult Close() override + { + auto result = Inner->Close(); + return Point == FailurePoint::Close ? VFSResult::Fail(VFSError::IOError) : result; + } + } File; + + FailurePoint Point = FailurePoint::None; + std::string FailurePath; + size_t WriteNumber = 0; + size_t FailureWriteNumber = 0; + VFSError MetadataReadError = VFSError::OK; + std::string MetadataReadPath; + + bool Matches(const VFSPath& path) const + { + return (FailureWriteNumber == 0 || FailureWriteNumber == WriteNumber) && std::string(path.CStr()).find(FailurePath) != std::string::npos; + } + + VFSResult CreateDir(const VFSPath& path) override + { + if (Point == FailurePoint::CreateDir && Matches(path)) + return VFSResult::Fail(VFSError::PermissionDenied); + return VFSContext::CreateDir(path); + } + + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + if (!HasFlag(flags, VFSOpenFlags::Write)) + { + const std::string_view name(path.CStr()); + if (MetadataReadError != VFSError::OK && name.ends_with(".meta") && name.find(MetadataReadPath) != std::string_view::npos) + return VFSResult::Fail(MetadataReadError); + return VFSContext::Open(path, flags); + } + ++WriteNumber; + if (Point == FailurePoint::Open && Matches(path)) + return VFSResult::Fail(VFSError::PermissionDenied); + auto result = VFSContext::Open(path, flags); + if (result.Failed()) + return result; + File.Inner = result.Value(); + File.Point = Matches(path) ? Point : FailurePoint::None; + return VFSResult::Ok(&File); + } + + void Close(IVFSFile* file) override + { + if (file == &File) + { + VFSContext::Close(File.Inner); + File.Inner = nullptr; + } + else + VFSContext::Close(file); + } + + VFSResult Rename(const VFSPath& src, const VFSPath& dst) override + { + if (Point == FailurePoint::Rename && Matches(dst)) + return VFSResult::Fail(VFSError::IOError); + return VFSContext::Rename(src, dst); + } + }; + + struct ImportCallbacks + { + int Completed = 0; + int Failed = 0; + float Progress = 0; + std::string Error; + std::vector Outputs; + + static void Complete(void* context, ArrayView outputs) + { + auto& self = *static_cast(context); + ++self.Completed; + self.Outputs.clear(); + for (size_t i = 0; i < outputs.size(); ++i) + self.Outputs.push_back(outputs[i]); + } + static void Fail(void* context, std::string_view message) + { + auto& self = *static_cast(context); + ++self.Failed; + self.Error = message; + } + static void Advance(void* context, float progress) + { + static_cast(context)->Progress = progress; + } + }; + struct PausedImportCallbacks + { + ImportCallbacks Results; + std::promise Entered; + std::shared_future Release; + bool Paused = false; + + static void Complete(void* context, ArrayView outputs) + { + ImportCallbacks::Complete(&static_cast(context)->Results, outputs); + } + static void Fail(void* context, std::string_view message) + { + ImportCallbacks::Fail(&static_cast(context)->Results, message); + } + static void Advance(void* context, float progress) + { + auto& self = *static_cast(context); + self.Results.Progress = progress; + if (!self.Paused) + { + self.Paused = true; + self.Entered.set_value(); + self.Release.wait(); + } + } + }; +} // namespace + +class ModelImporterOutputTest : public ::testing::Test +{ +protected: + // Declaration order keeps arena storage alive through backend/importer destruction. + ZEngine::Core::Memory::MemoryManager Memory; + ArenaAllocator& Arena = Memory.MainArena; + VFSMemoryBackend Backend; + FaultVFSContext Context; + GltfImporter Importer; + ImportConfiguration Config; + ImportCallbacks Callbacks; + std::filesystem::path TempDirectory; + + void SetUp() override + { + // FBX reserves a 512 MiB sub-arena; pages are committed only as used. + Memory.Initialize(ZMega(768), {}); + Backend.Initialize(&Arena); + Context.Initialize(&Arena, 4); + ASSERT_TRUE(Context.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + Importer.Initialize(&Arena); + + Config.OutputWorkingSpacePath.init(&Arena, "/project"); + Config.OutputAssetsPath.init(&Arena, "/meshes"); + Config.OutputMaterialPath.init(&Arena, "/materials"); + Config.OutputTextureFilesPath.init(&Arena, "/textures"); + Config.OutputAssetFile.init(&Arena, "triangle.zemesh"); + Config.AssetName.init(&Arena, "triangle"); + Config.VFS = &Context; + Config.Options.ImportMaterials = false; + Config.Options.ImportTextures = false; + Config.Options.NormalsMode = 0; + + std::random_device rd; + std::mt19937 random(rd()); + uuids::uuid_random_generator uuid(random); + TempDirectory = std::filesystem::temp_directory_path() / ("zengine_import_output_" + uuids::to_string(uuid())); + ASSERT_TRUE(std::filesystem::create_directory(TempDirectory)); + Config.InputBaseAssetFilePath.init(&Arena, TempDirectory.string().c_str()); + } + + void TearDown() override + { + Context.Shutdown(); + std::error_code error; + std::filesystem::remove_all(TempDirectory, error); + } + + VFSPath Path(const char* raw) + { + return VFSPath::Parse(raw).Value(); + } + + bool Exists(const char* raw) + { + auto result = Context.Exists(Path(raw)); + return result.Succeeded() && result.Value(); + } + + void ExpectNoTemporaryFiles(const char* directory) + { + auto entries = Context.List(Path(directory), &Arena); + if (entries.Failed() && entries.Error() == VFSError::NotFound) + return; + ASSERT_TRUE(entries.Succeeded()); + for (const auto& entry : entries.Value()) + { + EXPECT_FALSE(std::string_view(entry.Path.CStr()).ends_with(".tmp")) << entry.Path.CStr(); + if (entry.IsDirectory) + ExpectNoTemporaryFiles(entry.Path.CStr()); + } + } + + std::vector ReadFile(const char* raw) + { + auto open = Context.Open(Path(raw), VFSOpenFlags::Read); + EXPECT_TRUE(open.Succeeded()); + if (open.Failed()) + return {}; + auto size = open.Value()->Size(); + EXPECT_TRUE(size.Succeeded()); + std::vector bytes(size.Succeeded() ? size.Value() : 0); + auto read = open.Value()->ReadAll({bytes.data(), bytes.size()}); + EXPECT_TRUE(read.Succeeded()); + Context.Close(open.Value()); + return bytes; + } + + std::string WriteGltf(size_t material_count = 0) + { + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + { + std::ofstream file(TempDirectory / "triangle.bin", std::ios::binary); + file.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + } + auto document = nlohmann::json::parse(R"({ + "asset": {"version": "2.0"}, + "buffers": [{"uri": "triangle.bin", "byteLength": 36}], + "bufferViews": [{"buffer": 0, "byteLength": 36}], + "accessors": [{"bufferView": 0, "componentType": 5126, "count": 3, "type": "VEC3", "min": [0,0,0], "max": [1,1,0]}], + "meshes": [{"primitives": [{"attributes": {"POSITION": 0}}]}], + "nodes": [{"mesh": 0}], "scenes": [{"nodes": [0]}], "scene": 0 + })"); + if (material_count) + { + document["meshes"][0]["primitives"][0]["material"] = 0; + for (size_t i = 0; i < material_count; ++i) + document["materials"].push_back({ + {"name", std::string(80, 'm') + std::to_string(i)} + }); + } + const auto filename = (TempDirectory / "triangle.gltf").string(); + std::ofstream(filename) << document.dump(); + return filename; + } + + void ImportGltf(const std::string& filename) + { + Importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + } + + std::string WriteTexturedGltf() + { + const auto filename = WriteGltf(1); + nlohmann::json document; + { + std::ifstream file(filename); + file >> document; + } + document["images"] = { + {{"name", "one_pixel"}, {"uri", "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a3ioAAAAASUVORK5CYII="}} + }; + document["textures"] = {{{"source", 0}}}; + document["materials"][0]["pbrMetallicRoughness"]["baseColorTexture"]["index"] = 0; + std::ofstream(filename) << document.dump(); + return filename; + } + + void ChangeTriangleVertices() + { + const std::array vertices = {0.25f, 0, 0, 1, 0, 0, 0, 1, 0}; + std::ofstream file(TempDirectory / "triangle.bin", std::ios::binary); + file.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + } + + std::string WriteObj(bool with_normals = false) + { + const auto filename = (TempDirectory / "triangle.obj").string(); + std::ofstream file(filename); + file << "o Triangle\nv 0 0 0\nv 1 0 0\nv 0 1 0\n"; + file << (with_normals ? "vn 0 0 1\nf 1//1 2//1 3//1\n" : "f 1 2 3\n"); + return filename; + } + + std::string WriteFbx(bool with_material = false) + { + const auto filename = (TempDirectory / "triangle.fbx").string(); + std::string contents = R"(; FBX 7.4.0 project file +FBXHeaderExtension: { FBXHeaderVersion: 1003 + FBXVersion: 7400 +} +Objects: { + Geometry: 1, "Geometry::Triangle", "Mesh" { + Vertices: *9 { a: 0,0,0,1,0,0,0,1,0 } + PolygonVertexIndex: *3 { a: 0,1,-3 } + } + Model: 2, "Model::Triangle", "Mesh" { Version: 232 } +} +Connections: { + C: "OO",1,2 + C: "OO",2,0 +} +)"; + if (with_material) + { + contents.insert(contents.find("}\nConnections:"), R"( Material: 3, "Material::TestMaterial", "" { + Version: 102 + ShadingModel: "phong" + Properties70: { P: "DiffuseColor", "Color", "", "A", 1, 0, 0 } + } +)"); + contents.insert(contents.rfind('}'), " C: \"OO\",3,2\n"); + } + std::ofstream(filename) << contents; + return filename; + } + + void ReadCookedMesh(AssetMesh& mesh, AssetNodeHierarchy& hierarchy) + { + const auto bytes = ReadFile("/meshes/triangle.zemesh"); + const auto filename = (TempDirectory / "triangle.zemesh").string(); + { + std::ofstream file(filename, std::ios::binary); + file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + DeserializeMeshAssetFile(&Arena, filename.c_str(), mesh, hierarchy); + } + + template + void VerifyInvalidConfigurations(ModelImporter& importer, const std::string& filename) + { + const auto start = importer.Arena.m_current_offset; + auto verify = [&](const ImportConfiguration& config, const char* source) { + const auto failures = Callbacks.Failed; + importer.ImportFile(source, config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, failures + 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(importer.Arena.m_current_offset, start); + EXPECT_NE(Callbacks.Error.find("VFS error"), std::string::npos); + }; + const std::string long_path = "/" + std::string(MAX_FILE_PATH_COUNT, 'x'); + auto invalid = Config; + invalid.VFS = nullptr; + verify(invalid, filename.c_str()); + invalid = Config; + invalid.OutputAssetsPath.init(&Arena, long_path.c_str()); + verify(invalid, filename.c_str()); + invalid = Config; + invalid.OutputAssetFile.init(&Arena, ""); + verify(invalid, filename.c_str()); + invalid.OutputAssetFile.init(&Arena, std::string(MAX_FILE_PATH_COUNT, 'x').c_str()); + verify(invalid, filename.c_str()); + invalid.OutputAssetFile.init(&Arena, std::string(MAX_FILE_PATH_COUNT - 1 - std::string_view("/meshes/").size(), 'x').c_str()); + verify(invalid, filename.c_str()); // Fits VFS, but not its .meta.tmp sidecar. + invalid = Config; + invalid.Options.ImportMaterials = true; + invalid.OutputMaterialPath.init(&Arena, long_path.c_str()); + verify(invalid, filename.c_str()); + invalid = Config; + invalid.Options.ImportMaterials = invalid.Options.ImportTextures = true; + invalid.OutputTextureFilesPath.init(&Arena, long_path.c_str()); + verify(invalid, filename.c_str()); + invalid.OutputTextureFilesPath = Config.OutputTextureFilesPath; + invalid.AssetName.init(&Arena, ""); + verify(invalid, filename.c_str()); + verify(Config, nullptr); + verify(Config, ""); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + } + + template + void VerifySameInstanceSerialization(ModelImporter& importer, const std::string& filename, bool runtime_call) + { + using namespace std::chrono_literals; + std::promise release; + PausedImportCallbacks paused; + paused.Release = release.get_future().share(); + auto entered = paused.Entered.get_future(); + ImportConfiguration second_config = Config; + second_config.OutputAssetFile.init(&Arena, "second.zemesh"); + + auto first = std::async(std::launch::async, [&] { importer.ImportFile(filename.c_str(), Config, &Arena, &paused, PausedImportCallbacks::Complete, PausedImportCallbacks::Advance, PausedImportCallbacks::Fail, nullptr); }); + const bool first_paused = entered.wait_for(5s) == std::future_status::ready; + EXPECT_TRUE(first_paused); + if (!first_paused) + { + release.set_value(); + first.get(); + return; + } + + std::promise second_started; + auto started = second_started.get_future(); + auto second = std::async(std::launch::async, [&] { + second_started.set_value(); + if (runtime_call) + { + // Even a failed runtime import must respect the editor import's lock. + auto result = importer.Import(Context, Path("/zengine_nonexistent_import_source"), {}); + EXPECT_TRUE(result.Failed()); + } + else + importer.ImportFile(filename.c_str(), second_config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + }); + EXPECT_EQ(started.wait_for(5s), std::future_status::ready); + EXPECT_EQ(second.wait_for(100ms), std::future_status::timeout); + release.set_value(); + first.get(); + second.get(); + EXPECT_EQ(paused.Results.Completed, 1) << paused.Results.Error; + EXPECT_EQ(paused.Results.Failed, 0); + if (!runtime_call) + { + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + EXPECT_EQ(Callbacks.Failed, 0); + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + EXPECT_EQ(Callbacks.Outputs[0].Path, "/meshes/second.zemesh"); + } + } +}; + +TEST_F(ModelImporterOutputTest, MeshOnlyGltfOwnsShortAndEmptyOutputStringsAfterImport) +{ + Config.OutputAssetsPath.init(&Arena, "/m"); + Config.OutputAssetFile.init(&Arena, "x.zemesh"); + Config.OutputWorkingSpacePath.init(&Arena, ""); + // A source material must be harmless when material/texture import is disabled. + ImportGltf(WriteGltf(1)); + + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + EXPECT_EQ(Callbacks.Outputs[0].Type, AssetFileType::MESH); + EXPECT_EQ(Callbacks.Outputs[0].Path, "/m/x.zemesh"); + EXPECT_TRUE(Callbacks.Outputs[0].RootPath.empty()); + EXPECT_TRUE(Exists("/m/x.zemesh")); + EXPECT_FLOAT_EQ(Callbacks.Progress, 1.0f); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); +} + +TEST_F(ModelImporterOutputTest, InvalidGltfNodeGraphsFailInEditorAndRuntimeWithoutPublication) +{ + for (int scenario = 0; scenario < 8; ++scenario) + { + SCOPED_TRACE(scenario); + Callbacks = {}; + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + switch (scenario) + { + case 0: + document["scene"] = 1000000; + break; + case 1: + document["scenes"][0]["nodes"] = {1000000}; + break; + case 2: + document["nodes"][0]["mesh"] = 1000000; + break; + case 3: + document["nodes"][0]["children"] = {1000000}; + break; + case 4: + document["nodes"][0]["children"] = {0}; + break; + case 5: + document["nodes"][0]["children"] = {1}; + document["nodes"].push_back({ + {"children", {0}} + }); + break; + case 6: + document["nodes"][0]["children"] = {2}; + document["nodes"].push_back({ + {"children", {2}} + }); + document["nodes"].push_back(nlohmann::json::object()); + document["scenes"][0]["nodes"] = {0, 1}; + break; + case 7: + document["scenes"][0]["nodes"] = {0, 0}; + break; + } + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_EQ(Importer.Import(Context, Path(filename.c_str()), {}).Error(), VFSError::Corrupted); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + } +} + +TEST_F(ModelImporterOutputTest, DeepGltfHierarchyUsesIterativeTraversalAndSizedMaps) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document["nodes"] = nlohmann::json::array(); + constexpr size_t count = 4096; + for (size_t i = 0; i + 1 < count; ++i) + document["nodes"].push_back({ + {"children", {i + 1}} + }); + document["nodes"].push_back({ + {"mesh", 0} + }); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Failed, 0); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(hierarchy.Hierarchies.size(), count + 1); + EXPECT_EQ(hierarchy.NodeNames.size(), count + 1); + EXPECT_EQ(hierarchy.NodeMeshes.size(), 1u); + EXPECT_EQ(hierarchy.Hierarchies.back().DepthLevel, count); +} + +TEST_F(ModelImporterOutputTest, MultipleGltfScenesCanReuseTheSameTree) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document.erase("scene"); + document["scenes"].push_back(document["scenes"][0]); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + EXPECT_EQ(hierarchy.Hierarchies.size(), 4u); + EXPECT_EQ(hierarchy.NodeMeshes.size(), 2u); +} + +TEST_F(ModelImporterOutputTest, OversizedValidGltfFailsCleanlyAndImporterRemainsReusable) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + constexpr size_t count = 1050000; + const std::vector vertices(count * 3, 0.0f); + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), vertices.size() * sizeof(float)); + } + document["buffers"][0]["byteLength"] = vertices.size() * sizeof(float); + document["bufferViews"][0]["byteLength"] = vertices.size() * sizeof(float); + document["accessors"][0]["count"] = count; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_NE(Callbacks.Error.find("scratch memory budget"), std::string::npos); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_EQ(Importer.Import(Context, Path(filename.c_str()), {}).Error(), VFSError::OutOfMemory); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + ImportGltf(WriteGltf()); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, DuplicateGltfImageNamesAreDistinctButSharedImagesKeepIdentity) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + const std::array red = {255, 0, 0, 255}; + const std::array green = {0, 255, 0, 255}; + ASSERT_TRUE(stbi_write_png((TempDirectory / "red.png").string().c_str(), 1, 1, 4, red.data(), 4)); + ASSERT_TRUE(stbi_write_png((TempDirectory / "green.png").string().c_str(), 1, 1, 4, green.data(), 4)); + nlohmann::json document; + std::ifstream(filename) >> document; + document["images"][0]["uri"] = "red.png"; + document["images"].push_back(document["images"][0]); + document["images"][1]["uri"] = "green.png"; + document["textures"].push_back({ + {"source", 1} + }); + document["textures"].push_back({ + {"source", 0} + }); + for (int i = 1; i < 3; ++i) + { + auto material = document["materials"][0]; + material["name"] = std::to_string(i); + material["pbrMetallicRoughness"]["baseColorTexture"]["index"] = i; + document["materials"].push_back(material); + } + std::ofstream(filename) << document.dump(); + auto textures = [&] { + std::vector result; + for (size_t i = 1; i < Callbacks.Outputs.size(); ++i) + { + const auto bytes = ReadFile(Callbacks.Outputs[i].Path.c_str()); + result.push_back(nlohmann::json::parse(bytes.begin(), bytes.end())["textures"]["albedo"]); + } + return result; + }; + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto first = textures(); + ASSERT_EQ(first.size(), 3u); + EXPECT_NE(first[0]["path"], first[1]["path"]); + EXPECT_NE(first[0]["uuid"], first[1]["uuid"]); + EXPECT_EQ(first[0], first[2]); + EXPECT_NE(ReadFile("/textures/triangle/one_pixel_0.png"), ReadFile("/textures/triangle/one_pixel_1.png")); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + EXPECT_EQ(textures(), first); +} + +TEST_F(ModelImporterOutputTest, NativeTextureCopiesWithSameBasenameKeepDifferentContents) +{ + ASSERT_TRUE(std::filesystem::create_directory(TempDirectory / "red")); + ASSERT_TRUE(std::filesystem::create_directory(TempDirectory / "green")); + std::ofstream(TempDirectory / "red/same.png", std::ios::binary) << "red image"; + std::ofstream(TempDirectory / "green/same.png", std::ios::binary) << "green image"; + ZEngine::Core::Containers::Array textures; + textures.init(&Arena, 2, 2); + textures[0].Path.init(&Arena, "red/same.png"); + textures[1].Path.init(&Arena, "green/same.png"); + ASSERT_TRUE(CopyTextureFiles(ArrayView{textures}, Config).Succeeded()); + EXPECT_STREQ(textures[0].Path.c_str(), "/textures/triangle/same_0.png"); + EXPECT_STREQ(textures[1].Path.c_str(), "/textures/triangle/same_1.png"); + EXPECT_NE(ReadFile(textures[0].Path.c_str()), ReadFile(textures[1].Path.c_str())); +} + +TEST_F(ModelImporterOutputTest, UnnamedSharedGltfImageUsesOneStableDestination) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document["images"][0].erase("name"); + document["textures"].push_back({ + {"source", 0} + }); + document["materials"][0]["normalTexture"] = { + {"index", 1} + }; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Failed, 0); + const auto bytes = ReadFile(Callbacks.Outputs[1].Path.c_str()); + const auto material = nlohmann::json::parse(bytes.begin(), bytes.end()); + EXPECT_EQ(material["textures"]["albedo"]["path"], "/textures/triangle/image_0.png"); + EXPECT_EQ(material["textures"]["albedo"], material["textures"]["normal"]); +} + +TEST_F(ModelImporterOutputTest, CorruptTextureIdentityIsRepairedBeforeMaterialPublication) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + ASSERT_TRUE(Context.CreateDir(Path("/textures/triangle")).Succeeded()); + const std::string metadata = R"({"uuid":"invalid","source_hash":0})"; + ASSERT_TRUE(WriteFileAtomically(Context, Path("/textures/triangle/one_pixel_0.png.meta"), {reinterpret_cast(metadata.data()), metadata.size()}).Succeeded()); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Failed, 0); + const auto bytes = ReadFile(Callbacks.Outputs[1].Path.c_str()); + const auto material = nlohmann::json::parse(bytes.begin(), bytes.end()); + auto meta = MetaFileIO::Read(Context, Path("/textures/triangle/one_pixel_0.png")); + ASSERT_TRUE(meta.Succeeded()); + EXPECT_FALSE(meta.Value().AssetUUID.is_nil()); + EXPECT_EQ(material["textures"]["albedo"]["uuid"], uuids::to_string(meta.Value().AssetUUID)); +} + +TEST_F(ModelImporterOutputTest, ModelMetadataReadErrorsNeverPublishReplacementIdentities) +{ + auto verify = [&](auto& importer, const std::string& filename) { + for (bool material : {false, true}) + { + Config.Options.ImportMaterials = material; + Context.MetadataReadPath = material ? "/materials/" : "/meshes/"; + for (auto error : {VFSError::PermissionDenied, VFSError::IOError, VFSError::OutOfMemory}) + { + SCOPED_TRACE(static_cast(error)); + Context.MetadataReadError = error; + Callbacks = {}; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1) << Callbacks.Error; + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + } + } + Context.MetadataReadError = VFSError::OK; + }; + verify(Importer, WriteGltf(1)); + AssimpImporter assimp; + assimp.Initialize(&Arena); + verify(assimp, WriteObj()); + FbxImporter fbx; + fbx.Initialize(&Arena); + verify(fbx, WriteFbx(true)); +} + +TEST_F(ModelImporterOutputTest, MaterialOutputsExceedOldCapacityAndOwnLongPaths) +{ + const std::string root = "/project/" + std::string(100, 'p'); + Config.OutputWorkingSpacePath.init(&Arena, root.c_str()); + Config.Options.ImportMaterials = true; + ImportGltf(WriteGltf(20)); + + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 21u); + for (size_t i = 0; i < 20; ++i) + { + const auto& output = Callbacks.Outputs[i + 1]; + EXPECT_EQ(output.Type, AssetFileType::MATERIAL); + EXPECT_EQ(output.Path, "/materials/triangle/" + std::string(80, 'm') + std::to_string(i) + "_" + std::to_string(i) + ".zematerial"); + EXPECT_EQ(output.RootPath, root); + EXPECT_TRUE(Exists(output.Path.c_str())); + } +} + +TEST_F(ModelImporterOutputTest, MeshOnlyGltfArtifactRoundTrips) +{ + ImportGltf(WriteGltf()); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.Vertices.size(), 24u); + ASSERT_EQ(mesh.Indices.size(), 3u); + ASSERT_EQ(mesh.SubMeshes.size(), 1u); + EXPECT_EQ(mesh.MeshUUID, hierarchy.MeshUUID); + EXPECT_EQ(mesh.SubMeshes[0].VertexCount, 3u); + EXPECT_EQ(mesh.SubMeshes[0].IndexCount, 3u); + EXPECT_TRUE(hierarchy.MaterialNames.empty()); +} + +TEST_F(ModelImporterOutputTest, MaterialWriteFailureNeverReportsSuccess) +{ + Config.Options.ImportMaterials = true; + Context.Point = FailurePoint::Write; + Context.FailurePath = "/materials/"; + ImportGltf(WriteGltf(2)); + + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_TRUE(Callbacks.Outputs.empty()); + EXPECT_LT(Callbacks.Progress, 1.0f); + EXPECT_NE(Callbacks.Error.find("/materials/triangle/" + std::string(80, 'm') + "0_0.zematerial"), std::string::npos); + EXPECT_NE(Callbacks.Error.find("VFS error 8"), std::string::npos); + ExpectNoTemporaryFiles("/materials"); +} + +TEST_F(ModelImporterOutputTest, AssimpWriteFailureReleasesScratchAndAllowsRetry) +{ + const auto filename = WriteObj(true); + Config.Options.ImportMaterials = true; + AssimpImporter importer; + importer.Initialize(&Arena); + Context.Point = FailurePoint::Write; + Context.FailurePath = "/meshes/"; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_NE(Callbacks.Error.find("/meshes/triangle.zemesh"), std::string::npos); + EXPECT_EQ(importer.Arena.m_current_offset, importer.Arena.m_initial_current_offset); + // The same importer must still work after failure. + Context.Point = FailurePoint::None; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Completed, 1); +} + +TEST_F(ModelImporterOutputTest, FbxWriteFailureReleasesScratchAndAllowsRetry) +{ + const auto filename = WriteFbx(); + FbxImporter importer; + importer.Initialize(&Arena); + Context.Point = FailurePoint::Write; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + + ASSERT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_NE(Callbacks.Error.find("/meshes/triangle.zemesh"), std::string::npos) << Callbacks.Error; + Context.Point = FailurePoint::None; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + EXPECT_EQ(Callbacks.Outputs[0].Path, "/meshes/triangle.zemesh"); +} + +TEST_F(ModelImporterOutputTest, AssimpWithoutNormalsOrMaterialsUsesFallbacks) +{ + const auto filename = WriteObj(); + AssimpImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 1u); + + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.Vertices.size(), 24u); + ASSERT_EQ(mesh.SubMeshes.size(), 1u); + EXPECT_TRUE(mesh.SubMeshes[0].MaterialUUID.is_nil()); + EXPECT_TRUE(hierarchy.MaterialNames.empty()); + EXPECT_TRUE(hierarchy.NodeMaterials.empty()); + for (size_t vertex = 0; vertex < 3; ++vertex) + { + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 3], 0); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 4], 1); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 5], 0); + } +} + +TEST_F(ModelImporterOutputTest, AssimpPreservesProvidedNormalsAndMaterialIdentity) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteObj(true); + AssimpImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 2u); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.SubMeshes.size(), 1u); + EXPECT_FALSE(mesh.SubMeshes[0].MaterialUUID.is_nil()); + EXPECT_FALSE(hierarchy.NodeMaterials.empty()); + EXPECT_EQ(hierarchy.MaterialNames.size(), 1u); + for (size_t vertex = 0; vertex < 3; ++vertex) + { + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 3], 0); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 4], 0); + EXPECT_FLOAT_EQ(mesh.Vertices[vertex * 8 + 5], 1); + } +} + +TEST_F(ModelImporterOutputTest, GltfEditorImportsShareOneScratchLock) +{ + VerifySameInstanceSerialization(Importer, WriteGltf(), false); +} + +TEST_F(ModelImporterOutputTest, GltfRuntimeAndEditorImportsShareOneScratchLock) +{ + VerifySameInstanceSerialization(Importer, WriteGltf(), true); +} + +TEST_F(ModelImporterOutputTest, AssimpEditorImportsShareOneScratchLock) +{ + AssimpImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteObj(), false); +} + +TEST_F(ModelImporterOutputTest, AssimpRuntimeAndEditorImportsShareOneScratchLock) +{ + AssimpImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteObj(), true); +} + +TEST_F(ModelImporterOutputTest, FbxEditorImportsShareOneScratchLock) +{ + FbxImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteFbx(), false); +} + +TEST_F(ModelImporterOutputTest, FbxRuntimeAndEditorImportsShareOneScratchLock) +{ + FbxImporter importer; + importer.Initialize(&Arena); + VerifySameInstanceSerialization(importer, WriteFbx(), true); +} + +TEST_F(ModelImporterOutputTest, SeparateImporterInstancesRemainIndependent) +{ + using namespace std::chrono_literals; + const auto filename = WriteGltf(); + GltfImporter other; + other.Initialize(&Arena); + std::promise release; + PausedImportCallbacks paused; + paused.Release = release.get_future().share(); + auto entered = paused.Entered.get_future(); + auto first = std::async(std::launch::async, [&] { Importer.ImportFile(filename.c_str(), Config, &Arena, &paused, PausedImportCallbacks::Complete, PausedImportCallbacks::Advance, PausedImportCallbacks::Fail, nullptr); }); + EXPECT_EQ(entered.wait_for(5s), std::future_status::ready); + auto second = std::async(std::launch::async, [&] { other.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); }); + EXPECT_EQ(second.wait_for(5s), std::future_status::ready); + release.set_value(); + first.get(); + second.get(); + EXPECT_EQ(paused.Results.Completed, 1) << paused.Results.Error; + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, CodecRejectsInvalidOutputInsteadOfReturningAnEmptyAsset) +{ + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + Config.OutputAssetFile.init(&Arena, ""); + EXPECT_EQ(SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config).Error(), VFSError::InvalidPath); + Config.OutputAssetFile.init(&Arena, "triangle.zemesh"); + Config.OutputAssetsPath.init(&Arena, ("/" + std::string(MAX_FILE_PATH_COUNT, 'x')).c_str()); + EXPECT_EQ(SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config).Error(), VFSError::InvalidPath); + Config.VFS = nullptr; + EXPECT_EQ(SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config).Error(), VFSError::Unsupported); +} + +TEST_F(ModelImporterOutputTest, GltfInvalidConfigurationsReportErrorsAndAllowRetry) +{ + VerifyInvalidConfigurations(Importer, WriteGltf()); +} + +TEST_F(ModelImporterOutputTest, AssimpInvalidConfigurationsReportErrorsAndAllowRetry) +{ + AssimpImporter importer; + importer.Initialize(&Arena); + VerifyInvalidConfigurations(importer, WriteObj()); +} + +TEST_F(ModelImporterOutputTest, FbxInvalidConfigurationsReportErrorsAndAllowRetry) +{ + FbxImporter importer; + importer.Initialize(&Arena); + VerifyInvalidConfigurations(importer, WriteFbx()); +} + +TEST_F(ModelImporterOutputTest, InvalidMaterialNameReportsErrorBeforePublishingMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(1); + nlohmann::json document; + { + std::ifstream file(filename); + file >> document; + } + document["materials"][0]["name"] = std::string(MAX_FILE_PATH_COUNT, 'm'); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); +} + +TEST_F(ModelImporterOutputTest, FailedMaterialRecookPreservesPreviouslyPublishedMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(2); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1); + const auto before = ReadFile("/meshes/triangle.zemesh"); + ChangeTriangleVertices(); + // Fail the second material, after the first dependency was published. + Context.FailurePath = std::string(80, 'm') + "1_1.zematerial"; + // Unique temporary names no longer contain the destination filename; + // fail its final rename instead, preserving the old material as well. + Context.Point = FailurePoint::Rename; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before); + ExpectNoTemporaryFiles("/materials"); + Context.Point = FailurePoint::None; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 2); + EXPECT_NE(ReadFile("/meshes/triangle.zemesh"), before); +} + +TEST_F(ModelImporterOutputTest, AssimpMaterialWriteFailurePreservesPreviouslyPublishedMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteObj(); + AssimpImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + ASSERT_EQ(Callbacks.Completed, 1); + const auto before = ReadFile("/meshes/triangle.zemesh"); + std::ofstream(filename) << "o Triangle\nv 0.25 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n"; + Context.Point = FailurePoint::Write; + Context.FailurePath = "/materials/"; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before); + ExpectNoTemporaryFiles("/materials"); +} + +TEST_F(ModelImporterOutputTest, FbxMaterialWriteFailurePreservesPreviouslyPublishedMesh) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteFbx(true); + FbxImporter importer; + importer.Initialize(&Arena); + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Outputs.size(), 2u); + const auto before = ReadFile("/meshes/triangle.zemesh"); + std::string contents; + { + std::ifstream file(filename); + contents.assign(std::istreambuf_iterator(file), std::istreambuf_iterator()); + } + contents.replace(contents.find("0,0,0,1,0,0"), std::string_view("0,0,0,1,0,0").size(), "0.25,0,0,1,0,0"); + std::ofstream(filename) << contents; + Context.Point = FailurePoint::Write; + Context.FailurePath = "/materials/"; + importer.ImportFile(filename.c_str(), Config, &Arena, &Callbacks, ImportCallbacks::Complete, ImportCallbacks::Advance, ImportCallbacks::Fail, nullptr); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before); + ExpectNoTemporaryFiles("/materials"); +} + +TEST_F(ModelImporterOutputTest, IndependentAtomicWritersUseDifferentTemporarySiblings) +{ + struct InterleavedContext : VFSContext + { + bool Interleaved = false; + bool NestedSucceeded = false; + std::vector TemporaryPaths; + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + if (HasFlag(flags, VFSOpenFlags::Write)) + TemporaryPaths.push_back(path); + return VFSContext::Open(path, flags); + } + VFSResult Rename(const VFSPath& source, const VFSPath& destination) override + { + if (!Interleaved) + { + Interleaved = true; + const std::array other = {9, 9}; + NestedSucceeded = WriteFileAtomically(*this, destination, {other.data(), other.size()}).Succeeded(); + } + return VFSContext::Rename(source, destination); + } + } interleaved; + interleaved.Initialize(&Arena, 4); + ASSERT_TRUE(interleaved.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + ASSERT_TRUE(interleaved.CreateDir(Path("/meshes")).Succeeded()); + const std::array bytes = {1, 2, 3, 4}; + EXPECT_TRUE(WriteFileAtomically(interleaved, Path("/meshes/collision.zemesh"), {bytes.data(), bytes.size()}).Succeeded()); + EXPECT_TRUE(interleaved.NestedSucceeded); + ASSERT_EQ(interleaved.TemporaryPaths.size(), 2u); + EXPECT_NE(interleaved.TemporaryPaths[0], interleaved.TemporaryPaths[1]); + EXPECT_EQ(ReadFile("/meshes/collision.zemesh"), (std::vector{1, 2, 3, 4})); + ExpectNoTemporaryFiles("/meshes"); + interleaved.Shutdown(); +} + +TEST_F(ModelImporterOutputTest, AtomicWriteSupportsMaximumLengthDestinationAndEmptyData) +{ + const std::string filename(MAX_FILE_PATH_COUNT - 1 - std::string_view("/meshes/").size(), 'x'); + auto path = VFSPath::Parse(("/meshes/" + filename).c_str()); + ASSERT_TRUE(path.Succeeded()); + EXPECT_EQ(MakeOutputPath("/meshes", filename.c_str()).Error(), VFSError::InvalidPath); + ASSERT_TRUE(Context.CreateDir(Path("/meshes")).Succeeded()); + EXPECT_TRUE(WriteFileAtomically(Context, path.Value(), {}).Succeeded()); + EXPECT_TRUE(ReadFile(path.Value().CStr()).empty()); + EXPECT_EQ(WriteFileAtomically(Context, VFSPath::Root(), {}).Error(), VFSError::InvalidPath); + EXPECT_EQ(WriteFileAtomically(Context, VFSPath{}, {}).Error(), VFSError::InvalidPath); + ExpectNoTemporaryFiles("/meshes"); +} + +TEST_F(ModelImporterOutputTest, CookedPathsReserveRoomForMetadataSidecars) +{ + const std::string filename(MAX_FILE_PATH_COUNT - 1 - std::string_view("/meshes/").size() - std::string_view(".meta").size(), 'x'); + EXPECT_TRUE(MakeOutputPath("/meshes", filename.c_str()).Succeeded()); + EXPECT_EQ(MakeOutputPath("/meshes", (filename + "x").c_str()).Error(), VFSError::InvalidPath); + EXPECT_EQ(MakeOutputPath("/meshes", "../outside.zemesh").Error(), VFSError::InvalidPath); +} + +TEST_F(ModelImporterOutputTest, CodecSupportsUninitializedOptionalRootAndMaterialName) +{ + Config.OutputWorkingSpacePath = {}; + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + auto mesh_result = SerializeMeshAssetFile(&Arena, mesh, hierarchy, Config); + ASSERT_TRUE(mesh_result.Succeeded()); + EXPECT_TRUE(mesh_result.Value().RootPath.empty()); + AssetMaterial material; + auto material_result = SerializeMaterialAssetFile(&Arena, material, Config, 0); + ASSERT_TRUE(material_result.Succeeded()); + EXPECT_EQ(material_result.Value().Path, "/materials/triangle/material_0.zematerial"); + EXPECT_TRUE(material_result.Value().RootPath.empty()); + Config.AssetName = {}; + EXPECT_EQ(SerializeTextureAssetFiles(&Arena, {}, Config).Error(), VFSError::InvalidPath); +} + +TEST_F(ModelImporterOutputTest, SkippedPrimitivesDoNotShiftMaterialsOrHierarchySubmeshIndices) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(2); + nlohmann::json document; + { + std::ifstream file(filename); + file >> document; + } + auto triangle = document["meshes"][0]["primitives"][0]; + triangle["material"] = 1; + auto points = triangle; + points["mode"] = 0; + points["material"] = 0; + // Includes a mesh with no surviving primitives, plus a multi-primitive mesh. + document["meshes"] = {{{"primitives", {points}}}, {{"primitives", {points, triangle, triangle}}}, {{"primitives", {triangle}}}}; + document["nodes"] = {{{"mesh", 0}}, {{"mesh", 1}}, {{"mesh", 2}}}; + document["scenes"][0]["nodes"] = {0, 1, 2}; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + ASSERT_EQ(Callbacks.Outputs.size(), 3u); + const auto material = nlohmann::json::parse(ReadFile(Callbacks.Outputs[2].Path.c_str())); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.SubMeshes.size(), 3u); + for (const auto& submesh : mesh.SubMeshes) + EXPECT_EQ(uuids::to_string(submesh.MaterialUUID), material["uuid"].get()); + ASSERT_EQ(hierarchy.NodeMeshes.size(), 3u); + std::array mapped = {}; + for (const auto& [node, index] : hierarchy.NodeMeshes) + { + ASSERT_LT(index, mapped.size()); + mapped[index] = true; + } + EXPECT_TRUE(mapped[0] && mapped[1] && mapped[2]); +} + +TEST_F(ModelImporterOutputTest, DisabledGltfMaterialsLeaveNoDanglingHierarchyMaterialIndices) +{ + ImportGltf(WriteGltf(1)); + ASSERT_EQ(Callbacks.Completed, 1); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + EXPECT_TRUE(hierarchy.NodeMaterials.empty()); + EXPECT_TRUE(hierarchy.MaterialNames.empty()); + EXPECT_TRUE(mesh.SubMeshes[0].MaterialUUID.is_nil()); +} + +TEST_F(ModelImporterOutputTest, SharedTextureCopyReportsMissingSourcesAndWriteErrors) +{ + ZEngine::Core::Containers::Array textures; + textures.init(&Arena, 1, 1); + textures[0].Path.init(&Arena, "missing.png"); + EXPECT_EQ(CopyTextureFiles(ArrayView{textures}, Config).Error(), VFSError::NotFound); + std::ofstream(TempDirectory / "present.png", std::ios::binary) << "test image bytes"; + textures[0].Path.init(&Arena, "present.png"); + Context.Point = FailurePoint::Write; + Context.FailurePath = "/textures/"; + EXPECT_EQ(CopyTextureFiles(ArrayView{textures}, Config).Error(), VFSError::IOError); + EXPECT_STREQ(textures[0].Path.c_str(), "present.png"); + ExpectNoTemporaryFiles("/textures/triangle"); + Context.Point = FailurePoint::None; + EXPECT_TRUE(CopyTextureFiles(ArrayView{textures}, Config).Succeeded()); + EXPECT_STREQ(textures[0].Path.c_str(), "/textures/triangle/present_0.png"); +} + +TEST_F(ModelImporterOutputTest, SharedGltfTextureUpdatesEveryMaterialSlotAndSurvivesRecook) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + auto& material = document["materials"][0]; + material["emissiveTexture"] = { + {"index", 0} + }; + material["normalTexture"] = { + {"index", 0} + }; + material["occlusionTexture"] = { + {"index", 0} + }; + material["pbrMetallicRoughness"]["metallicRoughnessTexture"] = { + {"index", 0} + }; + auto second = material; + second["name"] = "second"; + document["materials"].push_back(second); + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Failed, 0) << Callbacks.Error; + ASSERT_EQ(Callbacks.Completed, 1); + ASSERT_EQ(Callbacks.Outputs.size(), 3u); + const auto first = nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str())); + const auto expected = first["textures"]["albedo"]; + EXPECT_EQ(expected["path"], "/textures/triangle/one_pixel_0.png"); + for (size_t m = 1; m < Callbacks.Outputs.size(); ++m) + { + const auto cooked = nlohmann::json::parse(ReadFile(Callbacks.Outputs[m].Path.c_str())); + for (const auto* slot : {"albedo", "emissive", "normal", "opacity", "specular"}) + EXPECT_EQ(cooked["textures"][slot], expected); + } + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2); + EXPECT_EQ(nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str()))["textures"]["albedo"], expected); +} + +TEST_F(ModelImporterOutputTest, UnnamedAndDuplicateMaterialsKeepDistinctStableOutputsAndDisplayNames) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(3); + nlohmann::json document; + std::ifstream(filename) >> document; + document["materials"][0].erase("name"); + document["materials"][1]["name"] = "same"; + document["materials"][2]["name"] = "same"; + const auto primitive = document["meshes"][0]["primitives"][0]; + for (size_t i = 1; i < 3; ++i) + { + auto next = primitive; + next["material"] = i; + document["meshes"][0]["primitives"].push_back(next); + } + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto outputs = Callbacks.Outputs; + EXPECT_EQ(outputs[1].Path, "/materials/triangle/material_0.zematerial"); + EXPECT_EQ(outputs[2].Path, "/materials/triangle/same_1.zematerial"); + EXPECT_EQ(outputs[3].Path, "/materials/triangle/same_2.zematerial"); + AssetMesh mesh; + AssetNodeHierarchy hierarchy; + ReadCookedMesh(mesh, hierarchy); + ASSERT_EQ(mesh.SubMeshes.size(), 3u); + for (size_t m = 0; m < 3; ++m) + { + const auto cooked = nlohmann::json::parse(ReadFile(outputs[m + 1].Path.c_str())); + EXPECT_EQ(cooked["name"], m == 0 ? "material" : "same"); + EXPECT_EQ(cooked["uuid"], uuids::to_string(mesh.SubMeshes[m].MaterialUUID)); + MetaFileData meta; + meta.AssetUUID = mesh.SubMeshes[m].MaterialUUID; + ASSERT_TRUE(MetaFileIO::Write(Context, Path(outputs[m + 1].Path.c_str()), meta).Succeeded()); + } + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + for (size_t m = 0; m < 3; ++m) + { + EXPECT_EQ(Callbacks.Outputs[m + 1].Path, outputs[m + 1].Path); + EXPECT_EQ(nlohmann::json::parse(ReadFile(outputs[m + 1].Path.c_str()))["uuid"], uuids::to_string(mesh.SubMeshes[m].MaterialUUID)); + } +} + +TEST_F(ModelImporterOutputTest, DifferentModelsCannotOverwriteEachOthersMaterials) +{ + Config.Options.ImportMaterials = true; + const auto filename = WriteGltf(1); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto original_path = Callbacks.Outputs[1].Path; + const auto original_bytes = ReadFile(original_path.c_str()); + Config.AssetName.init(&Arena, "other_model"); + Config.OutputAssetFile.init(&Arena, "other_model.zemesh"); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + EXPECT_NE(Callbacks.Outputs[1].Path, original_path); + EXPECT_EQ(ReadFile(original_path.c_str()), original_bytes); +} + +TEST_F(ModelImporterOutputTest, InvalidImageBufferRangesAreRejectedInEditorAndRuntime) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + nlohmann::json original; + std::ifstream(filename) >> original; + for (size_t variant = 0; variant < 5; ++variant) + { + auto document = original; + document["bufferViews"].push_back({ + { "buffer", 0}, + {"byteLength", 36} + }); + document["images"][0] = { + { "name", "bad"}, + {"bufferView", 1}, + { "mimeType", "image/png"} + }; + if (variant == 0) + document["bufferViews"][1]["byteOffset"] = 1073741824; + else if (variant == 1) + document["bufferViews"][1]["byteLength"] = 37; + else if (variant == 2) + document["bufferViews"][1]["buffer"] = 1; + else if (variant == 3) + document["images"][0]["bufferView"] = 2; + else + document["textures"][0]["source"] = 1; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Callbacks.Failed, static_cast(variant + 1)); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + MetaFileData meta; + EXPECT_TRUE(Importer.Import(Context, VFSPath::FromNative(filename.c_str()).Value(), meta).Failed()); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + } + std::ofstream(filename) << original.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, InvalidTriangleCountsAndAccessorRangesReportErrorsAndAllowRetry) +{ + const auto filename = WriteGltf(); + nlohmann::json original; + std::ifstream(filename) >> original; + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0}; + for (size_t variant = 0; variant < 5; ++variant) + { + auto document = original; + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + } + document["buffers"][0]["byteLength"] = sizeof(vertices); + document["bufferViews"][0]["byteLength"] = sizeof(vertices); + if (variant == 0) + document["accessors"][0]["count"] = 4; + else if (variant == 1) + document["accessors"][0]["byteOffset"] = 16; + else if (variant == 2) + document["bufferViews"][0]["byteStride"] = 0; + else if (variant == 3) + document["meshes"][0]["primitives"][0]["attributes"]["POSITION"] = 1; + else + { + document["accessors"].push_back(document["accessors"][0]); + document["accessors"][1]["count"] = 4; + document["meshes"][0]["primitives"][0]["attributes"]["NORMAL"] = 1; + } + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, static_cast(variant + 1)); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + } + std::ofstream(filename) << original.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; +} + +TEST_F(ModelImporterOutputTest, IndexedTrianglesRejectInvalidCountsAndVertexReferences) +{ + const auto filename = WriteGltf(); + nlohmann::json document; + std::ifstream(filename) >> document; + document["buffers"][0]["byteLength"] = 52; + document["bufferViews"].push_back({ + { "buffer", 0}, + {"byteOffset", 36}, + {"byteLength", 16} + }); + document["accessors"].push_back({ + { "bufferView", 1}, + {"componentType", 5125}, + { "count", 4}, + { "type", "SCALAR"} + }); + document["meshes"][0]["primitives"][0]["indices"] = 1; + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + for (size_t variant = 0; variant < 3; ++variant) + { + const std::array indices = {0, 1, variant == 1 ? 3u : 2u, 0}; + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + binary.write(reinterpret_cast(indices.data()), sizeof(indices)); + } + document["accessors"][1]["count"] = variant == 0 ? 4 : 3; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, variant == 2 ? 1 : 0) << Callbacks.Error; + EXPECT_EQ(Callbacks.Failed, variant == 0 ? 1 : 2); + } +} + +TEST_F(ModelImporterOutputTest, EmbeddedImageAtExactBufferBoundaryImportsSuccessfully) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto image = ReadFile("/textures/triangle/one_pixel_0.png"); + const auto cooked = nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str())); + nlohmann::json document; + std::ifstream(filename) >> document; + const std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + { + std::ofstream binary(TempDirectory / "triangle.bin", std::ios::binary); + binary.write(reinterpret_cast(vertices.data()), sizeof(vertices)); + binary.write(reinterpret_cast(image.data()), image.size()); + } + document["buffers"][0]["byteLength"] = sizeof(vertices) + image.size(); + document["bufferViews"].push_back({ + { "buffer", 0}, + {"byteOffset", sizeof(vertices)}, + {"byteLength", image.size()} + }); + document["images"][0] = { + { "name", "one_pixel"}, + {"bufferView", 1}, + { "mimeType", "image/png"} + }; + std::ofstream(filename) << document.dump(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 2) << Callbacks.Error; + EXPECT_EQ(Callbacks.Failed, 0); + EXPECT_EQ(ReadFile("/textures/triangle/one_pixel_0.png"), image); + EXPECT_EQ(nlohmann::json::parse(ReadFile(Callbacks.Outputs[1].Path.c_str()))["textures"], cooked["textures"]); +} + +TEST_F(ModelImporterOutputTest, MetadataWritersUseIndependentTemporaryFilesEvenWhenOneFails) +{ + struct InterleavedMetadataContext : VFSContext + { + bool InsideOtherWriter = false; + bool OtherSucceeded = false; + VFSPath AssetPath; + VFSPath FirstTemporary; + VFSPath OtherTemporary; + MetaFileData OtherData{}; + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + auto result = VFSContext::Open(path, flags); + if (result.Succeeded() && HasFlag(flags, VFSOpenFlags::Write)) + { + if (InsideOtherWriter) + OtherTemporary = path; + else + { + FirstTemporary = path; + InsideOtherWriter = true; + OtherSucceeded = MetaFileIO::Write(*this, AssetPath, OtherData).Succeeded(); + InsideOtherWriter = false; + } + } + return result; + } + VFSResult Rename(const VFSPath& src, const VFSPath& dst) override + { + return InsideOtherWriter ? VFSContext::Rename(src, dst) : VFSResult::Fail(VFSError::IOError); + } + } writers; + writers.Initialize(&Arena, 4); + ASSERT_TRUE(writers.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + ASSERT_TRUE(writers.CreateDir(Path("/textures")).Succeeded()); + writers.AssetPath = Path("/textures/shared.png"); + writers.OtherData.AssetUUID = uuids::uuid::from_string("22222222-2222-4222-8222-222222222222").value(); + MetaFileData first; + first.AssetUUID = uuids::uuid::from_string("11111111-1111-4111-8111-111111111111").value(); + EXPECT_TRUE(MetaFileIO::Write(writers, writers.AssetPath, first).Failed()); + ASSERT_TRUE(writers.OtherSucceeded); + EXPECT_NE(writers.FirstTemporary, writers.OtherTemporary); + auto published = MetaFileIO::Read(writers, writers.AssetPath); + ASSERT_TRUE(published.Succeeded()); + EXPECT_EQ(published.Value().AssetUUID, writers.OtherData.AssetUUID); + ExpectNoTemporaryFiles("/textures"); + writers.Shutdown(); +} + +TEST_F(ModelImporterOutputTest, ConcurrentMetadataCreationKeepsOneIdentity) +{ + ASSERT_TRUE(Context.CreateDir(Path("/textures")).Succeeded()); + auto create = [&] { return MetaFileIO::GetOrCreate(Context, Path("/textures/shared.png"), "Test", 1); }; + auto first = std::async(std::launch::async, create); + auto second = std::async(std::launch::async, create); + auto one = first.get(); + auto two = second.get(); + ASSERT_TRUE(one.Succeeded()); + ASSERT_TRUE(two.Succeeded()); + EXPECT_EQ(one.Value().AssetUUID, two.Value().AssetUUID); + EXPECT_FALSE(one.Value().AssetUUID.is_nil()); +} + +class ModelImporterMetadataFailureTest : public ModelImporterOutputTest, public ::testing::WithParamInterface +{ +}; + +TEST_P(ModelImporterMetadataFailureTest, TextureMetadataFailureStopsImportAndAllowsRetry) +{ + Config.Options.ImportMaterials = true; + Config.Options.ImportTextures = true; + Context.Point = GetParam(); + Context.FailurePath = "/textures/"; + Context.FailureWriteNumber = 2; // Image publication succeeds; its metadata fails. + const auto filename = WriteTexturedGltf(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + EXPECT_FALSE(Exists("/textures/triangle/one_pixel_0.png.meta")); + ExpectNoTemporaryFiles("/textures/triangle"); + Context.Point = FailurePoint::None; + Context.FailureWriteNumber = 0; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + EXPECT_TRUE(Exists("/textures/triangle/one_pixel_0.png.meta")); +} + +TEST_P(ModelImporterMetadataFailureTest, FailedMetadataUpdatePreservesPreviousIdentityAndHash) +{ + ASSERT_TRUE(Context.CreateDir(Path("/textures")).Succeeded()); + const auto path = Path("/textures/shared.png"); + auto initial = MetaFileIO::GetOrCreate(Context, path, "Test", 1); + ASSERT_TRUE(initial.Succeeded()); + Context.WriteNumber = 0; + Context.FailureWriteNumber = 1; + Context.Point = GetParam(); + Context.FailurePath = "/textures/"; + EXPECT_TRUE(MetaFileIO::GetOrCreate(Context, path, "Test", 2).Failed()); + auto old = MetaFileIO::Read(Context, path); + ASSERT_TRUE(old.Succeeded()); + EXPECT_EQ(old.Value().SourceHash, 1u); + EXPECT_EQ(old.Value().AssetUUID, initial.Value().AssetUUID); + ExpectNoTemporaryFiles("/textures"); +} + +INSTANTIATE_TEST_SUITE_P(VFSStages, ModelImporterMetadataFailureTest, ::testing::Values(FailurePoint::Open, FailurePoint::Write, FailurePoint::ShortWrite, FailurePoint::Flush, FailurePoint::Close, FailurePoint::Rename)); + +class ModelImporterWriteFailureTest : public ModelImporterOutputTest, public ::testing::WithParamInterface +{ +}; + +TEST_P(ModelImporterWriteFailureTest, MeshWriteFailureStopsImportAndAllowsRetry) +{ + const auto filename = WriteGltf(); + Context.Point = GetParam(); + ImportGltf(filename); + + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 0); + EXPECT_TRUE(Callbacks.Outputs.empty()); + EXPECT_LT(Callbacks.Progress, 1.0f); + EXPECT_NE(Callbacks.Error.find("/meshes/triangle.zemesh"), std::string::npos); + EXPECT_NE(Callbacks.Error.find("VFS error"), std::string::npos); + EXPECT_FALSE(Exists("/meshes/triangle.zemesh")); + ExpectNoTemporaryFiles("/meshes"); + EXPECT_EQ(Importer.Arena.m_current_offset, Importer.Arena.m_initial_current_offset); + + Context.Point = FailurePoint::None; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_TRUE(Exists("/meshes/triangle.zemesh")); +} + +TEST_P(ModelImporterWriteFailureTest, FailedRecookPreservesExistingArtifact) +{ + const auto filename = WriteGltf(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1); + const auto before = ReadFile("/meshes/triangle.zemesh"); + ASSERT_FALSE(before.empty()); + + Context.Point = GetParam(); + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_EQ(before, ReadFile("/meshes/triangle.zemesh")); + ExpectNoTemporaryFiles("/meshes"); +} + +TEST_P(ModelImporterWriteFailureTest, TextureFailureStopsImportAndPreservesExistingMeshAndTexture) +{ + Config.Options.ImportMaterials = Config.Options.ImportTextures = true; + const auto filename = WriteTexturedGltf(); + ImportGltf(filename); + ASSERT_EQ(Callbacks.Completed, 1) << Callbacks.Error; + const auto before_mesh = ReadFile("/meshes/triangle.zemesh"); + const auto before_texture = ReadFile("/textures/triangle/one_pixel_0.png"); + ChangeTriangleVertices(); + Context.Point = GetParam(); + Context.FailurePath = "/textures/"; + ImportGltf(filename); + EXPECT_EQ(Callbacks.Failed, 1); + EXPECT_EQ(Callbacks.Completed, 1); + EXPECT_LT(Callbacks.Progress, 1.0f); + EXPECT_EQ(ReadFile("/meshes/triangle.zemesh"), before_mesh); + EXPECT_EQ(ReadFile("/textures/triangle/one_pixel_0.png"), before_texture); + ExpectNoTemporaryFiles("/textures/triangle"); +} + +INSTANTIATE_TEST_SUITE_P(VFSStages, ModelImporterWriteFailureTest, ::testing::Values(FailurePoint::CreateDir, FailurePoint::Open, FailurePoint::Write, FailurePoint::ShortWrite, FailurePoint::Flush, FailurePoint::Close, FailurePoint::Rename)); diff --git a/ZEngine/tests/VFS/vfs_disk_test.cpp b/ZEngine/tests/VFS/vfs_disk_test.cpp index e362094c..96df978a 100644 --- a/ZEngine/tests/VFS/vfs_disk_test.cpp +++ b/ZEngine/tests/VFS/vfs_disk_test.cpp @@ -80,6 +80,34 @@ TEST_F(VFSDiskBackendTest, OpenMissingFileFailsWithNotFound) EXPECT_EQ(r.Error(), VFSError::NotFound); } +TEST_F(VFSDiskBackendTest, DirectoryOpenFailureIsNotReportedAsMissing) +{ + ASSERT_TRUE(std::filesystem::create_directory(std::filesystem::path(m_root.c_str()) / "existing")); + auto opened = m_backend.Open(VFSPath::Parse("/existing").Value(), VFSOpenFlags::Write); + ASSERT_TRUE(opened.Failed()); + EXPECT_NE(opened.Error(), VFSError::NotFound); +} + +#if !defined(_WIN32) +TEST_F(VFSDiskBackendTest, UnreadableExistingFileReturnsPermissionDenied) +{ + WriteFile("/restricted.meta", "existing metadata"); + const auto path = std::filesystem::path(m_root.c_str()) / "restricted.meta"; + std::error_code error; + std::filesystem::permissions(path, std::filesystem::perms::none, error); + ASSERT_FALSE(error); + auto opened = m_backend.Open(VFSPath::Parse("/restricted.meta").Value(), VFSOpenFlags::Read); + std::filesystem::permissions(path, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write, error); + ASSERT_FALSE(error); + if (opened.Succeeded()) + { + m_backend.Close(opened.Value()); + GTEST_SKIP() << "This user can bypass filesystem permission checks"; + } + EXPECT_EQ(opened.Error(), VFSError::PermissionDenied); +} +#endif + TEST_F(VFSDiskBackendTest, WriteThenReadBack) { cstring contents = "hello vfs world"; diff --git a/ZEngine/tests/VFS/vfs_meta_test.cpp b/ZEngine/tests/VFS/vfs_meta_test.cpp index 4565face..79398d54 100644 --- a/ZEngine/tests/VFS/vfs_meta_test.cpp +++ b/ZEngine/tests/VFS/vfs_meta_test.cpp @@ -1,9 +1,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -65,8 +65,9 @@ class MetaFileIOTest : public ::testing::Test TEST_F(MetaFileIOTest, MetaPathForAppendsMetaSuffix) { VFSPath asset = P("/project/textures/diffuse.png"); - VFSPath meta = MetaFileIO::MetaPathFor(asset); - EXPECT_STREQ(meta.CStr(), "/project/textures/diffuse.png.meta"); + auto meta = MetaFileIO::MetaPathFor(asset); + ASSERT_TRUE(meta.Succeeded()); + EXPECT_STREQ(meta.Value().CStr(), "/project/textures/diffuse.png.meta"); } // Test 2 — Read returns Fail when file does not exist @@ -85,6 +86,29 @@ TEST_F(MetaFileIOTest, MalformedJSONReturnsError) EXPECT_EQ(result.Error(), VFSError::Corrupted); } +TEST_F(MetaFileIOTest, InvalidIdentitiesAreCorruptAndGetOrCreateRepairsThem) +{ + for (const auto* document : {"{}", "{\"uuid\":42}", "{\"uuid\":\"invalid\"}", "{\"uuid\":\"00000000-0000-0000-0000-000000000000\"}"}) + { + SCOPED_TRACE(document); + WriteRaw("/x.glb.meta", document); + EXPECT_EQ(MetaFileIO::Read(m_ctx, P("/x.glb")).Error(), VFSError::Corrupted); + auto repaired = MetaFileIO::GetOrCreate(m_ctx, P("/x.glb"), "Test", 0); + ASSERT_TRUE(repaired.Succeeded()); + EXPECT_FALSE(repaired.Value().AssetUUID.is_nil()); + EXPECT_EQ(repaired.Value().Status, ImportStatus::New); + auto persisted = MetaFileIO::Read(m_ctx, P("/x.glb")); + ASSERT_TRUE(persisted.Succeeded()); + EXPECT_EQ(persisted.Value().AssetUUID, repaired.Value().AssetUUID); + } +} + +TEST_F(MetaFileIOTest, WritingNilIdentityIsRejected) +{ + EXPECT_EQ(MetaFileIO::Write(m_ctx, P("/x.glb"), {}).Error(), VFSError::Corrupted); + EXPECT_FALSE(FileExists("/x.glb.meta")); +} + // Test 4 — Round-trip: Write then Read returns identical data TEST_F(MetaFileIOTest, RoundTrip) { @@ -120,7 +144,7 @@ TEST_F(MetaFileIOTest, StatusNotSerialised) d.Status = ImportStatus::Stale; ASSERT_TRUE(MetaFileIO::Write(m_ctx, P("/x.glb"), d).Succeeded()); - auto open = m_ctx.Open(MetaFileIO::MetaPathFor(P("/x.glb")), VFSOpenFlags::Read); + auto open = m_ctx.Open(MetaFileIO::MetaPathFor(P("/x.glb")).Value(), VFSOpenFlags::Read); ASSERT_TRUE(open.Succeeded()); auto* file = open.Value(); uint8_t raw[4096] = {}; @@ -178,7 +202,7 @@ TEST_F(MetaFileIOTest, GetOrCreateChangedHashReturnsStale) TEST_F(MetaFileIOTest, AtomicWritePublishesDestinationWithoutTemporaryFile) { constexpr char content[] = "texture-bytes"; - ASSERT_TRUE(ZEngine::Importers::AssetCodec::WriteFileAtomically(m_ctx, P("/generated/diffuse.png"), Bytes(content, sizeof(content) - 1)).Succeeded()); + ASSERT_TRUE(WriteFileAtomically(m_ctx, P("/generated/diffuse.png"), Bytes(content, sizeof(content) - 1)).Succeeded()); EXPECT_TRUE(FileExists("/generated/diffuse.png")); EXPECT_FALSE(FileExists("/generated/diffuse.png.tmp")); @@ -240,3 +264,25 @@ TEST_F(MetaFileIOTest, SettingsCountCappedAtMax) ASSERT_TRUE(result.Succeeded()); EXPECT_EQ(result.Value().SettingsCount, META_MAX_SETTINGS); } + +TEST_F(MetaFileIOTest, SidecarPathBoundaryNeverOverwritesTheAsset) +{ + const std::string longest = "/" + std::string(MAX_FILE_PATH_COUNT - 2 - std::string_view(".meta").size(), 'x'); + auto sidecar = MetaFileIO::MetaPathFor(P(longest.c_str())); + ASSERT_TRUE(sidecar.Succeeded()); + EXPECT_EQ(sidecar.Value().Length(), MAX_FILE_PATH_COUNT - 1); + const auto invalid = longest + "x"; + WriteRaw(invalid.c_str(), "original asset bytes"); + MetaFileData data; + EXPECT_EQ(MetaFileIO::Write(m_ctx, P(invalid.c_str()), data).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::Read(m_ctx, P(invalid.c_str())).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::GetOrCreate(m_ctx, P(invalid.c_str()), "Test", 0).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::MetaPathFor(VFSPath{}).Error(), VFSError::InvalidPath); + EXPECT_EQ(MetaFileIO::MetaPathFor(VFSPath::Root()).Error(), VFSError::InvalidPath); + auto file = m_ctx.Open(P(invalid.c_str()), VFSOpenFlags::Read); + ASSERT_TRUE(file.Succeeded()); + uint8_t bytes[32] = {}; + ASSERT_TRUE(file.Value()->ReadAll({bytes, sizeof(bytes) - 1}).Succeeded()); + m_ctx.Close(file.Value()); + EXPECT_STREQ(reinterpret_cast(bytes), "original asset bytes"); +} From 359bc14eac8a07b5a26bb37e16698447a56f3819 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Mon, 5 Oct 2026 23:07:00 +0900 Subject: [PATCH 26/27] fix(vfs): bound material metadata completion --- Tetragrama/Panels/AssetImporterPanel.cpp | 16 +- ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp | 75 ++-- ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h | 9 +- ZEngine/ZEngine/Core/VFS/VFSError.h | 25 +- ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp | 164 ++++++++ ZEngine/ZEngine/Core/VFS/VFSFileIO.h | 10 +- ZEngine/ZEngine/Core/VFS/VFSScanner.cpp | 143 ++----- ZEngine/ZEngine/Core/VFS/VFSScanner.h | 3 + ZEngine/tests/VFS/vfs_asset_identity_test.cpp | 398 ++++++++++++++++++ ZEngine/tests/VFS/vfs_meta_test.cpp | 48 +++ 10 files changed, 739 insertions(+), 152 deletions(-) create mode 100644 ZEngine/tests/VFS/vfs_asset_identity_test.cpp diff --git a/Tetragrama/Panels/AssetImporterPanel.cpp b/Tetragrama/Panels/AssetImporterPanel.cpp index 11349a51..ccb4ec60 100644 --- a/Tetragrama/Panels/AssetImporterPanel.cpp +++ b/Tetragrama/Panels/AssetImporterPanel.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -839,13 +840,12 @@ namespace Tetragrama::Panels if (!rel.Succeeded()) continue; - char native_mat_path[MAX_FILE_PATH_COUNT] = {}; - rel.Value().ResolveNative(ws, native_mat_path, sizeof(native_mat_path)); - - ZEngine::Importers::AssetMaterial material{}; - ZEngine::Importers::AssetCodec::DeserializeMaterialAssetFile(&self->m_local_arena, native_mat_path, material); - if (material.MaterialUUID.is_nil()) - continue; + auto material_uuid = ReadEmbeddedAssetUUID(*vfs, rel.Value()); + if (material_uuid.Failed()) + { + CompleteImportErrorOnMainThread(self, fmt::format("Failed to read material identity for '{}' (VFS error {})", mat_path, static_cast(material_uuid.Error()))); + return; + } auto meta_result = ZEngine::Core::VFS::MetaFileIO::Read(*vfs, rel.Value()); if (meta_result.Failed() && meta_result.Error() != ZEngine::Core::VFS::VFSError::NotFound && meta_result.Error() != ZEngine::Core::VFS::VFSError::Corrupted) @@ -854,7 +854,7 @@ namespace Tetragrama::Panels return; } ZEngine::Core::VFS::MetaFileData meta = meta_result.Succeeded() ? meta_result.Value() : ZEngine::Core::VFS::MetaFileData{}; - meta.AssetUUID = material.MaterialUUID; + meta.AssetUUID = material_uuid.Value(); secure_strncpy(meta.SourcePath, sizeof(meta.SourcePath), self->m_path_buf, sizeof(meta.SourcePath) - 1); secure_strncpy(meta.ArtifactPath, sizeof(meta.ArtifactPath), mat_path, sizeof(meta.ArtifactPath) - 1); secure_strncpy(meta.ImporterName, sizeof(meta.ImporterName), "GltfImporter/AssimpImporter", sizeof(meta.ImporterName) - 1); diff --git a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp index c340e0e0..d221d3df 100644 --- a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp +++ b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.cpp @@ -4,18 +4,17 @@ #include #include #include +#include #include #include #include +#include namespace ZEngine::Core::VFS { namespace { - // .meta files are small JSON — 8 KB is a generous upper bound. - static constexpr size_t k_meta_read_cap = 8192; - - static int64_t NowNs() + static int64_t NowNs() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } @@ -56,24 +55,22 @@ namespace ZEngine::Core::VFS } uint64_t size = size_result.Value(); - if (size >= k_meta_read_cap) + if (size > MaxFileSize) { ctx.Close(file); - return VFSResult::Fail(VFSError::Corrupted); + return VFSResult::Fail(VFSError::SizeLimitExceeded); } - uint8_t buf[k_meta_read_cap]; - auto read_result = file->ReadAll({buf, (size_t) size}); + std::vector buf(static_cast(size)); + auto read_result = file->ReadAll({buf.data(), buf.size()}); ctx.Close(file); if (read_result.Failed()) return VFSResult::Fail(read_result.Error()); if (read_result.Value() != size) - return VFSResult::Fail(VFSError::Corrupted); - - buf[size] = '\0'; + return VFSResult::Fail(VFSError::IOError); - auto j = nlohmann::json::parse(reinterpret_cast(buf), nullptr, /*allow_exceptions=*/false); + auto j = nlohmann::json::parse(buf.begin(), buf.end(), nullptr, /*allow_exceptions=*/false); if (j.is_discarded() || !j.is_object()) return VFSResult::Fail(VFSError::Corrupted); @@ -144,10 +141,12 @@ namespace ZEngine::Core::VFS } const std::string serialized = j.dump(4); + if (serialized.size() > MaxFileSize) + return VFSResult::Fail(VFSError::SizeLimitExceeded); return WriteFileAtomically(ctx, meta_path.Value(), {reinterpret_cast(serialized.data()), serialized.size()}); } - VFSResult MetaFileIO::GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash) + VFSResult MetaFileIO::GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash, bool use_embedded_identity) { // Scanner and importer callers must not mint different identities for // the same absent sidecar. Writes remain atomic; serialize this RMW. @@ -177,8 +176,17 @@ namespace ZEngine::Core::VFS if (read_result.Error() != VFSError::NotFound && read_result.Error() != VFSError::Corrupted) return VFSResult::Fail(read_result.Error()); - // No .meta or corrupt .meta — generate a fresh UUID. + // Resolve and persist identity under the same lock; scanners must not seed + // sidecars separately or mint random identities for unreadable cooked assets. MetaFileData fresh{}; + if (use_embedded_identity && (asset_path.Extension().Equals(".zemesh") || asset_path.Extension().Equals(".zematerial"))) + { + auto embedded = ReadEmbeddedAssetUUID(ctx, asset_path); + if (embedded.Failed()) + return VFSResult::Fail(embedded.Error()); + fresh.AssetUUID = embedded.Value(); + } + else { std::random_device rd; std::mt19937 generator(rd()); @@ -201,26 +209,37 @@ namespace ZEngine::Core::VFS if (open_result.Failed()) return VFSResult::Fail(open_result.Error()); - IVFSFile* file = open_result.Value(); + IVFSFile* file = open_result.Value(); + + auto size_result = file->Size(); + if (size_result.Failed()) + { + ctx.Close(file); + return VFSResult::Fail(size_result.Error()); + } // Stream through the file in 4 KB chunks, chaining rapidhash via the seed parameter. - uint8_t chunk[4096]; - uint64_t hash = 0; - uint64_t offset = 0; + uint8_t chunk[4096]; + uint64_t hash = 0; + uint64_t offset = 0; - while (true) + while (offset < size_result.Value()) { - auto read_result = file->Read({chunk, sizeof(chunk)}, offset); - if (read_result.Failed()) + const size_t count = static_cast(std::min(sizeof(chunk), size_result.Value() - offset)); + size_t filled = 0; + while (filled < count) { - ctx.Close(file); - return VFSResult::Fail(read_result.Error()); + auto read = file->Read({chunk + filled, count - filled}, offset + filled); + if (read.Failed() || read.Value() == 0 || read.Value() > count - filled) + { + ctx.Close(file); + return VFSResult::Fail(read.Failed() ? read.Error() : VFSError::IOError); + } + filled += read.Value(); } - size_t n = read_result.Value(); - if (n == 0) - break; - hash = rapidhash_withSeed(chunk, n, hash); - offset += (uint64_t) n; + // Keep chunk boundaries stable even when the backend returns partial reads. + hash = rapidhash_withSeed(chunk, count, hash); + offset += count; } ctx.Close(file); return VFSResult::Ok(hash); diff --git a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h index 8b6e9de1..02b87476 100644 --- a/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h +++ b/ZEngine/ZEngine/Core/VFS/Meta/MetaFileIO.h @@ -8,6 +8,10 @@ namespace ZEngine::Core::VFS { struct MetaFileIO { + // Covers all fixed-capacity fields, including worst-case JSON escaping. + static constexpr size_t MaxFileSize = 64 * 1024; + static_assert(MaxFileSize >= 6 * sizeof(MetaFileData) + 4096, "Metadata capacity must cover escaped fields and JSON formatting"); + // Returns the sidecar path for an asset: "/project/mesh.glb" -> "/project/mesh.glb.meta" static VFSResult MetaPathFor(const VFSPath& asset_path); @@ -22,8 +26,9 @@ namespace ZEngine::Core::VFS // - No .meta or corrupt .meta -> generate UUID, write, return New // - .meta exists, hash matches -> return UpToDate (no write) // - .meta exists, hash differs -> update hash + timestamp, write, return Stale - // The UUID in an existing .meta is never replaced. - static VFSResult GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash); + // Valid existing identities are preserved; missing/corrupt cooked sidecars can use the embedded identity. + // Access and size-limit errors are returned without changing the sidecar. + static VFSResult GetOrCreate(IVFSContext& ctx, const VFSPath& asset_path, const char* importer_name, uint64_t current_hash, bool use_embedded_identity = false); // Computes the rapidhash-64 digest of the asset file content. static VFSResult ComputeHash(IVFSContext& ctx, const VFSPath& asset_path); diff --git a/ZEngine/ZEngine/Core/VFS/VFSError.h b/ZEngine/ZEngine/Core/VFS/VFSError.h index 4083939d..386e8611 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSError.h +++ b/ZEngine/ZEngine/Core/VFS/VFSError.h @@ -6,18 +6,19 @@ namespace ZEngine::Core::VFS { enum class VFSError : uint32_t { - OK = 0, - NotFound = 1, - PermissionDenied = 2, - AlreadyExists = 3, - NotADirectory = 4, - NotAFile = 5, - InvalidPath = 6, - Unsupported = 7, - IOError = 8, - OutOfMemory = 9, - Corrupted = 10, - Cancelled = 11, + OK = 0, + NotFound = 1, + PermissionDenied = 2, + AlreadyExists = 3, + NotADirectory = 4, + NotAFile = 5, + InvalidPath = 6, + Unsupported = 7, + IOError = 8, + OutOfMemory = 9, + Corrupted = 10, + Cancelled = 11, + SizeLimitExceeded = 12, }; template diff --git a/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp b/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp index b2d53108..d81afe6b 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSFileIO.cpp @@ -1,9 +1,173 @@ #include +#include #include +#include +#include +#include #include namespace ZEngine::Core::VFS { + namespace + { + // Retain only a 4 KB input chunk and the UUID, even for large materials. + class VFSReadBuffer : public std::streambuf + { + public: + VFSReadBuffer(IVFSFile* file, uint64_t size) : m_file(file), m_size(size) {} + VFSError Error = VFSError::OK; + + protected: + int_type underflow() override + { + if (m_offset == m_size) + return traits_type::eof(); + const size_t count = static_cast(std::min(sizeof(m_buffer), m_size - m_offset)); + auto read = m_file->Read({reinterpret_cast(m_buffer), count}, m_offset); + if (read.Failed() || read.Value() == 0 || read.Value() > count) + { + Error = read.Failed() ? read.Error() : VFSError::IOError; + return traits_type::eof(); + } + m_offset += read.Value(); + setg(m_buffer, m_buffer, m_buffer + read.Value()); + return traits_type::to_int_type(*gptr()); + } + + private: + IVFSFile* m_file; + uint64_t m_size; + uint64_t m_offset = 0; + char m_buffer[4096]; + }; + + struct MaterialUUIDReader : nlohmann::json_sax + { + uuids::uuid UUID{}; + int Depth = 0; + bool RootObject = false; + bool UUIDKey = false; + bool null() override + { + UUIDKey = false; + return true; + } + bool boolean(bool) override + { + return null(); + } + bool number_integer(number_integer_t) override + { + return null(); + } + bool number_unsigned(number_unsigned_t) override + { + return null(); + } + bool number_float(number_float_t, const string_t&) override + { + return null(); + } + bool string(string_t& value) override + { + if (UUIDKey) + { + auto parsed = uuids::uuid::from_string(value); + UUID = parsed.has_value() ? *parsed : uuids::uuid{}; + } + UUIDKey = false; + return true; + } + bool binary(binary_t&) override + { + return null(); + } + bool start_object(std::size_t) override + { + if (Depth == 0) + RootObject = true; + ++Depth; + UUIDKey = false; + return true; + } + bool key(string_t& value) override + { + UUIDKey = Depth == 1 && value == "uuid"; + if (UUIDKey) + UUID = {}; + return true; + } + bool end_object() override + { + --Depth; + UUIDKey = false; + return true; + } + bool start_array(std::size_t) override + { + ++Depth; + UUIDKey = false; + return true; + } + bool end_array() override + { + --Depth; + UUIDKey = false; + return true; + } + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception&) override + { + return false; + } + }; + } // namespace + + VFSResult ReadEmbeddedAssetUUID(IVFSContext& ctx, const VFSPath& path) + { + const bool mesh = path.Extension().Equals(".zemesh"); + if (!mesh && !path.Extension().Equals(".zematerial")) + return VFSResult::Fail(VFSError::Unsupported); + auto open = ctx.Open(path, VFSOpenFlags::Read); + if (open.Failed()) + return VFSResult::Fail(open.Error()); + auto* file = open.Value(); + auto result = [&]() -> VFSResult { + if (mesh) + { + uint8_t header[24]; + auto read = file->ReadAll({header, sizeof(header)}); + if (read.Failed()) + return VFSResult::Fail(read.Error()); + if (read.Value() != sizeof(header)) + return VFSResult::Fail(VFSError::Corrupted); + uint32_t magic, version; + uuids::uuid id; + std::memcpy(&magic, header, 4); + std::memcpy(&version, header + 4, 4); + std::memcpy(&id, header + 8, 16); + if (magic != ZEMESH_MAGIC || version != ASSET_FILE_VERSION || id.is_nil()) + return VFSResult::Fail(VFSError::Corrupted); + return VFSResult::Ok(id); + } + auto size = file->Size(); + if (size.Failed()) + return VFSResult::Fail(size.Error()); + if (size.Value() > MATERIAL_IDENTITY_MAX_FILE_SIZE) + return VFSResult::Fail(VFSError::SizeLimitExceeded); + VFSReadBuffer buffer(file, size.Value()); + std::istream input(&buffer); + MaterialUUIDReader reader; + const bool valid = nlohmann::json::sax_parse(input, &reader); + if (buffer.Error != VFSError::OK) + return VFSResult::Fail(buffer.Error); + if (!valid || !reader.RootObject || reader.UUID.is_nil()) + return VFSResult::Fail(VFSError::Corrupted); + return VFSResult::Ok(reader.UUID); + }(); + ctx.Close(file); + return result; + } + VFSResult WriteFileAtomically(IVFSContext& ctx, const VFSPath& path, Containers::ArrayView data) { if (!path.IsValid() || path.IsRoot()) diff --git a/ZEngine/ZEngine/Core/VFS/VFSFileIO.h b/ZEngine/ZEngine/Core/VFS/VFSFileIO.h index 2492d6cb..f95aa83b 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSFileIO.h +++ b/ZEngine/ZEngine/Core/VFS/VFSFileIO.h @@ -1,8 +1,16 @@ #pragma once #include +#include namespace ZEngine::Core::VFS { + // Material identity extraction uses a JSON parser that may temporarily retain one + // string token. Keep that bounded without imposing a limit on mesh assets. + inline constexpr uint64_t MATERIAL_IDENTITY_MAX_FILE_SIZE = 1024ULL * 1024ULL; + + // Read only the embedded identity; material JSON is streamed without an arena or DOM. + [[nodiscard]] VFSResult ReadEmbeddedAssetUUID(IVFSContext& ctx, const VFSPath& path); + // Publish using a unique temporary sibling on the same mount/filesystem. - [[nodiscard]] VFSResult WriteFileAtomically(IVFSContext& ctx, const VFSPath& path, Containers::ArrayView data); + [[nodiscard]] VFSResult WriteFileAtomically(IVFSContext& ctx, const VFSPath& path, Containers::ArrayView data); } // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp b/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp index f9adf35a..7d4ea083 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSScanner.cpp @@ -4,10 +4,6 @@ #include #include #include -#include -#include -#include -#include #include namespace ZEngine::Core::VFS @@ -23,69 +19,6 @@ namespace ZEngine::Core::VFS return true; return false; } - - // .zemesh/.zematerial embed their own UUID at cook time. Peeking it here - // lets the caller pre-seed a .meta instead of GetOrCreate minting an - // unrelated random one (#755). Duplicated rather than depending on - // Importers from Core::VFS — ZEMESH_MAGIC/ASSET_FILE_VERSION are already - // shared, low-level constants (ZEngineDef.h). - static std::optional PeekEmbeddedUUID(IVFSContext& ctx, const VFSPath& path, Managers::AssetType type) - { - if (type != Managers::AssetType::MESH && type != Managers::AssetType::MATERIAL) - return std::nullopt; - - auto open_result = ctx.Open(path, VFSOpenFlags::Read); - if (open_result.Failed()) - return std::nullopt; - IVFSFile* file = open_result.Value(); - - std::optional result; - - if (type == Managers::AssetType::MESH) - { - // Layout matches AssetCodec::AssetMeshFileHeader: uint32 magic, uint32 version, 16-byte uuid. - uint8_t buf[24]; - auto read_result = file->Read({buf, sizeof(buf)}, 0); - if (read_result.Succeeded() && read_result.Value() == sizeof(buf)) - { - uint32_t magic = 0, version = 0; - std::memcpy(&magic, buf, sizeof(magic)); - std::memcpy(&version, buf + sizeof(magic), sizeof(version)); - if (magic == ZEMESH_MAGIC && version == ASSET_FILE_VERSION) - { - uuids::uuid id; - std::memcpy(&id, buf + sizeof(magic) + sizeof(version), sizeof(id)); - if (!id.is_nil()) - result = id; - } - } - } - else // MATERIAL — JSON, top-level "uuid" field (matches AssetCodec::SerializeMaterialAssetFile) - { - static constexpr size_t kReadCap = 16384; - auto size_result = file->Size(); - if (size_result.Succeeded() && size_result.Value() < kReadCap) - { - uint8_t buf[kReadCap]; - size_t size = (size_t) size_result.Value(); - auto read_result = file->ReadAll({buf, size}); - if (read_result.Succeeded()) - { - buf[size] = '\0'; - auto j = nlohmann::json::parse(reinterpret_cast(buf), nullptr, /*allow_exceptions=*/false); - if (!j.is_discarded() && j.contains("uuid") && j["uuid"].is_string()) - { - auto parsed = uuids::uuid::from_string(j["uuid"].get()); - if (parsed.has_value() && !parsed.value().is_nil()) - result = parsed.value(); - } - } - } - } - - ctx.Close(file); - return result; - } } // namespace void VFSScanner::Initialize(Core::Memory::ArenaAllocator* page_source) @@ -189,6 +122,7 @@ namespace ZEngine::Core::VFS m_metas_created.value.store(0, std::memory_order_relaxed); m_metas_updated.value.store(0, std::memory_order_relaxed); m_metas_up_to_date.value.store(0, std::memory_order_relaxed); + m_errors.value.store(0, std::memory_order_relaxed); m_pending_tasks.value.store(0, std::memory_order_relaxed); m_is_scanning.value.store(true, std::memory_order_release); m_scan_start = std::chrono::steady_clock::now(); @@ -216,7 +150,10 @@ namespace ZEngine::Core::VFS m_dir_semaphore.release(); if (result.Failed()) + { + ReportError(dir, "list directory", result.Error()); return; + } Containers::Array& entries = result.Value(); @@ -238,45 +175,42 @@ namespace ZEngine::Core::VFS if (IsAssetExtension(entries[i].Path)) { - ZEngine::Managers::AssetType type = ZEngine::Core::VFS::AssetRegistry::InferTypeFromExtension(entries[i].Path); + ZEngine::Managers::AssetType type = ZEngine::Core::VFS::AssetRegistry::InferTypeFromExtension(entries[i].Path); + + // Never write metadata using a fabricated hash after an I/O failure. + auto hash_result = MetaFileIO::ComputeHash(*ctx.Context, entries[i].Path); + if (hash_result.Failed()) + { + ReportError(entries[i].Path, "hash asset", hash_result.Error()); + continue; + } - // Pre-seed .meta from the embedded UUID before GetOrCreate mints an - // unrelated random one (#755). - if (MetaFileIO::Read(*ctx.Context, entries[i].Path).Failed()) + // GetOrCreate reads and, if necessary, seeds the embedded identity + // under its lock, avoiding redundant metadata reads and unchecked writes. + auto meta = MetaFileIO::GetOrCreate(*ctx.Context, entries[i].Path, "VFSScanner", hash_result.Value(), /*use_embedded_identity=*/true); + if (meta.Failed()) + { + ReportError(entries[i].Path, "resolve metadata", meta.Error()); + continue; + } + switch (meta.Value().Status) { - auto embedded = PeekEmbeddedUUID(*ctx.Context, entries[i].Path, type); - if (embedded.has_value()) - { - MetaFileData seed = {}; - seed.AssetUUID = *embedded; - Helpers::secure_strncpy(seed.ImporterName, sizeof(seed.ImporterName), "VFSScanner", sizeof(seed.ImporterName) - 1); - MetaFileIO::Write(*ctx.Context, entries[i].Path, seed); - } + case ImportStatus::New: + m_metas_created.value.fetch_add(1, std::memory_order_relaxed); + break; + case ImportStatus::Stale: + m_metas_updated.value.fetch_add(1, std::memory_order_relaxed); + break; + case ImportStatus::UpToDate: + m_metas_up_to_date.value.fetch_add(1, std::memory_order_relaxed); + break; + default: + break; } - auto hash_result = MetaFileIO::ComputeHash(*ctx.Context, entries[i].Path); - auto meta = MetaFileIO::GetOrCreate(*ctx.Context, entries[i].Path, "VFSScanner", hash_result.Succeeded() ? hash_result.Value() : 0); - if (meta.Succeeded()) + if (m_registry != nullptr) { - switch (meta.Value().Status) - { - case ImportStatus::New: - m_metas_created.value.fetch_add(1, std::memory_order_relaxed); - break; - case ImportStatus::Stale: - m_metas_updated.value.fetch_add(1, std::memory_order_relaxed); - break; - case ImportStatus::UpToDate: - m_metas_up_to_date.value.fetch_add(1, std::memory_order_relaxed); - break; - default: - break; - } - - if (m_registry != nullptr) - { - m_registry->OnScanFileDiscovered(*ctx.Context, entries[i].Path, type); - } + m_registry->OnScanFileDiscovered(*ctx.Context, entries[i].Path, type); } } } @@ -285,6 +219,12 @@ namespace ZEngine::Core::VFS ctx.Cache->SetListing(dir, std::move(entries)); } + void VFSScanner::ReportError(const VFSPath& path, const char* operation, VFSError error) + { + m_errors.value.fetch_add(1, std::memory_order_relaxed); + ZENGINE_LOG_VFS_WARN("VFS scan failed to {} for '{}' (VFS error {})", operation, path.CStr(), static_cast(error)) + } + void VFSScanner::OnTaskComplete(bool cancelled) { const int32_t remaining = m_pending_tasks.value.fetch_sub(1, std::memory_order_acq_rel) - 1; @@ -302,6 +242,7 @@ namespace ZEngine::Core::VFS stats.MetasCreated = m_metas_created.value.load(std::memory_order_relaxed); stats.MetasUpdated = m_metas_updated.value.load(std::memory_order_relaxed); stats.MetasUpToDate = m_metas_up_to_date.value.load(std::memory_order_relaxed); + stats.Errors = m_errors.value.load(std::memory_order_relaxed); m_complete_callback(m_complete_callback_ctx, stats); } } diff --git a/ZEngine/ZEngine/Core/VFS/VFSScanner.h b/ZEngine/ZEngine/Core/VFS/VFSScanner.h index d5e9f6e5..10e8d383 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSScanner.h +++ b/ZEngine/ZEngine/Core/VFS/VFSScanner.h @@ -29,6 +29,7 @@ namespace ZEngine::Core::VFS uint64_t MetasCreated = 0; // .meta generated for the first time uint64_t MetasUpdated = 0; // .meta existed but source SHA changed uint64_t MetasUpToDate = 0; // .meta existed and SHA matched + uint64_t Errors = 0; // failed directory/asset operations; affected sidecars are preserved }; struct VFSScanner @@ -74,6 +75,7 @@ namespace ZEngine::Core::VFS bool TryAcquireTask(uint32_t& out_slot); void ReleaseTask(uint32_t slot); void OnTaskComplete(bool cancelled); + void ReportError(const VFSPath& path, const char* operation, VFSError error); static void RunScanTask(void* context); int AcquireSlot(); @@ -90,6 +92,7 @@ namespace ZEngine::Core::VFS PaddedAtomic m_metas_created{}; PaddedAtomic m_metas_updated{}; PaddedAtomic m_metas_up_to_date{}; + PaddedAtomic m_errors{}; std::chrono::steady_clock::time_point m_scan_start{}; std::counting_semaphore m_dir_semaphore{MaxConcurrentDirLists}; diff --git a/ZEngine/tests/VFS/vfs_asset_identity_test.cpp b/ZEngine/tests/VFS/vfs_asset_identity_test.cpp new file mode 100644 index 00000000..06c3ed9f --- /dev/null +++ b/ZEngine/tests/VFS/vfs_asset_identity_test.cpp @@ -0,0 +1,398 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ZEngine::Core::Memory; +using namespace ZEngine::Core::VFS; + +namespace +{ + struct FaultFile : IVFSFile + { + IVFSFile* File = nullptr; + uint64_t ReadLimit = UINT64_MAX; + size_t MaxChunk = SIZE_MAX; + bool ReadIOError = false; + bool SizeIOError = false; + VFSResult Read(ZEngine::Core::Containers::ArrayView buffer, uint64_t offset) override + { + if (offset >= ReadLimit) + return ReadIOError ? VFSResult::Fail(VFSError::IOError) : VFSResult::Ok(0); + const size_t count = std::min({buffer.size(), MaxChunk, static_cast(ReadLimit - offset)}); + return File->Read({buffer.data(), count}, offset); + } + VFSResult Write(ZEngine::Core::Containers::ArrayView buffer, uint64_t offset) override + { + return File->Write(buffer, offset); + } + VFSResult Size() const override + { + return SizeIOError ? VFSResult::Fail(VFSError::IOError) : File->Size(); + } + VFSResult Stat() const override + { + return File->Stat(); + } + VFSResult Flush() override + { + return File->Flush(); + } + const VFSPath& Path() const override + { + return File->Path(); + } + VFSResult Close() override + { + return File->Close(); + } + }; + + // Faults apply to scanner operations; direct backend access remains available + // to verify that failed scans did not alter the original sidecar. + struct FaultContext : VFSContext + { + bool DenyMetaOnce = false; + bool FailAssetRead = false; + bool FailWrite = false; + bool FailRename = false; + bool FailList = false; + int Writes = 0; + FaultFile AssetFile; + bool WrapAssetReads = false; + bool WrapMetadataReads = false; + VFSResult Open(const VFSPath& path, VFSOpenFlags flags) override + { + if (HasFlag(flags, VFSOpenFlags::Read)) + { + if (path.Extension().Equals(".meta") && DenyMetaOnce) + { + DenyMetaOnce = false; + return VFSResult::Fail(VFSError::PermissionDenied); + } + if (!path.Extension().Equals(".meta") && FailAssetRead) + return VFSResult::Fail(VFSError::IOError); + } + if (HasFlag(flags, VFSOpenFlags::Write)) + { + ++Writes; + if (FailWrite) + return VFSResult::Fail(VFSError::PermissionDenied); + } + auto result = VFSContext::Open(path, flags); + if (result.Succeeded() && HasFlag(flags, VFSOpenFlags::Read) && ((WrapAssetReads && !path.Extension().Equals(".meta")) || (WrapMetadataReads && path.Extension().Equals(".meta")))) + { + AssetFile.File = result.Value(); + return VFSResult::Ok(&AssetFile); + } + return result; + } + void Close(IVFSFile* file) override + { + VFSContext::Close(file == &AssetFile ? AssetFile.File : file); + } + VFSResult Rename(const VFSPath& from, const VFSPath& to) override + { + return FailRename ? VFSResult::Fail(VFSError::IOError) : VFSContext::Rename(from, to); + } + VFSResult> List(const VFSPath& path, ArenaAllocator* arena) override + { + if (FailList) + return VFSResult>::Fail(VFSError::IOError); + return VFSContext::List(path, arena); + } + }; +} // namespace + +class VFSAssetIdentityTest : public ::testing::Test +{ +protected: + MemoryManager Memory; + VFSMemoryBackend Backend; + FaultContext Context; + const uuids::uuid ID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + void SetUp() override + { + Memory.Initialize(ZGiga(1), {}); + Backend.Initialize(&Memory.MainArena); + Context.Initialize(&Memory.MainArena, 4); + ASSERT_TRUE(Context.Mount(&Backend, VFSPath::Root(), 0).Succeeded()); + } + void TearDown() override + { + Context.Shutdown(); + } + VFSPath P(const char* path) + { + return VFSPath::Parse(path).Value(); + } + void Put(const char* path, const std::string& data) + { + ASSERT_TRUE(Backend.WriteFile(P(path), {reinterpret_cast(data.data()), data.size()}).Succeeded()); + } + std::string Material() + { + return "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"textures\":{\"albedo\":{\"path\":\"" + std::string(200, 'p') + "\"}}}"; + } + void Scan(VFSScanner& scanner, VFSDirectoryCache& cache, ScanStats& stats) + { + std::atomic complete{false}; + struct Completion + { + ScanStats* Stats; + std::atomic* Complete; + } completion{&stats, &complete}; + scanner.SetOnScanComplete(&completion, [](void* ctx, ScanStats value) { + auto* done = static_cast(ctx); + *done->Stats = value; + done->Complete->store(true); + }); + scanner.Scan(&Context, VFSPath::Root(), &cache); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!complete.load() && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + // Keep callback context alive until all tasks finish, even on a timeout. + if (!complete.load()) + { + scanner.Cancel(); + while (scanner.IsScanning()) + std::this_thread::yield(); + } + ASSERT_TRUE(complete.load()); + } +}; + +TEST_F(VFSAssetIdentityTest, Reads1000MaterialIdentitiesWithoutAnArena) +{ + Put("/material.zematerial", Material()); + for (int i = 0; i < 1000; ++i) + { + auto result = ReadEmbeddedAssetUUID(Context, P("/material.zematerial")); + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Value(), ID); + } +} + +TEST_F(VFSAssetIdentityTest, StreamsLargeMaterialAndUsesOnlyTopLevelIdentity) +{ + const std::string json = "{\"textures\":{\"albedo\":{\"uuid\":\"550e8400-e29b-41d4-a716-446655440001\",\"path\":\"" + std::string(20000, 'p') + "\"}},\"uuid\":\"" + uuids::to_string(ID) + "\"}"; + Put("/large.zematerial", json); + auto result = ReadEmbeddedAssetUUID(Context, P("/large.zematerial")); + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Value(), ID); +} + +TEST_F(VFSAssetIdentityTest, OversizedMaterialIdentityIsRejectedWithoutReplacingCorruptMetadata) +{ + Put("/large.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(MATERIAL_IDENTITY_MAX_FILE_SIZE, 'n') + "\"}"); + Put("/large.zematerial.meta", "{corrupt metadata"); + + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/large.zematerial")).Error(), VFSError::SizeLimitExceeded); + + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + Context.Writes = 0; + ScanStats stats; + Scan(scanner, cache, stats); + + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(Context.Writes, 0); + auto sidecar = Context.Open(P("/large.zematerial.meta"), VFSOpenFlags::Read); + ASSERT_TRUE(sidecar.Succeeded()); + std::array contents{}; + auto read = sidecar.Value()->ReadAll({contents.data(), contents.size()}); + Context.Close(sidecar.Value()); + ASSERT_TRUE(read.Succeeded()); + EXPECT_STREQ(reinterpret_cast(contents.data()), "{corrupt metadata"); +} + +TEST_F(VFSAssetIdentityTest, RejectsInvalidMaterialsAndTruncatedMeshes) +{ + for (const char* json : {"{}", "[]", "{\"uuid\":null}", "{\"uuid\":\"invalid\"}", "{\"uuid\":\"00000000-0000-0000-0000-000000000000\"}", "{\"nested\":{\"uuid\":\"550e8400-e29b-41d4-a716-446655440000\"}}", "{\"uuid\":\"550e8400-e29b-41d4-a716-446655440000\"}garbage"}) + { + Put("/bad.zematerial", json); + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/bad.zematerial")).Error(), VFSError::Corrupted); + } + Put("/bad.zemesh", std::string(23, '\0')); + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/bad.zemesh")).Error(), VFSError::Corrupted); + Context.FailAssetRead = true; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/bad.zemesh")).Error(), VFSError::IOError); +} + +TEST_F(VFSAssetIdentityTest, ScannerPreservesSidecarOnMetadataHashAndPublicationFailures) +{ + Put("/material.zematerial", Material()); + MetaFileData data{}; + data.AssetUUID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440001").value(); + data.SourceHash = 123; + data.SettingsCount = 1; + std::strcpy(data.Settings[0].Key, "keep"); + std::strcpy(data.Settings[0].Value, "settings"); + ASSERT_TRUE(MetaFileIO::Write(Context, P("/material.zematerial"), data).Succeeded()); + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + for (int fault = 0; fault < 4; ++fault) + { + SCOPED_TRACE(fault); + Context.DenyMetaOnce = fault == 0; + Context.FailAssetRead = fault == 1; + Context.FailWrite = fault == 2; + Context.FailRename = fault == 3; + Context.Writes = 0; + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(stats.MetasCreated + stats.MetasUpdated + stats.MetasUpToDate, 0u); + if (fault < 2) + EXPECT_EQ(Context.Writes, 0); + Context.FailAssetRead = Context.FailWrite = Context.FailRename = false; + auto after = MetaFileIO::Read(Context, P("/material.zematerial")); + ASSERT_TRUE(after.Succeeded()); + EXPECT_EQ(after.Value().AssetUUID, data.AssetUUID); + EXPECT_EQ(after.Value().SourceHash, data.SourceHash); + EXPECT_EQ(after.Value().SettingsCount, 1u); + EXPECT_STREQ(after.Value().Settings[0].Value, "settings"); + } +} + +TEST_F(VFSAssetIdentityTest, ScannerReportsDirectoryErrorsAndDoesNotCreateIdentityForInvalidCookedAsset) +{ + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + Context.FailList = true; + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + Context.FailList = false; + Put("/bad.zematerial", "{}"); + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_FALSE(Context.Exists(P("/bad.zematerial.meta")).Value()); +} + +TEST_F(VFSAssetIdentityTest, ScannerSeedsLargeMaterialIdentityAndPreservesOversizedSidecar) +{ + Put("/large.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(20000, 'n') + "\"}"); + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 0u); + EXPECT_EQ(stats.MetasCreated, 1u); + auto meta = MetaFileIO::Read(Context, P("/large.zematerial")); + ASSERT_TRUE(meta.Succeeded()); + EXPECT_EQ(meta.Value().AssetUUID, ID); + Put("/large.zematerial.meta", std::string(MetaFileIO::MaxFileSize + 1, ' ')); + Context.Writes = 0; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(Context.Writes, 0); + EXPECT_EQ(MetaFileIO::Read(Context, P("/large.zematerial")).Error(), VFSError::SizeLimitExceeded); +} + +TEST_F(VFSAssetIdentityTest, IdentityReadsPropagateIOErrorsAndDetectPrematureEOF) +{ + Put("/material.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(5000, 'n') + "\"}"); + Context.WrapAssetReads = true; + Context.AssetFile.ReadLimit = 4096; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/material.zematerial")).Error(), VFSError::IOError); + Context.AssetFile.ReadIOError = true; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/material.zematerial")).Error(), VFSError::IOError); + Context.AssetFile.ReadLimit = UINT64_MAX; + Context.AssetFile.SizeIOError = true; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/material.zematerial")).Error(), VFSError::IOError); + Context.AssetFile.SizeIOError = false; + Context.AssetFile.MaxChunk = 7; + auto material = ReadEmbeddedAssetUUID(Context, P("/material.zematerial")); + ASSERT_TRUE(material.Succeeded()); + EXPECT_EQ(material.Value(), ID); + std::string mesh(24, '\0'); + uint32_t magic = ZEMESH_MAGIC, version = ASSET_FILE_VERSION; + std::memcpy(mesh.data(), &magic, 4); + std::memcpy(mesh.data() + 4, &version, 4); + std::memcpy(mesh.data() + 8, &ID, 16); + Put("/mesh.zemesh", mesh); + auto header = ReadEmbeddedAssetUUID(Context, P("/mesh.zemesh")); + ASSERT_TRUE(header.Succeeded()); + EXPECT_EQ(header.Value(), ID); + Context.AssetFile.ReadLimit = 12; + Context.AssetFile.ReadIOError = false; + EXPECT_EQ(ReadEmbeddedAssetUUID(Context, P("/mesh.zemesh")).Error(), VFSError::Corrupted); +} + +TEST_F(VFSAssetIdentityTest, ScannerDoesNotPersistHashFromPrematureEOFOrFailedSizeQuery) +{ + Put("/material.zematerial", Material()); + MetaFileData before{}; + before.AssetUUID = ID; + before.SourceHash = 123; + ASSERT_TRUE(MetaFileIO::Write(Context, P("/material.zematerial"), before).Succeeded()); + Context.Writes = 0; + Context.WrapAssetReads = true; + VFSDirectoryCache cache; + cache.Initialize(&Memory.MainArena); + VFSScanner scanner; + scanner.Initialize(&Memory.MainArena); + for (bool size_failure : {false, true}) + { + Context.AssetFile.ReadLimit = size_failure ? UINT64_MAX : 12; + Context.AssetFile.SizeIOError = size_failure; + ScanStats stats; + Scan(scanner, cache, stats); + EXPECT_EQ(stats.Errors, 1u); + EXPECT_EQ(Context.Writes, 0); + auto retained = MetaFileIO::Read(Context, P("/material.zematerial")); + ASSERT_TRUE(retained.Succeeded()); + EXPECT_EQ(retained.Value().SourceHash, before.SourceHash); + EXPECT_EQ(retained.Value().AssetUUID, before.AssetUUID); + } +} + +TEST_F(VFSAssetIdentityTest, ShortMetadataReadsAreIOErrorsAndNeverTriggerRepair) +{ + MetaFileData before{}; + before.AssetUUID = ID; + before.SourceHash = 123; + ASSERT_TRUE(MetaFileIO::Write(Context, P("/material.zematerial"), before).Succeeded()); + Context.Writes = 0; + Context.WrapMetadataReads = true; + Context.AssetFile.ReadLimit = 32; + EXPECT_EQ(MetaFileIO::Read(Context, P("/material.zematerial")).Error(), VFSError::IOError); + EXPECT_EQ(MetaFileIO::GetOrCreate(Context, P("/material.zematerial"), "Test", 456, true).Error(), VFSError::IOError); + EXPECT_EQ(Context.Writes, 0); + Context.WrapMetadataReads = false; + auto retained = MetaFileIO::Read(Context, P("/material.zematerial")); + ASSERT_TRUE(retained.Succeeded()); + EXPECT_EQ(retained.Value().AssetUUID, before.AssetUUID); + EXPECT_EQ(retained.Value().SourceHash, before.SourceHash); +} + +TEST_F(VFSAssetIdentityTest, HashIsStableAcrossPartialBackendReads) +{ + Put("/material.zematerial", "{\"uuid\":\"" + uuids::to_string(ID) + "\",\"name\":\"" + std::string(5000, 'n') + "\"}"); + auto expected = MetaFileIO::ComputeHash(Context, P("/material.zematerial")); + ASSERT_TRUE(expected.Succeeded()); + Context.WrapAssetReads = true; + Context.AssetFile.MaxChunk = 7; + auto fragmented = MetaFileIO::ComputeHash(Context, P("/material.zematerial")); + ASSERT_TRUE(fragmented.Succeeded()); + EXPECT_EQ(fragmented.Value(), expected.Value()); +} diff --git a/ZEngine/tests/VFS/vfs_meta_test.cpp b/ZEngine/tests/VFS/vfs_meta_test.cpp index 79398d54..28a680b3 100644 --- a/ZEngine/tests/VFS/vfs_meta_test.cpp +++ b/ZEngine/tests/VFS/vfs_meta_test.cpp @@ -286,3 +286,51 @@ TEST_F(MetaFileIOTest, SidecarPathBoundaryNeverOverwritesTheAsset) m_ctx.Close(file.Value()); EXPECT_STREQ(reinterpret_cast(bytes), "original asset bytes"); } + +TEST_F(MetaFileIOTest, MaximumEscapedFieldsRoundTrip) +{ + MetaFileData data{}; + data.AssetUUID = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + auto fill = [](auto& field) { std::memset(field, '\x01', sizeof(field) - 1); }; + fill(data.ImporterName); + fill(data.SourcePath); + fill(data.ArtifactPath); + data.SettingsCount = META_MAX_SETTINGS; + for (auto& setting : data.Settings) + { + fill(setting.Key); + fill(setting.Value); + } + ASSERT_TRUE(MetaFileIO::Write(m_ctx, P("/max.glb"), data).Succeeded()); + auto result = MetaFileIO::Read(m_ctx, P("/max.glb")); + ASSERT_TRUE(result.Succeeded()); + EXPECT_EQ(result.Value().AssetUUID, data.AssetUUID); + EXPECT_STREQ(result.Value().ImporterName, data.ImporterName); + EXPECT_STREQ(result.Value().SourcePath, data.SourcePath); + EXPECT_STREQ(result.Value().ArtifactPath, data.ArtifactPath); + EXPECT_EQ(result.Value().SettingsCount, META_MAX_SETTINGS); + for (uint32_t i = 0; i < META_MAX_SETTINGS; ++i) + { + EXPECT_STREQ(result.Value().Settings[i].Key, data.Settings[i].Key); + EXPECT_STREQ(result.Value().Settings[i].Value, data.Settings[i].Value); + } +} + +TEST_F(MetaFileIOTest, SizeBoundaryAcceptsLimitAndPreservesOversizedMetadata) +{ + std::string json = "{\"uuid\":\"550e8400-e29b-41d4-a716-446655440000\"}"; + json.resize(MetaFileIO::MaxFileSize, ' '); + WriteRaw("/large.glb.meta", json.c_str()); + ASSERT_TRUE(MetaFileIO::Read(m_ctx, P("/large.glb")).Succeeded()); + json.push_back(' '); + WriteRaw("/large.glb.meta", json.c_str()); + EXPECT_EQ(MetaFileIO::Read(m_ctx, P("/large.glb")).Error(), VFSError::SizeLimitExceeded); + EXPECT_EQ(MetaFileIO::GetOrCreate(m_ctx, P("/large.glb"), "Test", 123).Error(), VFSError::SizeLimitExceeded); + auto open = m_ctx.Open(P("/large.glb.meta"), VFSOpenFlags::Read); + ASSERT_TRUE(open.Succeeded()); + std::string retained(json.size(), '\0'); + auto read = open.Value()->ReadAll({reinterpret_cast(retained.data()), retained.size()}); + m_ctx.Close(open.Value()); + ASSERT_TRUE(read.Succeeded()); + EXPECT_EQ(retained, json); +} From 2806d60afcb19f0d9684e3ddf7ac7b5471432053 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Wed, 7 Oct 2026 09:04:43 +0900 Subject: [PATCH 27/27] fix(vfs): defer watcher processing for import publications --- Tetragrama/Panels/AssetImporterPanel.cpp | 90 ++++- Tetragrama/Panels/AssetImporterPanel.h | 27 +- ZEngine/ZEngine/Core/VFS/IVFSContext.h | 23 +- .../Core/VFS/Platform/VFSFSEventsWatcher.mm | 2 + .../Core/VFS/Platform/VFSInotifyWatcher.cpp | 11 +- .../Core/VFS/Platform/VFSRDCWatcher.cpp | 6 +- ZEngine/ZEngine/Core/VFS/VFSContext.cpp | 348 +++++++++++++----- ZEngine/ZEngine/Core/VFS/VFSContext.h | 62 +++- ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h | 8 + ZEngine/ZEngine/Importers/AssetCodec.cpp | 13 + ZEngine/ZEngine/Importers/AssetCodec.h | 9 +- ZEngine/ZEngine/Importers/AssimpImporter.cpp | 8 +- ZEngine/ZEngine/Importers/FbxImporter.cpp | 2 + ZEngine/ZEngine/Importers/GltfImporter.cpp | 1 + ZEngine/tests/VFS/VFSFileWatcherTest.cpp | 20 + .../tests/VFS/vfs_import_publication_test.cpp | 82 +++++ 16 files changed, 591 insertions(+), 121 deletions(-) create mode 100644 ZEngine/tests/VFS/vfs_import_publication_test.cpp diff --git a/Tetragrama/Panels/AssetImporterPanel.cpp b/Tetragrama/Panels/AssetImporterPanel.cpp index ccb4ec60..4f2d284f 100644 --- a/Tetragrama/Panels/AssetImporterPanel.cpp +++ b/Tetragrama/Panels/AssetImporterPanel.cpp @@ -45,10 +45,15 @@ namespace Tetragrama::Panels // Scratch arena for ImportConfiguration strings — carved from layer arena // ZKilo(64): ~8 path strings × ≤512 bytes each — a few KB is all we need layer->LocalArena.CreateSubArena(ZKilo(64), &m_local_arena, "EditorContext/AssetImporterPanelScratch"); - // Importer arenas carved from the engine's ImportPipeline budget so all // import memory — engine importers and editor importers — is budget-tracked. auto* import_arena = &ZEngine::Engine::GetContext()->ImportPipelineArena; + // Publication results can contain one entry per material and texture. Keep + // them outside the 64 KiB configuration scratch arena and reuse storage on + // the next import. The layer's shared local arena is only 4 MiB, so use the + // import budget for this bounded 8 MiB result slab. + m_import_result_slab.Init(import_arena, ZMega(8)); + m_import_task.PublishedPaths.init(&m_import_result_slab, 32); import_arena->CreateSubArena(ZMega(64), &m_gltf_importer_arena, "ImportPipeline/EditorGltfImporter"); import_arena->CreateSubArena(ZMega(128), &m_assimp_importer_arena, "ImportPipeline/EditorAssimpImporter"); @@ -647,9 +652,17 @@ namespace Tetragrama::Panels m_state.value.store(ImporterState::Importing, std::memory_order_release); m_progress.value.store(0.f, std::memory_order_relaxed); - m_import_task = {}; - m_import_task.Panel = this; - m_import_task.Configuration = *config; + m_import_task.Outputs.clear(); + m_import_task.PublishedPaths.clear(); + m_import_task.Error.clear(); + m_import_task.Outcome = ImportTaskOutcome::Pending; + m_import_task.Publication = {}; + m_import_task.Panel = this; + m_import_task.Configuration = *config; + m_import_task.Configuration.ArtifactContext = &m_import_task; + m_import_task.Configuration.OnArtifactPublished = &AssetImporterPanel::OnImportArtifactPublished; + if (m_import_task.Configuration.VFS) + m_import_task.Publication = m_import_task.Configuration.VFS->BeginImportPublication(); secure_strncpy(m_import_task.SourcePath, sizeof(m_import_task.SourcePath), m_path_buf, secure_strlen(m_path_buf)); const auto ext = vfs_value.Extension(); @@ -669,6 +682,7 @@ namespace Tetragrama::Panels if (!ZEngine::Helpers::ThreadPoolHelper::Submit(&m_import_task, &AssetImporterPanel::RunImportTask)) { + EndImportPublication(m_import_task); m_state.value.store(ImporterState::Options, std::memory_order_release); PushLog("Import task rejected because the thread pool is shutting down", kRed[0], kRed[1], kRed[2]); } @@ -682,7 +696,10 @@ namespace Tetragrama::Panels auto* panel = task->Panel; if (!panel) + { + EndImportPublication(*task); return; + } const auto& config = task->Configuration; switch (task->Kind) @@ -712,18 +729,28 @@ namespace Tetragrama::Panels void AssetImporterPanel::FinalizeImportTask(void* context) { auto* task = static_cast(context); - if (!task || !task->Panel) + if (!task) return; auto* panel = task->Panel; - if (task->Outcome == ImportTaskOutcome::Completed) + if (panel) { - ZEngine::Core::Containers::ArrayView outputs(task->Outputs.data(), task->Outputs.size()); - CompleteImportOnMainThread(panel, outputs); - return; + if (task->Outcome == ImportTaskOutcome::Completed) + { + ZEngine::Core::Containers::ArrayView outputs(task->Outputs.data(), task->Outputs.size()); + CompleteImportOnMainThread(panel, outputs); + } + else + { + CompleteImportErrorOnMainThread(panel, task->Error.empty() ? "Importer returned without an error message" : task->Error); + } } - CompleteImportErrorOnMainThread(panel, task->Error.empty() ? "Importer returned without an error message" : task->Error); + // CompleteImportOnMainThread can return early on metadata errors. Release the + // publication afterwards in every case so watcher processing cannot remain + // deferred after an import failure. + EndImportPublication(*task); + task->PublishedPaths.clear(); } void AssetImporterPanel::CreateMeshActor(uuids::uuid mesh_uuid, uint32_t render_instance_id, const char* name) @@ -793,14 +820,57 @@ namespace Tetragrama::Panels auto* task = static_cast(ctx); if (!task) return; + // An importer may have published some artifacts before reporting an error. + // Once it reports that error, completion must not turn the task into a + // success and hide it from the editor. + if (task->Outcome == ImportTaskOutcome::Failed) + return; task->Outputs.clear(); if (outputs.size() > 0) + { task->Outputs.assign(outputs.data(), outputs.data() + outputs.size()); + for (const auto& output : task->Outputs) + RecordPublishedPath(*task, output.Path.c_str()); + } task->Error.clear(); task->Outcome = ImportTaskOutcome::Completed; } + void AssetImporterPanel::OnImportArtifactPublished(void* ctx, ZEngine::Importers::AssetFileType, const char* path) + { + auto* task = static_cast(ctx); + if (task) + RecordPublishedPath(*task, path); + } + + void AssetImporterPanel::RecordPublishedPath(ImportTask& task, const char* path) + { + if (!path || path[0] == '\0') + return; + auto parsed = VFSPath::Parse(path); + if (parsed.Failed()) + return; + + for (const ZEngine::Core::VFS::VFSPath& existing : task.PublishedPaths) + if (existing == parsed.Value()) + return; + + task.PublishedPaths.push(parsed.Value()); + } + + void AssetImporterPanel::EndImportPublication(ImportTask& task) + { + auto* vfs = task.Configuration.VFS; + if (!vfs || !task.Publication.IsValid()) + return; + + for (const ZEngine::Core::VFS::VFSPath& published : task.PublishedPaths) + vfs->RecordImportPublicationArtifact(task.Publication, published); + vfs->EndImportPublication(task.Publication); + task.Publication = {}; + } + void AssetImporterPanel::CompleteImportOnMainThread(void* ctx, ZEngine::Core::Containers::ArrayView outputs) { auto* self = reinterpret_cast(ctx); diff --git a/Tetragrama/Panels/AssetImporterPanel.h b/Tetragrama/Panels/AssetImporterPanel.h index 5ecf923a..cbcf4563 100644 --- a/Tetragrama/Panels/AssetImporterPanel.h +++ b/Tetragrama/Panels/AssetImporterPanel.h @@ -1,7 +1,11 @@ #pragma once #include #include +#include +#include #include +#include +#include #include #include #include @@ -42,6 +46,10 @@ namespace Tetragrama::Panels // Scratch arena for ImportConfiguration strings (cleared before each import) ZEngine::Core::Memory::ArenaAllocator m_local_arena = {}; + // Task results must outlive the worker callback, but must not consume the + // 64 KiB configuration scratch arena. TLSF reuses and reclaims Array growth + // between imports without a per-import arena leak. + ZEngine::Core::Memory::TLSFSlab m_import_result_slab = {}; // Importer-dedicated arenas carved from the engine ImportPipeline budget ZEngine::Core::Memory::ArenaAllocator m_gltf_importer_arena = {}; ZEngine::Core::Memory::ArenaAllocator m_assimp_importer_arena = {}; @@ -67,15 +75,17 @@ namespace Tetragrama::Panels struct ImportTask { - AssetImporterPanel* Panel = nullptr; - ZEngine::Importers::AssetCodec::ImportConfiguration Configuration = {}; - ImporterKind Kind = ImporterKind::Gltf; - char SourcePath[MAX_FILE_PATH_COUNT] = {}; + AssetImporterPanel* Panel = nullptr; + ZEngine::Importers::AssetCodec::ImportConfiguration Configuration = {}; + ImporterKind Kind = ImporterKind::Gltf; + char SourcePath[MAX_FILE_PATH_COUNT] = {}; // The importer releases its scratch arena after it calls its callback. // Retain the small output list until the main thread consumes it. - std::vector Outputs = {}; - std::string Error = {}; - ImportTaskOutcome Outcome = ImportTaskOutcome::Pending; + std::vector Outputs = {}; + ZEngine::Core::Containers::Array PublishedPaths = {}; + ZEngine::Core::VFS::VFSImportPublication Publication = {}; + std::string Error = {}; + ImportTaskOutcome Outcome = ImportTaskOutcome::Pending; }; ImportTask m_import_task = {}; @@ -156,6 +166,7 @@ namespace Tetragrama::Panels // Static callbacks for ImportFile. They execute on a worker, so they only // copy the result into ImportTask; FinalizeImportTask performs editor work. static void OnImportFileComplete(void* ctx, ZEngine::Core::Containers::ArrayView outputs); + static void OnImportArtifactPublished(void* ctx, ZEngine::Importers::AssetFileType type, const char* path); static void OnImportProgress(void* ctx, float pct); static void OnImportError(void* ctx, std::string_view err); static void OnImportLog(void* ctx, std::string_view msg); @@ -163,6 +174,8 @@ namespace Tetragrama::Panels // Main-thread-only completion handlers. static void CompleteImportOnMainThread(void* ctx, ZEngine::Core::Containers::ArrayView outputs); static void CompleteImportErrorOnMainThread(void* ctx, std::string_view err); + static void RecordPublishedPath(ImportTask& task, const char* path); + static void EndImportPublication(ImportTask& task); }; } // namespace Tetragrama::Panels diff --git a/ZEngine/ZEngine/Core/VFS/IVFSContext.h b/ZEngine/ZEngine/Core/VFS/IVFSContext.h index 8ba5f5b9..1b7900e4 100644 --- a/ZEngine/ZEngine/Core/VFS/IVFSContext.h +++ b/ZEngine/ZEngine/Core/VFS/IVFSContext.h @@ -4,10 +4,21 @@ #include #include #include +#include namespace ZEngine::Core::VFS { + struct VFSImportPublication + { + uint64_t Id = 0; + + [[nodiscard]] bool IsValid() const + { + return Id != 0; + } + }; + struct IVFSContext { virtual ~IVFSContext() = default; @@ -34,7 +45,17 @@ namespace ZEngine::Core::VFS virtual void Close(IVFSFile* const file) = 0; - virtual void Shutdown() = 0; + // Editor importers use this scope to publish a group of generated files without + // letting watcher callbacks index the group while it is incomplete. Default + // no-ops preserve the lightweight VFS contexts used by tools and tests. + [[nodiscard]] virtual VFSImportPublication BeginImportPublication() + { + return {}; + } + virtual void RecordImportPublicationArtifact(VFSImportPublication, const VFSPath&) {} + virtual void EndImportPublication(VFSImportPublication) {} + + virtual void Shutdown() = 0; }; } // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm b/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm index 1ba1079f..3070ecb1 100644 --- a/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm +++ b/ZEngine/ZEngine/Core/VFS/Platform/VFSFSEventsWatcher.mm @@ -21,6 +21,7 @@ Helpers::secure_strncpy(ev.Path, sizeof(ev.Path), path, ClampedLength(path)); ev.Kind = kind; ev.IsDirectory = is_directory; + ev.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); return ev; } @@ -103,6 +104,7 @@ { VFSWatchEvent overflow{}; overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); self->PushEvent(overflow); continue; } diff --git a/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp b/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp index 9f066e82..fc4c661c 100644 --- a/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp +++ b/ZEngine/ZEngine/Core/VFS/Platform/VFSInotifyWatcher.cpp @@ -57,8 +57,9 @@ namespace ZEngine::Core::VFS { VFSWatchEvent ev{}; Helpers::secure_strncpy(ev.Path, sizeof(ev.Path), path, ClampedLength(path)); - ev.Kind = kind; - ev.IsDirectory = is_directory; + ev.Kind = kind; + ev.IsDirectory = is_directory; + ev.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); return ev; } @@ -112,7 +113,8 @@ namespace ZEngine::Core::VFS if (wd < 0) { VFSWatchEvent overflow{}; - overflow.Kind = WatchEventKind::Overflow; + overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); PushEvent(overflow); return -1; } @@ -236,7 +238,8 @@ namespace ZEngine::Core::VFS if (raw->mask & IN_Q_OVERFLOW) { VFSWatchEvent overflow{}; - overflow.Kind = WatchEventKind::Overflow; + overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); m_batch.push(overflow); continue; } diff --git a/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp b/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp index 69fc450b..8be12734 100644 --- a/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp +++ b/ZEngine/ZEngine/Core/VFS/Platform/VFSRDCWatcher.cpp @@ -59,7 +59,8 @@ namespace ZEngine::Core::VFS { VFSWatchEvent ev{}; Helpers::secure_strncpy(ev.Path, sizeof(ev.Path), path, ClampedLength(path)); - ev.Kind = kind; + ev.Kind = kind; + ev.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); return ev; } @@ -172,7 +173,8 @@ namespace ZEngine::Core::VFS if (bytes == 0) { VFSWatchEvent overflow{}; - overflow.Kind = WatchEventKind::Overflow; + overflow.Kind = WatchEventKind::Overflow; + overflow.ObservedAtNanoseconds = VFSWatchTimestampNowNanoseconds(); PushEvent(overflow); return; } diff --git a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp index 2269ab37..eac1eb18 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp @@ -20,6 +20,12 @@ namespace ZEngine::Core::VFS { m_arena = arena; m_mount_table.Initialize(m_arena, mount_table_capacity); + + m_active_import_publication.Artifacts.init(m_arena, 16); + m_active_import_publication.DirtyDirectories.init(m_arena, 8); + m_active_import_publication.DeferredEvents.init(m_arena, 32); + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + m_completed_import_publications[i].Artifacts.init(m_arena, 16); } void VFSContext::InitWatcher(const char* project_root_native, VFSDirectoryCache* cache, VFSScanner* scanner, AssetRegistry* registry, Importers::ImportCoordinator* coordinator, FileChangeListener file_change_listener, void* file_change_context) @@ -119,107 +125,271 @@ namespace ZEngine::Core::VFS // simultaneous pending debounce entries than that before any flush. m_file_watcher->Initialize(m_arena, 256); - // ev.Path/ev.OldPath are native absolute paths from the OS watcher, but every - // downstream consumer expects a workspace-relative VFSPath and prepends the - // workspace root itself — VFSPath::FromNative is just Parse, so it wouldn't strip - // that root. Do it here, once, instead of double-prefixing everywhere downstream. - auto to_relative_vfs_path = [this](cstring native) -> VFSResult { - const size_t root_len = Helpers::secure_strlen(m_project_root_native); - const size_t native_len = Helpers::secure_strlen(native); - if (native_len >= root_len && strncmp(native, m_project_root_native, root_len) == 0 && (native[root_len] == '\0' || native[root_len] == PLATFORM_OS_BACKSLASH)) - return VFSPath::Parse(native[root_len] != '\0' ? native + root_len : "/"); - return VFSPath::FromNative(native); - }; + const WatchHandle root_handle = m_file_watcher->Watch(m_project_root_native, /*recursive=*/true, [this](const VFSWatchEvent& event) { HandleWatchEvent(event); }); + + if (root_handle == INVALID_WATCH_HANDLE) + { + ZENGINE_LOG_VFS_ERR("InitWatcher: failed to watch root path '{}'", m_project_root_native); + m_file_watcher->~VFSFileWatcher(); + m_file_watcher = nullptr; + m_platform_watcher->~IVFSPlatformWatcher(); + m_platform_watcher = nullptr; + return; + } + + m_platform_watcher->StartThread(); + } + + VFSImportPublication VFSContext::BeginImportPublication() + { + if (m_active_import_publication.Publication.IsValid()) + { + ++m_active_import_publication.Depth; + return m_active_import_publication.Publication; + } + + m_active_import_publication.Publication = {.Id = m_next_import_publication_id++}; + m_active_import_publication.StartedAtNs = VFSWatchTimestampNowNanoseconds(); + m_active_import_publication.Depth = 1; + m_active_import_publication.RequiresScan = false; + m_active_import_publication.Artifacts.clear(); + m_active_import_publication.DirtyDirectories.clear(); + m_active_import_publication.DeferredEvents.clear(); + return m_active_import_publication.Publication; + } - const WatchHandle root_handle = m_file_watcher->Watch(m_project_root_native, /*recursive=*/true, [this, to_relative_vfs_path](const VFSWatchEvent& ev) { - const bool full_rescan = (ev.Kind == WatchEventKind::Overflow); + void VFSContext::RecordImportPublicationArtifact(VFSImportPublication publication, const VFSPath& path) + { + if (!publication.IsValid() || publication.Id != m_active_import_publication.Publication.Id || !path.IsValid()) + return; - VFSResult path = full_rescan ? to_relative_vfs_path(m_project_root_native) : to_relative_vfs_path(ev.Path); - if (path.Failed()) + for (size_t i = 0; i < m_active_import_publication.Artifacts.size(); ++i) + if (m_active_import_publication.Artifacts[i] == path) return; - const VFSPath target = (ev.IsDirectory || full_rescan) ? path.Value() : path.Value().Parent(); - const VFSPath file_path = path.Value(); + m_active_import_publication.Artifacts.push(path); - // Invalidate directory cache - if (m_directory_cache) + const VFSPath parent = path.Parent(); + for (size_t i = 0; i < m_active_import_publication.DirtyDirectories.size(); ++i) + if (m_active_import_publication.DirtyDirectories[i] == parent) + return; + m_active_import_publication.DirtyDirectories.push(parent); + } + + void VFSContext::EndImportPublication(VFSImportPublication publication) + { + if (!publication.IsValid() || publication.Id != m_active_import_publication.Publication.Id) + return; + if (m_active_import_publication.Depth > 1) + { + --m_active_import_publication.Depth; + return; + } + + const uint64_t ended_at_ns = VFSWatchTimestampNowNanoseconds(); + + // Events arriving during a publication are held until all artifact paths are + // known. Generated paths are coalesced into the deferred scan; unrelated + // paths retain the normal registry/coordinator behavior. + for (size_t i = 0; i < m_active_import_publication.DeferredEvents.size(); ++i) + { + const VFSWatchEvent& event = m_active_import_publication.DeferredEvents[i]; + if (event.Kind == WatchEventKind::Overflow || IsPublicationEvent(m_active_import_publication.StartedAtNs, m_active_import_publication.Artifacts, event)) { - m_directory_cache->Invalidate(target); - if (ev.Kind == WatchEventKind::Renamed && ev.OldPath[0] != '\0') - { - VFSResult old_path = to_relative_vfs_path(ev.OldPath); - if (old_path.Succeeded()) - m_directory_cache->Invalidate(ev.IsDirectory ? old_path.Value() : old_path.Value().Parent()); - } + m_active_import_publication.RequiresScan = true; + continue; } + ProcessWatchEvent(event); + } + m_active_import_publication.DeferredEvents.clear(); - // Notify AssetRegistry and ImportCoordinator for file (non-directory) events - // Skip .meta sidecar files — they are written by the coordinator itself and must - // not be fed back into the import queue. - auto is_temporary = [](const VFSPath& p) -> bool { - auto ext = p.Extension(); - return ext.Data && ext.Length == 4 && // ".tmp" - ext.Data[0] == '.' && ext.Data[1] == 't' && ext.Data[2] == 'm' && ext.Data[3] == 'p'; - }; - auto is_meta = [](const VFSPath& p) -> bool { - auto ext = p.Extension(); - return ext.Data && ext.Length == 5 && // ".meta" - ext.Data[0] == '.' && ext.Data[1] == 'm' && ext.Data[2] == 'e' && ext.Data[3] == 't' && ext.Data[4] == 'a'; - }; - // All generated assets use an atomic .tmp -> rename write. The temporary - // file is not an importable project source and must not enter the import - // queue (or trigger a directory scan) before publication completes. - if (!ev.IsDirectory && !full_rescan && is_temporary(file_path)) - return; - if (!ev.IsDirectory && !full_rescan && !is_meta(file_path)) + if (!m_active_import_publication.Artifacts.empty()) + { + for (size_t i = 0; i < m_active_import_publication.DirtyDirectories.size(); ++i) + if (m_directory_cache) + m_directory_cache->Invalidate(m_active_import_publication.DirtyDirectories[i]); + m_active_import_publication.RequiresScan = true; + } + + // Retain the exact output set until watcher delivery has settled. A raw + // inotify/FSEvents event may arrive after the importer returns, so using + // EndImportPublication's timestamp as a causal boundary is not reliable. + CompletedImportPublication& completed = m_completed_import_publications[m_next_completed_import_publication]; + completed.StartedAtNs = m_active_import_publication.StartedAtNs; + completed.SettleUntilNs = ended_at_ns + IMPORT_PUBLICATION_SETTLE_NS; + completed.RequiresScan = m_active_import_publication.RequiresScan; + completed.Artifacts.clear(); + for (size_t i = 0; i < m_active_import_publication.Artifacts.size(); ++i) + completed.Artifacts.push(m_active_import_publication.Artifacts[i]); + m_next_completed_import_publication = (m_next_completed_import_publication + 1) % COMPLETED_IMPORT_PUBLICATION_COUNT; + + m_active_import_publication.Publication = {}; + m_active_import_publication.StartedAtNs = 0; + m_active_import_publication.Depth = 0; + m_active_import_publication.RequiresScan = false; + m_active_import_publication.Artifacts.clear(); + m_active_import_publication.DirtyDirectories.clear(); + } + + VFSResult VFSContext::ToRelativeVFSPath(cstring native) const + { + if (!native) + return VFSResult::Fail(VFSError::InvalidPath); + + const size_t root_len = Helpers::secure_strlen(m_project_root_native); + const size_t native_len = Helpers::secure_strlen(native); + if (native_len >= root_len && strncmp(native, m_project_root_native, root_len) == 0 && (native[root_len] == '\0' || native[root_len] == PLATFORM_OS_BACKSLASH)) + return VFSPath::Parse(native[root_len] != '\0' ? native + root_len : "/"); + return VFSPath::FromNative(native); + } + + bool VFSContext::IsPublicationArtifact(const Containers::Array& artifacts, const VFSPath& path) const + { + for (size_t i = 0; i < artifacts.size(); ++i) + { + const VFSPath& artifact = artifacts[i]; + if (artifact == path) + return true; + // Metadata is stored beside its registered artifact as .meta. + if (path.Length() == artifact.Length() + 5 && strncmp(path.CStr(), artifact.CStr(), artifact.Length()) == 0 && strncmp(path.CStr() + artifact.Length(), ".meta", 5) == 0) + return true; + } + return false; + } + + bool VFSContext::IsPublicationEvent(uint64_t started_at_ns, const Containers::Array& artifacts, const VFSWatchEvent& event) const + { + if (event.Kind == WatchEventKind::Overflow || event.ObservedAtNanoseconds == 0 || event.ObservedAtNanoseconds < started_at_ns) + return false; + + VFSResult path = ToRelativeVFSPath(event.Path); + return path.Succeeded() && IsPublicationArtifact(artifacts, path.Value()); + } + + void VFSContext::HandleWatchEvent(const VFSWatchEvent& event) + { + if (m_active_import_publication.Publication.IsValid() && event.ObservedAtNanoseconds >= m_active_import_publication.StartedAtNs) + { + m_active_import_publication.DeferredEvents.push(event); + return; + } + + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + CompletedImportPublication& completed = m_completed_import_publications[i]; + if (completed.SettleUntilNs != 0 && VFSWatchTimestampNowNanoseconds() <= completed.SettleUntilNs && IsPublicationEvent(completed.StartedAtNs, completed.Artifacts, event)) { - switch (ev.Kind) - { - case WatchEventKind::Modified: - if (m_registry) - m_registry->OnAssetModified(file_path); - if (m_coordinator) - m_coordinator->Enqueue(file_path, Importers::ImportPriority::Immediate); - break; + completed.RequiresScan = true; + return; + } + } - case WatchEventKind::Deleted: - if (m_registry) - m_registry->OnAssetDeleted(file_path); - break; + ProcessWatchEvent(event); + } - case WatchEventKind::Renamed: - if (m_registry && ev.OldPath[0] != '\0') - { - VFSResult old_path = to_relative_vfs_path(ev.OldPath); - if (old_path.Succeeded()) - m_registry->OnAssetRenamed(old_path.Value(), file_path); - } - break; + void VFSContext::ProcessWatchEvent(const VFSWatchEvent& event) + { + const bool full_rescan = (event.Kind == WatchEventKind::Overflow); + VFSResult path = full_rescan ? ToRelativeVFSPath(m_project_root_native) : ToRelativeVFSPath(event.Path); + if (path.Failed()) + return; - default: - break; - } + const VFSPath target = (event.IsDirectory || full_rescan) ? path.Value() : path.Value().Parent(); + const VFSPath file_path = path.Value(); - if (m_file_change_listener) - m_file_change_listener(m_file_change_context, file_path, ev.Kind); + if (m_directory_cache) + { + m_directory_cache->Invalidate(target); + if (event.Kind == WatchEventKind::Renamed && event.OldPath[0] != '\0') + { + VFSResult old_path = ToRelativeVFSPath(event.OldPath); + if (old_path.Succeeded()) + m_directory_cache->Invalidate(event.IsDirectory ? old_path.Value() : old_path.Value().Parent()); } + } - // Rescan directory - if (m_scanner && m_directory_cache && !m_scanner->IsScanning()) - m_scanner->Scan(this, target, m_directory_cache); - }); + auto is_temporary = [](const VFSPath& path) -> bool { + const VFSPathComponent extension = path.Extension(); + return extension.Data && extension.Length == 4 && extension.Data[0] == '.' && extension.Data[1] == 't' && extension.Data[2] == 'm' && extension.Data[3] == 'p'; + }; + auto is_meta = [](const VFSPath& path) -> bool { + const VFSPathComponent extension = path.Extension(); + return extension.Data && extension.Length == 5 && extension.Data[0] == '.' && extension.Data[1] == 'm' && extension.Data[2] == 'e' && extension.Data[3] == 't' && extension.Data[4] == 'a'; + }; - if (root_handle == INVALID_WATCH_HANDLE) + if (!event.IsDirectory && !full_rescan && is_temporary(file_path)) + return; + if (!event.IsDirectory && !full_rescan && !is_meta(file_path)) { - ZENGINE_LOG_VFS_ERR("InitWatcher: failed to watch root path '{}'", m_project_root_native); - m_file_watcher->~VFSFileWatcher(); - m_file_watcher = nullptr; - m_platform_watcher->~IVFSPlatformWatcher(); - m_platform_watcher = nullptr; + switch (event.Kind) + { + case WatchEventKind::Modified: + if (m_registry) + m_registry->OnAssetModified(file_path); + if (m_coordinator) + m_coordinator->Enqueue(file_path, Importers::ImportPriority::Immediate); + break; + + case WatchEventKind::Deleted: + if (m_registry) + m_registry->OnAssetDeleted(file_path); + break; + + case WatchEventKind::Renamed: + if (m_registry && event.OldPath[0] != '\0') + { + VFSResult old_path = ToRelativeVFSPath(event.OldPath); + if (old_path.Succeeded()) + m_registry->OnAssetRenamed(old_path.Value(), file_path); + } + break; + + default: + break; + } + + if (m_file_change_listener) + m_file_change_listener(m_file_change_context, file_path, event.Kind); + } + + if (m_scanner && m_directory_cache && !m_scanner->IsScanning()) + m_scanner->Scan(this, target, m_directory_cache); + } + + void VFSContext::FlushDeferredImportScan() + { + if (!m_scanner || !m_directory_cache) return; + + const uint64_t now_ns = VFSWatchTimestampNowNanoseconds(); + bool should_scan = false; + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + const CompletedImportPublication& completed = m_completed_import_publications[i]; + if (completed.SettleUntilNs != 0 && completed.RequiresScan && now_ns >= completed.SettleUntilNs) + { + should_scan = true; + break; + } } + if (!should_scan || m_scanner->IsScanning()) + return; - m_platform_watcher->StartThread(); + // Start one scan for every completed publication whose watcher grace period + // elapsed. The scan observes the final filesystem state, including a real + // external edit that happened during the grace period. + ScanProject(); + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + CompletedImportPublication& completed = m_completed_import_publications[i]; + if (completed.SettleUntilNs != 0 && now_ns >= completed.SettleUntilNs) + { + completed.StartedAtNs = 0; + completed.SettleUntilNs = 0; + completed.RequiresScan = false; + completed.Artifacts.clear(); + } + } } void VFSContext::ScanProject() @@ -234,6 +404,7 @@ namespace ZEngine::Core::VFS { m_file_watcher->Tick(); } + FlushDeferredImportScan(); } void VFSContext::ShutdownWatcher() @@ -252,8 +423,19 @@ namespace ZEngine::Core::VFS m_platform_watcher->~IVFSPlatformWatcher(); m_platform_watcher = nullptr; } - m_directory_cache = nullptr; - m_scanner = nullptr; + m_directory_cache = nullptr; + m_scanner = nullptr; + m_active_import_publication.Publication = {}; + m_active_import_publication.Artifacts.clear(); + m_active_import_publication.DirtyDirectories.clear(); + m_active_import_publication.DeferredEvents.clear(); + for (uint32_t i = 0; i < COMPLETED_IMPORT_PUBLICATION_COUNT; ++i) + { + m_completed_import_publications[i].StartedAtNs = 0; + m_completed_import_publications[i].SettleUntilNs = 0; + m_completed_import_publications[i].RequiresScan = false; + m_completed_import_publications[i].Artifacts.clear(); + } } VFSResult VFSContext::Open(const VFSPath& absolute_path, VFSOpenFlags flags) diff --git a/ZEngine/ZEngine/Core/VFS/VFSContext.h b/ZEngine/ZEngine/Core/VFS/VFSContext.h index 7cabeaa0..bc8aa793 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSContext.h +++ b/ZEngine/ZEngine/Core/VFS/VFSContext.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -26,6 +27,7 @@ namespace ZEngine namespace ZEngine::Core::VFS { struct VFSScanner; + struct VFSContextTestAccess; using FileChangeListener = void (*)(void* context, const VFSPath& path, WatchEventKind kind); @@ -50,6 +52,10 @@ namespace ZEngine::Core::VFS void ShutdownWatcher(); + [[nodiscard]] VFSImportPublication BeginImportPublication() override; + void RecordImportPublicationArtifact(VFSImportPublication publication, const VFSPath& path) override; + void EndImportPublication(VFSImportPublication publication) override; + [[nodiscard]] VFSResult Open(const VFSPath& absolute_path, VFSOpenFlags flags) override; void Close(IVFSFile* file) override; [[nodiscard]] VFSResult> List(const VFSPath& absolute_dir, Memory::ArenaAllocator* out_arena) override; @@ -64,21 +70,59 @@ namespace ZEngine::Core::VFS void Shutdown() override; private: + friend struct VFSContextTestAccess; + + struct ImportPublicationState + { + VFSImportPublication Publication = {}; + uint64_t StartedAtNs = 0; + uint32_t Depth = 0; + bool RequiresScan = false; + Containers::Array Artifacts = {}; + Containers::Array DirtyDirectories = {}; + Containers::Array DeferredEvents = {}; + }; + + struct CompletedImportPublication + { + uint64_t StartedAtNs = 0; + uint64_t SettleUntilNs = 0; + bool RequiresScan = false; + Containers::Array Artifacts = {}; + }; + [[nodiscard]] VFSResult ResolveWritable(const VFSPath& path) const; + [[nodiscard]] VFSResult ToRelativeVFSPath(cstring native) const; + [[nodiscard]] bool IsPublicationEvent(uint64_t started_at_ns, const Containers::Array& artifacts, const VFSWatchEvent& event) const; + [[nodiscard]] bool IsPublicationArtifact(const Containers::Array& artifacts, const VFSPath& path) const; + void HandleWatchEvent(const VFSWatchEvent& event); + void ProcessWatchEvent(const VFSWatchEvent& event); + void FlushDeferredImportScan(); VFSMountTable m_mount_table = {}; Memory::ArenaAllocator* m_arena = nullptr; mutable std::mutex m_arena_mutex; - IVFSPlatformWatcher* m_platform_watcher = nullptr; - VFSFileWatcher* m_file_watcher = nullptr; - VFSDirectoryCache* m_directory_cache = nullptr; - VFSScanner* m_scanner = nullptr; - AssetRegistry* m_registry = nullptr; - Importers::ImportCoordinator* m_coordinator = nullptr; - FileChangeListener m_file_change_listener = nullptr; - void* m_file_change_context = nullptr; - char m_project_root_native[MAX_FILE_PATH_COUNT] = {}; + IVFSPlatformWatcher* m_platform_watcher = nullptr; + VFSFileWatcher* m_file_watcher = nullptr; + VFSDirectoryCache* m_directory_cache = nullptr; + VFSScanner* m_scanner = nullptr; + AssetRegistry* m_registry = nullptr; + Importers::ImportCoordinator* m_coordinator = nullptr; + FileChangeListener m_file_change_listener = nullptr; + void* m_file_change_context = nullptr; + char m_project_root_native[MAX_FILE_PATH_COUNT] = {}; + + // Native watchers observe filesystem changes asynchronously. Keep each + // completed publication long enough for the raw event and its debounce + // window to drain before a single reconciliation scan is allowed through. + static constexpr uint64_t IMPORT_PUBLICATION_SETTLE_NS = 500'000'000ULL; + static constexpr uint32_t COMPLETED_IMPORT_PUBLICATION_COUNT = 8; + + ImportPublicationState m_active_import_publication = {}; + CompletedImportPublication m_completed_import_publications[COMPLETED_IMPORT_PUBLICATION_COUNT] = {}; + uint32_t m_next_completed_import_publication = 0; + uint64_t m_next_import_publication_id = 1; }; } // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h b/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h index 21994ef3..9c313b3c 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h +++ b/ZEngine/ZEngine/Core/VFS/VFSWatchEvent.h @@ -1,8 +1,14 @@ #pragma once #include +#include namespace ZEngine::Core::VFS { + [[nodiscard]] inline uint64_t VFSWatchTimestampNowNanoseconds() + { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + } + enum class WatchEventKind : uint8_t { Created = 0, @@ -18,5 +24,7 @@ namespace ZEngine::Core::VFS char OldPath[MAX_FILE_PATH_COUNT] = {}; WatchEventKind Kind = WatchEventKind::Created; bool IsDirectory = false; + // Captured by the platform watcher before VFSFileWatcher debounces the event. + uint64_t ObservedAtNanoseconds = 0; }; } // namespace ZEngine::Core::VFS diff --git a/ZEngine/ZEngine/Importers/AssetCodec.cpp b/ZEngine/ZEngine/Importers/AssetCodec.cpp index 0329589e..1d194a77 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.cpp +++ b/ZEngine/ZEngine/Importers/AssetCodec.cpp @@ -99,6 +99,12 @@ namespace ZEngine::Importers::AssetCodec return path; } + void ReportPublishedArtifact(const ImportConfiguration& config, AssetFileType type, const VFSPath& path) + { + if (config.OnArtifactPublished && path.IsValid()) + config.OnArtifactPublished(config.ArtifactContext, type, path.CStr()); + } + VFSResult CopyTextureFiles(ArrayView textures, const ImportConfiguration& config) { if (textures.size() == 0) @@ -142,6 +148,7 @@ namespace ZEngine::Importers::AssetCodec ZENGINE_CORE_ERROR("Failed to copy texture '{}' to '{}' (VFS error {})", source.string(), destination.Value().CStr(), static_cast(write.Error())) return write; } + ReportPublishedArtifact(config, AssetFileType::TEXTURES, destination.Value()); texture.Path.clear(); texture.Path.append(destination.Value().CStr()); } @@ -205,6 +212,8 @@ namespace ZEngine::Importers::AssetCodec if (write.Failed()) return VFSResult::Fail(write.Error()); + ReportPublishedArtifact(config, AssetFileType::MESH, mesh_path); + return VFSResult::Ok({.Type = AssetFileType::MESH, .Path = mesh_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); } @@ -295,6 +304,8 @@ namespace ZEngine::Importers::AssetCodec if (write.Failed()) return VFSResult::Fail(write.Error()); + ReportPublishedArtifact(config, AssetFileType::MATERIAL, mat_path); + return VFSResult::Ok({.Type = AssetFileType::MATERIAL, .Path = mat_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); } @@ -323,6 +334,8 @@ namespace ZEngine::Importers::AssetCodec if (write.Failed()) return VFSResult::Fail(write.Error()); + ReportPublishedArtifact(config, AssetFileType::TEXTURES, tex_path); + return VFSResult::Ok({.Type = AssetFileType::TEXTURES, .Path = tex_path.CStr(), .RootPath = config.OutputWorkingSpacePath.empty() ? "" : config.OutputWorkingSpacePath.c_str()}); } diff --git a/ZEngine/ZEngine/Importers/AssetCodec.h b/ZEngine/ZEngine/Importers/AssetCodec.h index 22c4b114..991073b3 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.h +++ b/ZEngine/ZEngine/Importers/AssetCodec.h @@ -53,14 +53,19 @@ namespace ZEngine::Importers::AssetCodec Core::Containers::String InputBaseAssetFilePath; Core::Containers::String OutputWorkingSpacePath; Core::Containers::String OutputTextureFilesPath; - Core::VFS::IVFSContext* VFS = nullptr; - ImportOptions Options = {}; + Core::VFS::IVFSContext* VFS = nullptr; + // Called synchronously after an artifact commits. This remains separate from + // the completion callback so a failed import can still report partial output. + void* ArtifactContext = nullptr; + void (*OnArtifactPublished)(void* context, AssetFileType type, const char* path) = nullptr; + ImportOptions Options = {}; }; // Validate before parsing a source or accessing VFS metadata; returns the mesh destination. [[nodiscard]] Core::VFS::VFSResult ValidateImportConfiguration(const ImportConfiguration& config); [[nodiscard]] Core::VFS::VFSResult MakeOutputPath(const char* directory, const char* filename); [[nodiscard]] Core::VFS::VFSResult CopyTextureFiles(Core::Containers::ArrayView textures, const ImportConfiguration& config); + void ReportPublishedArtifact(const ImportConfiguration& config, AssetFileType type, const Core::VFS::VFSPath& path); std::string MaterialOutputFilename(const AssetMaterial& material, size_t index); [[nodiscard]] Core::VFS::VFSResult MaterialOutputPath(const AssetMaterial& material, const ImportConfiguration& config, size_t index); [[nodiscard]] Core::VFS::VFSResult SynchronizeTextureMetadata(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, Core::Containers::ArrayView materials, const ImportConfiguration& config, const char* importer); diff --git a/ZEngine/ZEngine/Importers/AssimpImporter.cpp b/ZEngine/ZEngine/Importers/AssimpImporter.cpp index 97c75ed3..c0b8f609 100644 --- a/ZEngine/ZEngine/Importers/AssimpImporter.cpp +++ b/ZEngine/ZEngine/Importers/AssimpImporter.cpp @@ -117,10 +117,12 @@ namespace ZEngine::Importers config.AssetName.init(arena, cfg.AssetName.c_str()); config.OutputAssetFile.init(arena, cfg.OutputAssetFile.c_str()); config.InputBaseAssetFilePath.init(arena, cfg.InputBaseAssetFilePath.c_str()); - config.VFS = cfg.VFS; - config.Options = cfg.Options; + config.VFS = cfg.VFS; + config.ArtifactContext = cfg.ArtifactContext; + config.OnArtifactPublished = cfg.OnArtifactPublished; + config.Options = cfg.Options; - uint32_t flags = aiProcess_Triangulate | aiProcess_SortByPType; + uint32_t flags = aiProcess_Triangulate | aiProcess_SortByPType; if (config.Options.MergeVertices) flags |= aiProcess_JoinIdenticalVertices; if (config.Options.FlipUVs) diff --git a/ZEngine/ZEngine/Importers/FbxImporter.cpp b/ZEngine/ZEngine/Importers/FbxImporter.cpp index eefe0b3b..6032ae91 100644 --- a/ZEngine/ZEngine/Importers/FbxImporter.cpp +++ b/ZEngine/ZEngine/Importers/FbxImporter.cpp @@ -235,6 +235,8 @@ namespace ZEngine::Importers config.OutputAssetFile.init(arena, cfg.OutputAssetFile.c_str()); config.InputBaseAssetFilePath.init(arena, cfg.InputBaseAssetFilePath.c_str()); config.VFS = cfg.VFS; + config.ArtifactContext = cfg.ArtifactContext; + config.OnArtifactPublished = cfg.OnArtifactPublished; config.Options = cfg.Options; ufbx_load_opts opts = {}; diff --git a/ZEngine/ZEngine/Importers/GltfImporter.cpp b/ZEngine/ZEngine/Importers/GltfImporter.cpp index eac778e8..f2318fd1 100644 --- a/ZEngine/ZEngine/Importers/GltfImporter.cpp +++ b/ZEngine/ZEngine/Importers/GltfImporter.cpp @@ -1045,6 +1045,7 @@ namespace ZEngine::Importers if (write.Succeeded()) { ZENGINE_LOG_ASSET_INFO("GltfImporter: extracted texture '{}' ({} bytes)", output_path.Value().CStr(), nbytes) + AssetCodec::ReportPublishedArtifact(config, AssetFileType::TEXTURES, output_path.Value()); textures[tex_idx].Path.init(&scratch, output_path.Value().CStr()); } else diff --git a/ZEngine/tests/VFS/VFSFileWatcherTest.cpp b/ZEngine/tests/VFS/VFSFileWatcherTest.cpp index ca7c935c..07bf5907 100644 --- a/ZEngine/tests/VFS/VFSFileWatcherTest.cpp +++ b/ZEngine/tests/VFS/VFSFileWatcherTest.cpp @@ -248,6 +248,26 @@ TEST_F(VFSFileWatcherTest, RenamePreservesOldPath) EXPECT_EQ(received.Kind, WatchEventKind::Renamed); } +TEST_F(VFSFileWatcherTest, DebouncePreservesObservationTimestamp) +{ + MockPlatformWatcher mock; + VFSFileWatcher watcher(&mock, std::chrono::milliseconds{10}); + watcher.Initialize(&m_manager.MainArena); + + uint64_t observed_at = 0; + watcher.Watch("/project", true, [&](const VFSWatchEvent& event) { observed_at = event.ObservedAtNanoseconds; }); + + VFSWatchEvent event = MakeEvent("/project/imported.zemesh", WatchEventKind::Renamed); + event.ObservedAtNanoseconds = 123456789; + mock.InjectEvent(event); + + watcher.Tick(); + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + watcher.Tick(); + + EXPECT_EQ(observed_at, 123456789u); +} + TEST_F(VFSFileWatcherTest, MultipleWatchesRouteCorrectly) { MockPlatformWatcher mock; diff --git a/ZEngine/tests/VFS/vfs_import_publication_test.cpp b/ZEngine/tests/VFS/vfs_import_publication_test.cpp new file mode 100644 index 00000000..e257f568 --- /dev/null +++ b/ZEngine/tests/VFS/vfs_import_publication_test.cpp @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +using ZEngine::Core::Memory::MemoryManager; + +namespace ZEngine::Core::VFS +{ + // Keeps watcher ingress private in production while allowing this focused + // regression test to model an inotify event delivered after publication end. + struct VFSContextTestAccess + { + static void Configure(VFSContext& context, const char* project_root, FileChangeListener listener, void* listener_context) + { + Helpers::secure_strcpy(context.m_project_root_native, sizeof(context.m_project_root_native), project_root); + context.m_file_change_listener = listener; + context.m_file_change_context = listener_context; + } + + static void Deliver(VFSContext& context, const VFSWatchEvent& event) + { + context.HandleWatchEvent(event); + } + }; +} // namespace ZEngine::Core::VFS + +namespace +{ + struct ListenerCounter + { + int Count = 0; + }; + + void CountEvent(void* context, const ZEngine::Core::VFS::VFSPath&, ZEngine::Core::VFS::WatchEventKind) + { + static_cast(context)->Count++; + } + + ZEngine::Core::VFS::VFSWatchEvent MakeEvent(const char* path) + { + ZEngine::Core::VFS::VFSWatchEvent event{}; + ZEngine::Helpers::secure_strcpy(event.Path, sizeof(event.Path), path); + event.Kind = ZEngine::Core::VFS::WatchEventKind::Modified; + event.ObservedAtNanoseconds = ZEngine::Core::VFS::VFSWatchTimestampNowNanoseconds(); + return event; + } +} // namespace + +TEST(VFSImportPublicationTest, SuppressesArtifactEventDeliveredAfterPublicationEnds) +{ + MemoryManager manager; + + manager.Initialize(ZMega(2), {}); + { + ZEngine::Core::VFS::VFSContext context; + ListenerCounter listener; + + context.Initialize(&manager.MainArena); + ZEngine::Core::VFS::VFSContextTestAccess::Configure(context, "/project", &CountEvent, &listener); + + const auto artifact = ZEngine::Core::VFS::VFSPath::Parse("/Assets/Meshes/robot.zemesh"); + ASSERT_TRUE(artifact.Succeeded()); + + const ZEngine::Core::VFS::VFSImportPublication publication = context.BeginImportPublication(); + context.RecordImportPublicationArtifact(publication, artifact.Value()); + context.EndImportPublication(publication); + + // The raw event is observed after EndImportPublication. It must still be + // absorbed by the publication grace period instead of notifying the import + // coordinator through the generic file-change listener. + ZEngine::Core::VFS::VFSContextTestAccess::Deliver(context, MakeEvent("/project/Assets/Meshes/robot.zemesh")); + EXPECT_EQ(listener.Count, 0); + + // The grace period only suppresses artifacts owned by this import. Other + // project edits retain the normal watcher behavior. + ZEngine::Core::VFS::VFSContextTestAccess::Deliver(context, MakeEvent("/project/Assets/Meshes/other.zemesh")); + EXPECT_EQ(listener.Count, 1); + } + + manager.Shutdown(); +}