diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d3b658..9931d03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,10 @@ set(libp2p_INCLUDE_DIR "${_THIRDPARTY_BUILD_DIR}/libp2p/include") #add_subdirectory(generated) add_subdirectory(src) +if(BUILD_TESTING) + add_subdirectory(test) +endif() +add_subdirectory(tools) install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SGProcessingManager" FILES_MATCHING PATTERN "*.h*") install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/generated/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SGProcessingManager/generated" FILES_MATCHING PATTERN "*.h*") diff --git a/cmake/CommonBuildParameters.cmake b/cmake/CommonBuildParameters.cmake index c82ef98..55444eb 100644 --- a/cmake/CommonBuildParameters.cmake +++ b/cmake/CommonBuildParameters.cmake @@ -7,7 +7,14 @@ set(BOOST_PATCH_VERSION "0" CACHE STRING "Boost Patch Version") set(BOOST_VERSION "${BOOST_MAJOR_VERSION}.${BOOST_MINOR_VERSION}.${BOOST_PATCH_VERSION}") set(BOOST_VERSION_2U "${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION}") -set(CMAKE_CXX_STANDARD 20) +# Default to C++20 only when the including project has not already chosen a +# standard. The standalone build (build/CommonCompilerOptions.cmake) sets +# C++17 before including this file, matching SuperGenius and all other +# projects; forcing 20 here breaks fmt/spdlog consteval format-string checks +# on newer clang (e.g. SPDLOG_LOGGER_CATCH in spdlog/logger.h). +if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 20) +endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -32,15 +39,81 @@ set(OPENSSL_INCLUDE_DIR "${_THIRDPARTY_BUILD_DIR}/openssl/build/include" CACHE P find_package(OpenSSL REQUIRED CONFIG) +# VulkanHeaders +set(VulkanHeaders_DIR "${_THIRDPARTY_BUILD_DIR}/Vulkan-Headers/share/cmake/VulkanHeaders" CACHE PATH "Path to Vulkan-Headers install folder") +find_package(VulkanHeaders CONFIG REQUIRED) + # Vulkan -find_package(Vulkan) +# +# On macOS, create the Vulkan::Vulkan target manually pointing at the MoltenVK +# dylib nested inside the thirdparty-built MoltenVK.xcframework. MoltenVK is a +# complete Vulkan implementation that exports the full loader API — it can be +# used directly without any ICD plumbing, exactly as MNN's Vulkan backend does. +# +# On other platforms, use the vendored Khronos Vulkan-Loader found via the +# standard find_package(Vulkan) / VULKAN_SDK mechanism. +if(APPLE) + if(NOT TARGET Vulkan::Vulkan) + set(_MVK_LIB "${_THIRDPARTY_BUILD_DIR}/MoltenVK/build/lib/MoltenVK.xcframework/macos-arm64_x86_64/libMoltenVK.a") + add_library(Vulkan::Vulkan STATIC IMPORTED GLOBAL) + set_target_properties(Vulkan::Vulkan PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_THIRDPARTY_BUILD_DIR}/Vulkan-Headers/include" + IMPORTED_LOCATION "${_MVK_LIB}" + ) + # Frameworks MoltenVK links against; inherited by every consumer of Vulkan::Vulkan. + target_link_libraries(Vulkan::Vulkan INTERFACE + "-framework Metal" + "-framework IOSurface" + "-framework QuartzCore" + "-framework Foundation" + "-framework CoreFoundation" + "-framework CoreGraphics" + "-framework IOKit" + "-framework AppKit" + ) + endif() +else() + find_package(Vulkan) + + if(NOT TARGET Vulkan::Vulkan) + set(Vulkan_INCLUDE_DIR "${_THIRDPARTY_BUILD_DIR}/Vulkan-Headers/include") + if(NOT DEFINED ENV{VULKAN_SDK}) + set(ENV{VULKAN_SDK} "${_THIRDPARTY_BUILD_DIR}/Vulkan-Loader") + endif() -if(NOT TARGET Vulkan::Vulkan) - if(NOT DEFINED $ENV{VULKAN_SDK}) - set(ENV{VULKAN_SDK} "${_THIRDPARTY_BUILD_DIR}/Vulkan-Loader") + find_package(Vulkan REQUIRED) endif() - find_package(Vulkan REQUIRED) + # Override Vulkan::Vulkan to use our vendored Vulkan-Headers on all non-Apple + # platforms. vk-bootstrap was built against our headers (v1.4); mixing with + # system/NDK headers (v1.3 or other versions) causes unknown-type errors in + # VkBootstrapDispatch.h and VkBootstrapFeatureChain.h. + set_target_properties(Vulkan::Vulkan PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_THIRDPARTY_BUILD_DIR}/Vulkan-Headers/include" + ) +endif() + +# vk-bootstrap +set(vk-bootstrap_DIR "${_THIRDPARTY_BUILD_DIR}/vk-bootstrap/lib/cmake/vk-bootstrap") +find_package(vk-bootstrap CONFIG REQUIRED) + +# SPIRV-Tools — no longer a standalone build. libshaderc_combined (linked via +# shaderc::shaderc below) statically bundles the exact same SPIRV-Tools code at the +# exact same pinned commit (v2024.3 DEPS). The spirv-tools include path is folded into +# shaderc::shaderc's INTERFACE_INCLUDE_DIRECTORIES so resolves. + +# shaderc — installs no CMake package config (confirmed in 02-02-RESEARCH.md against +# github.com/google/shaderc/issues/1369 and github.com/microsoft/vcpkg/issues/23208); hand-written +# IMPORTED target required, mirroring thirdparty/build/CommonTargets.cmake's own target. +# libshaderc_combined statically bundles glslang+SPIRV-Tools and installs spirv-tools +# headers into /include/spirv-tools/, so resolves +# for consumers that call spvtools::SpirvTools::Validate() directly (SHADER-02). +if(NOT TARGET shaderc::shaderc) + add_library(shaderc::shaderc STATIC IMPORTED GLOBAL) + set_target_properties(shaderc::shaderc PROPERTIES + IMPORTED_LOCATION "${_THIRDPARTY_BUILD_DIR}/shaderc/lib/${CMAKE_STATIC_LIBRARY_PREFIX}shaderc_combined${CMAKE_STATIC_LIBRARY_SUFFIX}" + INTERFACE_INCLUDE_DIRECTORIES "${_THIRDPARTY_BUILD_DIR}/shaderc/include" + ) endif() # for compression, we need snappy @@ -181,6 +254,10 @@ find_package(libp2p CONFIG REQUIRED) set(ipfs-lite-cpp_DIR "${_THIRDPARTY_BUILD_DIR}/ipfs-lite-cpp/lib/cmake/ipfs-lite-cpp") find_package(ipfs-lite-cpp CONFIG REQUIRED) +# ipfs-bitswap-cpp +set(ipfs-bitswap-cpp_DIR "${_THIRDPARTY_BUILD_DIR}/ipfs-bitswap-cpp/lib/cmake/ipfs-bitswap-cpp") +find_package(ipfs-bitswap-cpp CONFIG REQUIRED) + # MNN set(MNN_DIR "${_THIRDPARTY_BUILD_DIR}/MNN/lib/cmake/MNN") find_package(MNN CONFIG REQUIRED) @@ -195,6 +272,31 @@ elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") get_target_property(MNN_LIB_PATH MNN::MNN IMPORTED_LOCATION_RELWITHDEBINFO) endif() +# zlib +set(ZLIB_ROOT "${_THIRDPARTY_BUILD_DIR}/zlib") + +# Prefer package config files while loading Libssh2's dependencies. +# Libssh2 config calls `find_dependency(ZLIB)` without `CONFIG`, which can +# otherwise resolve to CMake's FindZLIB module on Windows CI. +set(_SGNS_CMAKE_FIND_PACKAGE_PREFER_CONFIG_WAS_DEFINED FALSE) +if(DEFINED CMAKE_FIND_PACKAGE_PREFER_CONFIG) + set(_SGNS_CMAKE_FIND_PACKAGE_PREFER_CONFIG_WAS_DEFINED TRUE) + set(_SGNS_CMAKE_FIND_PACKAGE_PREFER_CONFIG_PREV "${CMAKE_FIND_PACKAGE_PREFER_CONFIG}") +endif() +set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON) + +# libssh2 +set(Libssh2_DIR "${_THIRDPARTY_BUILD_DIR}/libssh2/lib/cmake/libssh2") +find_package(Libssh2 CONFIG REQUIRED) + +if(_SGNS_CMAKE_FIND_PACKAGE_PREFER_CONFIG_WAS_DEFINED) + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG "${_SGNS_CMAKE_FIND_PACKAGE_PREFER_CONFIG_PREV}") +else() + unset(CMAKE_FIND_PACKAGE_PREFER_CONFIG) +endif() +unset(_SGNS_CMAKE_FIND_PACKAGE_PREFER_CONFIG_PREV) +unset(_SGNS_CMAKE_FIND_PACKAGE_PREFER_CONFIG_WAS_DEFINED) + # AsyncioManager set(AsyncIOManager_INCLUDE_DIR "${_THIRDPARTY_BUILD_DIR}/AsyncIOManager/include") set(AsyncIOManager_DIR "${_THIRDPARTY_BUILD_DIR}/AsyncIOManager/lib/cmake/AsyncIOManager") @@ -206,20 +308,8 @@ include_directories( ) add_subdirectory(${PROJECT_ROOT}/src ${CMAKE_BINARY_DIR}/src) - -# if(BUILD_TESTS) - # add_executable(${PROJECT_NAME}_test - # "${CMAKE_CURRENT_LIST_DIR}/../test/main_test.cpp" - # "${CMAKE_CURRENT_LIST_DIR}/../test/BitcoinKeyGenerator_test.cpp" - # "${CMAKE_CURRENT_LIST_DIR}/../test/EthereumKeyGenerator_test.cpp" - # "${CMAKE_CURRENT_LIST_DIR}/../test/ElGamalKeyGenerator_test.cpp" - # "${CMAKE_CURRENT_LIST_DIR}/../test/ECElGamalKeyGenerator_test.cpp" - # "${CMAKE_CURRENT_LIST_DIR}/../test/TransactionVerifierCircuit_test.cpp" - # "${CMAKE_CURRENT_LIST_DIR}/../test/MPCVerifierCircuit_test.cpp" - # "${CMAKE_CURRENT_LIST_DIR}/../test/Requestor.cpp" - # ) - # target_link_libraries(${PROJECT_NAME}_test PUBLIC ${PROJECT_NAME} SGProofCircuits GTest::gtest Boost::random) -# endif() +add_subdirectory(${PROJECT_ROOT}/tools ${CMAKE_BINARY_DIR}/tools) +add_subdirectory(${PROJECT_ROOT}/test ${CMAKE_BINARY_DIR}/test) # Install Headers install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SGProcessingManager" FILES_MATCHING PATTERN "*.h*") diff --git a/doc/vulkan-validation-layers-decision.md b/doc/vulkan-validation-layers-decision.md new file mode 100644 index 0000000..273f77d --- /dev/null +++ b/doc/vulkan-validation-layers-decision.md @@ -0,0 +1,21 @@ +# Vulkan Validation Layers Deferral Decision + +## Decision + +Vulkan-ValidationLayers is explicitly **not vendored** and **not wired** into debug or CI builds for v1.0 of the render pipeline. No validation-layer extension or callback is requested at `vkCreateInstance` time in any code path this phase delivers. + +## Rationale + +CTX-04 scoped validation-layer vendoring out of Phase 1 to keep the net-new vendoring surface to `vk-bootstrap` only. Validation layers are a development and debugging aid — they validate correct API usage, not runtime-correctness requirements — and this phase's deliverable (CTX-01..03, DISP-01..03) does not depend on them: + +- A headless Vulkan context can be created, tested for coexistence safety, and dispatched to without validation layers present. +- This project's CI has zero macOS Vulkan signal today regardless of validation-layer presence (a pre-existing gap tracked for Phase 4). +- Adding validation layers would introduce an additional vendoring step (`thirdparty/Vulkan-ValidationLayers`, its own submodule + ExternalProject_Add build wiring) plus platform-specific library path resolution at runtime (the loader must find `VK_LAYER_KHRONOS_validation` on disk, and the path differs across Windows/Linux/macOS/MoltenVK) — all for a tooling aid that has no effect on the shipped code path. + +## Deferred To + +Tracked as v2 requirement **VALLAYER-01** in `.planning/workstreams/sgproc-render/REQUIREMENTS.md` ("Vulkan-ValidationLayers wired into debug/CI builds"), to be implemented in v1.x. + +## Future Hook Point + +If implemented later, validation layers can be enabled via `vkb::InstanceBuilder::request_validation_layers(true)` (vk-bootstrap has first-class support for this). The toggle would be gated behind a new CMake debug-only option — analogous to the existing `SANITIZE_CODE` debug-only toggle pattern already present in `CommonCompilerOptions.cmake` — so it never activates in release/optimized builds. diff --git a/generated/ColorFormat.hpp b/generated/ColorFormat.hpp new file mode 100644 index 0000000..baad276 --- /dev/null +++ b/generated/ColorFormat.hpp @@ -0,0 +1,27 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// ColorFormat.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + /** + * Color attachment format + */ + + using nlohmann::json; + + /** + * Color attachment format + */ + enum class ColorFormat : int { RGB8, RGBA8 }; +} diff --git a/generated/ShaderType.hpp b/generated/CullMode.hpp similarity index 72% rename from generated/ShaderType.hpp rename to generated/CullMode.hpp index e70865a..51be291 100644 --- a/generated/ShaderType.hpp +++ b/generated/CullMode.hpp @@ -5,7 +5,7 @@ // // Then include this file, and then do // -// ShaderType.hpp data = nlohmann::json::parse(jsonString); +// CullMode.hpp data = nlohmann::json::parse(jsonString); #pragma once @@ -16,5 +16,5 @@ namespace sgns { using nlohmann::json; - enum class ShaderType : int { GLSL, HLSL, METAL, SPIRV }; + enum class CullMode : int { BACK, FRONT, NONE }; } diff --git a/generated/DepthFormat.hpp b/generated/DepthFormat.hpp new file mode 100644 index 0000000..f5f391f --- /dev/null +++ b/generated/DepthFormat.hpp @@ -0,0 +1,27 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// DepthFormat.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + /** + * Depth attachment format + */ + + using nlohmann::json; + + /** + * Depth attachment format + */ + enum class DepthFormat : int { D24_UNORM_S8_UINT, D32_SFLOAT }; +} diff --git a/generated/DepthTest.hpp b/generated/DepthTest.hpp new file mode 100644 index 0000000..53d7d32 --- /dev/null +++ b/generated/DepthTest.hpp @@ -0,0 +1,20 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// DepthTest.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + using nlohmann::json; + + enum class DepthTest : int { DISABLED, ENABLED }; +} diff --git a/generated/FrontFace.hpp b/generated/FrontFace.hpp new file mode 100644 index 0000000..a777200 --- /dev/null +++ b/generated/FrontFace.hpp @@ -0,0 +1,20 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// FrontFace.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + using nlohmann::json; + + enum class FrontFace : int { CCW, CW }; +} diff --git a/generated/Generators.hpp b/generated/Generators.hpp index 4c45a93..ad20ae5 100644 --- a/generated/Generators.hpp +++ b/generated/Generators.hpp @@ -15,10 +15,25 @@ #include "SgnsProcessing.hpp" #include "Pass.hpp" +#include "VertexLayoutEntry.hpp" +#include "VertexLayoutFormat.hpp" +#include "VertexBuffer.hpp" #include "PassType.hpp" #include "ShaderConfig.hpp" -#include "Uniform.hpp" -#include "ShaderType.hpp" +#include "ShaderUniform.hpp" +#include "RenderTarget.hpp" +#include "DepthFormat.hpp" +#include "ColorFormat.hpp" +#include "RenderShaderConfig.hpp" +#include "RenderShaderUniform.hpp" +#include "ShaderStage.hpp" +#include "ShaderSourceType.hpp" +#include "Stage.hpp" +#include "PipelineState.hpp" +#include "Topology.hpp" +#include "FrontFace.hpp" +#include "DepthTest.hpp" +#include "CullMode.hpp" #include "ModelConfig.hpp" #include "OptimizerConfig.hpp" #include "OptimizerType.hpp" @@ -26,6 +41,8 @@ #include "ModelNode.hpp" #include "ModelFormat.hpp" #include "PassIoBinding.hpp" +#include "IndexBuffer.hpp" +#include "IndexType.hpp" #include "DataTransform.hpp" #include "DataTransformType.hpp" #include "Params.hpp" @@ -56,6 +73,9 @@ namespace sgns { void from_json(const json & j, DataTransform & x); void to_json(json & j, const DataTransform & x); + void from_json(const json & j, IndexBuffer & x); + void to_json(json & j, const IndexBuffer & x); + void from_json(const json & j, PassIoBinding & x); void to_json(json & j, const PassIoBinding & x); @@ -68,12 +88,33 @@ namespace sgns { void from_json(const json & j, ModelConfig & x); void to_json(json & j, const ModelConfig & x); - void from_json(const json & j, Uniform & x); - void to_json(json & j, const Uniform & x); + void from_json(const json & j, PipelineState & x); + void to_json(json & j, const PipelineState & x); + + void from_json(const json & j, ShaderStage & x); + void to_json(json & j, const ShaderStage & x); + + void from_json(const json & j, RenderShaderUniform & x); + void to_json(json & j, const RenderShaderUniform & x); + + void from_json(const json & j, RenderShaderConfig & x); + void to_json(json & j, const RenderShaderConfig & x); + + void from_json(const json & j, RenderTarget & x); + void to_json(json & j, const RenderTarget & x); + + void from_json(const json & j, ShaderUniform & x); + void to_json(json & j, const ShaderUniform & x); void from_json(const json & j, ShaderConfig & x); void to_json(json & j, const ShaderConfig & x); + void from_json(const json & j, VertexBuffer & x); + void to_json(json & j, const VertexBuffer & x); + + void from_json(const json & j, VertexLayoutEntry & x); + void to_json(json & j, const VertexLayoutEntry & x); + void from_json(const json & j, Pass & x); void to_json(json & j, const Pass & x); @@ -92,6 +133,9 @@ namespace sgns { void from_json(const json & j, DataTransformType & x); void to_json(json & j, const DataTransformType & x); + void from_json(const json & j, IndexType & x); + void to_json(json & j, const IndexType & x); + void from_json(const json & j, ModelFormat & x); void to_json(json & j, const ModelFormat & x); @@ -101,12 +145,36 @@ namespace sgns { void from_json(const json & j, OptimizerType & x); void to_json(json & j, const OptimizerType & x); - void from_json(const json & j, ShaderType & x); - void to_json(json & j, const ShaderType & x); + void from_json(const json & j, CullMode & x); + void to_json(json & j, const CullMode & x); + + void from_json(const json & j, DepthTest & x); + void to_json(json & j, const DepthTest & x); + + void from_json(const json & j, FrontFace & x); + void to_json(json & j, const FrontFace & x); + + void from_json(const json & j, Topology & x); + void to_json(json & j, const Topology & x); + + void from_json(const json & j, Stage & x); + void to_json(json & j, const Stage & x); + + void from_json(const json & j, ShaderSourceType & x); + void to_json(json & j, const ShaderSourceType & x); + + void from_json(const json & j, ColorFormat & x); + void to_json(json & j, const ColorFormat & x); + + void from_json(const json & j, DepthFormat & x); + void to_json(json & j, const DepthFormat & x); void from_json(const json & j, PassType & x); void to_json(json & j, const PassType & x); + void from_json(const json & j, VertexLayoutFormat & x); + void to_json(json & j, const VertexLayoutFormat & x); + inline void from_json(const json & j, Dimensions& x) { x.set_batch(get_stack_optional(j, "batch")); x.set_block_len(get_stack_optional(j, "block_len")); @@ -229,6 +297,17 @@ namespace sgns { j["type"] = x.get_type(); } + inline void from_json(const json & j, IndexBuffer& x) { + x.set_index_type(get_stack_optional(j, "index_type")); + x.set_source(get_stack_optional(j, "source")); + } + + inline void to_json(json & j, const IndexBuffer & x) { + j = json::object(); + j["index_type"] = x.get_index_type(); + j["source"] = x.get_source(); + } + inline void from_json(const json & j, PassIoBinding& x) { x.set_name(j.at("name").get()); x.set_source(get_stack_optional(j, "source")); @@ -301,13 +380,86 @@ namespace sgns { j["source_uri_param"] = x.get_source_uri_param(); } - inline void from_json(const json & j, Uniform& x) { + inline void from_json(const json & j, PipelineState& x) { + x.set_cull_mode(get_stack_optional(j, "cull_mode")); + x.set_depth_test(get_stack_optional(j, "depth_test")); + x.set_front_face(get_stack_optional(j, "front_face")); + x.set_topology(get_stack_optional(j, "topology")); + } + + inline void to_json(json & j, const PipelineState & x) { + j = json::object(); + j["cull_mode"] = x.get_cull_mode(); + j["depth_test"] = x.get_depth_test(); + j["front_face"] = x.get_front_face(); + j["topology"] = x.get_topology(); + } + + inline void from_json(const json & j, ShaderStage& x) { + x.set_entry_point(get_stack_optional(j, "entry_point")); + x.set_source(j.at("source").get()); + x.set_stage(j.at("stage").get()); + x.set_type(j.at("type").get()); + } + + inline void to_json(json & j, const ShaderStage & x) { + j = json::object(); + j["entry_point"] = x.get_entry_point(); + j["source"] = x.get_source(); + j["stage"] = x.get_stage(); + j["type"] = x.get_type(); + } + + inline void from_json(const json & j, RenderShaderUniform& x) { x.set_source(get_stack_optional(j, "source")); x.set_type(get_stack_optional(j, "type")); x.set_value(get_untyped(j, "value")); } - inline void to_json(json & j, const Uniform & x) { + inline void to_json(json & j, const RenderShaderUniform & x) { + j = json::object(); + j["source"] = x.get_source(); + j["type"] = x.get_type(); + j["value"] = x.get_value(); + } + + inline void from_json(const json & j, RenderShaderConfig& x) { + x.set_stages(j.at("stages").get>()); + x.set_uniforms(get_stack_optional>(j, "uniforms")); + } + + inline void to_json(json & j, const RenderShaderConfig & x) { + j = json::object(); + j["stages"] = x.get_stages(); + j["uniforms"] = x.get_uniforms(); + } + + inline void from_json(const json & j, RenderTarget& x) { + x.set_clear_color(j.at("clear_color").get>()); + x.set_clear_depth(j.at("clear_depth").get()); + x.set_color_format(j.at("color_format").get()); + x.set_depth_format(j.at("depth_format").get()); + x.set_height(j.at("height").get()); + x.set_width(j.at("width").get()); + } + + inline void to_json(json & j, const RenderTarget & x) { + j = json::object(); + j["clear_color"] = x.get_clear_color(); + j["clear_depth"] = x.get_clear_depth(); + j["color_format"] = x.get_color_format(); + j["depth_format"] = x.get_depth_format(); + j["height"] = x.get_height(); + j["width"] = x.get_width(); + } + + inline void from_json(const json & j, ShaderUniform& x) { + x.set_source(get_stack_optional(j, "source")); + x.set_type(get_stack_optional(j, "type")); + x.set_value(get_untyped(j, "value")); + } + + inline void to_json(json & j, const ShaderUniform & x) { j = json::object(); j["source"] = x.get_source(); j["type"] = x.get_type(); @@ -317,8 +469,8 @@ namespace sgns { inline void from_json(const json & j, ShaderConfig& x) { x.set_entry_point(get_stack_optional(j, "entry_point")); x.set_source(j.at("source").get()); - x.set_type(get_stack_optional(j, "type")); - x.set_uniforms(get_stack_optional>(j, "uniforms")); + x.set_type(get_stack_optional(j, "type")); + x.set_uniforms(get_stack_optional>(j, "uniforms")); } inline void to_json(json & j, const ShaderConfig & x) { @@ -329,16 +481,44 @@ namespace sgns { j["uniforms"] = x.get_uniforms(); } + inline void from_json(const json & j, VertexBuffer& x) { + x.set_source(j.at("source").get()); + } + + inline void to_json(json & j, const VertexBuffer & x) { + j = json::object(); + j["source"] = x.get_source(); + } + + inline void from_json(const json & j, VertexLayoutEntry& x) { + x.set_format(j.at("format").get()); + x.set_name(j.at("name").get()); + x.set_offset(j.at("offset").get()); + } + + inline void to_json(json & j, const VertexLayoutEntry & x) { + j = json::object(); + j["format"] = x.get_format(); + j["name"] = x.get_name(); + j["offset"] = x.get_offset(); + } + inline void from_json(const json & j, Pass& x) { x.set_data_transforms(get_stack_optional>(j, "data_transforms")); x.set_description(get_stack_optional(j, "description")); x.set_enabled(get_stack_optional(j, "enabled")); + x.set_index_buffer(get_stack_optional(j, "index_buffer")); x.set_inputs(get_stack_optional>(j, "inputs")); x.set_model(get_stack_optional(j, "model")); x.set_name(j.at("name").get()); x.set_outputs(get_stack_optional>(j, "outputs")); + x.set_pipeline_state(get_stack_optional(j, "pipeline_state")); + x.set_render_shader(get_stack_optional(j, "render_shader")); + x.set_render_target(get_stack_optional(j, "render_target")); x.set_shader(get_stack_optional(j, "shader")); x.set_type(j.at("type").get()); + x.set_vertex_buffer(get_stack_optional(j, "vertex_buffer")); + x.set_vertex_layout(get_stack_optional>(j, "vertex_layout")); } inline void to_json(json & j, const Pass & x) { @@ -346,12 +526,18 @@ namespace sgns { j["data_transforms"] = x.get_data_transforms(); j["description"] = x.get_description(); j["enabled"] = x.get_enabled(); + j["index_buffer"] = x.get_index_buffer(); j["inputs"] = x.get_inputs(); j["model"] = x.get_model(); j["name"] = x.get_name(); j["outputs"] = x.get_outputs(); + j["pipeline_state"] = x.get_pipeline_state(); + j["render_shader"] = x.get_render_shader(); + j["render_target"] = x.get_render_target(); j["shader"] = x.get_shader(); j["type"] = x.get_type(); + j["vertex_buffer"] = x.get_vertex_buffer(); + j["vertex_layout"] = x.get_vertex_layout(); } inline void from_json(const json & j, SgnsProcessing& x) { @@ -514,6 +700,20 @@ namespace sgns { } } + inline void from_json(const json & j, IndexType & x) { + if (j == "uint16") x = IndexType::UINT16; + else if (j == "uint32") x = IndexType::UINT32; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const IndexType & x) { + switch (x) { + case IndexType::UINT16: j = "uint16"; break; + case IndexType::UINT32: j = "uint32"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"IndexType\": " + std::to_string(static_cast(x))); + } + } + inline void from_json(const json & j, ModelFormat & x) { if (j == "MNN") x = ModelFormat::MNN; else if (j == "ONNX") x = ModelFormat::ONNX; @@ -576,21 +776,119 @@ namespace sgns { } } - inline void from_json(const json & j, ShaderType & x) { - if (j == "glsl") x = ShaderType::GLSL; - else if (j == "hlsl") x = ShaderType::HLSL; - else if (j == "metal") x = ShaderType::METAL; - else if (j == "spirv") x = ShaderType::SPIRV; + inline void from_json(const json & j, CullMode & x) { + if (j == "back") x = CullMode::BACK; + else if (j == "front") x = CullMode::FRONT; + else if (j == "none") x = CullMode::NONE; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const CullMode & x) { + switch (x) { + case CullMode::BACK: j = "back"; break; + case CullMode::FRONT: j = "front"; break; + case CullMode::NONE: j = "none"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"CullMode\": " + std::to_string(static_cast(x))); + } + } + + inline void from_json(const json & j, DepthTest & x) { + if (j == "disabled") x = DepthTest::DISABLED; + else if (j == "enabled") x = DepthTest::ENABLED; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const DepthTest & x) { + switch (x) { + case DepthTest::DISABLED: j = "disabled"; break; + case DepthTest::ENABLED: j = "enabled"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"DepthTest\": " + std::to_string(static_cast(x))); + } + } + + inline void from_json(const json & j, FrontFace & x) { + if (j == "ccw") x = FrontFace::CCW; + else if (j == "cw") x = FrontFace::CW; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const FrontFace & x) { + switch (x) { + case FrontFace::CCW: j = "ccw"; break; + case FrontFace::CW: j = "cw"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"FrontFace\": " + std::to_string(static_cast(x))); + } + } + + inline void from_json(const json & j, Topology & x) { + if (j == "line_list") x = Topology::LINE_LIST; + else if (j == "point_list") x = Topology::POINT_LIST; + else if (j == "triangle_list") x = Topology::TRIANGLE_LIST; else { throw std::runtime_error("Input JSON does not conform to schema!"); } } - inline void to_json(json & j, const ShaderType & x) { + inline void to_json(json & j, const Topology & x) { switch (x) { - case ShaderType::GLSL: j = "glsl"; break; - case ShaderType::HLSL: j = "hlsl"; break; - case ShaderType::METAL: j = "metal"; break; - case ShaderType::SPIRV: j = "spirv"; break; - default: throw std::runtime_error("Unexpected value in enumeration \"ShaderType\": " + std::to_string(static_cast(x))); + case Topology::LINE_LIST: j = "line_list"; break; + case Topology::POINT_LIST: j = "point_list"; break; + case Topology::TRIANGLE_LIST: j = "triangle_list"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"Topology\": " + std::to_string(static_cast(x))); + } + } + + inline void from_json(const json & j, Stage & x) { + if (j == "fragment") x = Stage::FRAGMENT; + else if (j == "vertex") x = Stage::VERTEX; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const Stage & x) { + switch (x) { + case Stage::FRAGMENT: j = "fragment"; break; + case Stage::VERTEX: j = "vertex"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"Stage\": " + std::to_string(static_cast(x))); + } + } + + inline void from_json(const json & j, ShaderSourceType & x) { + if (j == "glsl") x = ShaderSourceType::GLSL; + else if (j == "spirv") x = ShaderSourceType::SPIRV; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const ShaderSourceType & x) { + switch (x) { + case ShaderSourceType::GLSL: j = "glsl"; break; + case ShaderSourceType::SPIRV: j = "spirv"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"ShaderSourceType\": " + std::to_string(static_cast(x))); + } + } + + inline void from_json(const json & j, ColorFormat & x) { + if (j == "RGB8") x = ColorFormat::RGB8; + else if (j == "RGBA8") x = ColorFormat::RGBA8; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const ColorFormat & x) { + switch (x) { + case ColorFormat::RGB8: j = "RGB8"; break; + case ColorFormat::RGBA8: j = "RGBA8"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"ColorFormat\": " + std::to_string(static_cast(x))); + } + } + + inline void from_json(const json & j, DepthFormat & x) { + if (j == "D24_UNORM_S8_UINT") x = DepthFormat::D24_UNORM_S8_UINT; + else if (j == "D32_SFLOAT") x = DepthFormat::D32_SFLOAT; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const DepthFormat & x) { + switch (x) { + case DepthFormat::D24_UNORM_S8_UINT: j = "D24_UNORM_S8_UINT"; break; + case DepthFormat::D32_SFLOAT: j = "D32_SFLOAT"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"DepthFormat\": " + std::to_string(static_cast(x))); } } @@ -613,4 +911,20 @@ namespace sgns { default: throw std::runtime_error("Unexpected value in enumeration \"PassType\": " + std::to_string(static_cast(x))); } } + + inline void from_json(const json & j, VertexLayoutFormat & x) { + if (j == "FLOAT16") x = VertexLayoutFormat::FLOAT16; + else if (j == "FLOAT32") x = VertexLayoutFormat::FLOAT32; + else if (j == "INT32") x = VertexLayoutFormat::INT32; + else { throw std::runtime_error("Input JSON does not conform to schema!"); } + } + + inline void to_json(json & j, const VertexLayoutFormat & x) { + switch (x) { + case VertexLayoutFormat::FLOAT16: j = "FLOAT16"; break; + case VertexLayoutFormat::FLOAT32: j = "FLOAT32"; break; + case VertexLayoutFormat::INT32: j = "INT32"; break; + default: throw std::runtime_error("Unexpected value in enumeration \"VertexLayoutFormat\": " + std::to_string(static_cast(x))); + } + } } diff --git a/generated/IndexBuffer.hpp b/generated/IndexBuffer.hpp new file mode 100644 index 0000000..9bc3467 --- /dev/null +++ b/generated/IndexBuffer.hpp @@ -0,0 +1,56 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// IndexBuffer.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + enum class IndexType : int; +} + +namespace sgns { + /** + * Index buffer binding + index type for render passes + * + * Index buffer binding for render passes; index_type is schema-configurable per D-17 + */ + + using nlohmann::json; + + /** + * Index buffer binding + index type for render passes + * + * Index buffer binding for render passes; index_type is schema-configurable per D-17 + */ + class IndexBuffer { + public: + IndexBuffer() : + source_constraint(boost::none, boost::none, boost::none, boost::none, boost::none, boost::none, std::string("^(input|output|internal|parameter):[a-zA-Z][a-zA-Z0-9_]*$")) + {} + virtual ~IndexBuffer() = default; + + private: + boost::optional index_type; + boost::optional source; + ClassMemberConstraints source_constraint; + + public: + boost::optional get_index_type() const { return index_type; } + void set_index_type(boost::optional value) { this->index_type = value; } + + /** + * Data source using prefix notation + */ + boost::optional get_source() const { return source; } + void set_source(boost::optional value) { if (value) CheckConstraint("source", source_constraint, *value); this->source = value; } + }; +} diff --git a/generated/IndexType.hpp b/generated/IndexType.hpp new file mode 100644 index 0000000..a5b7daa --- /dev/null +++ b/generated/IndexType.hpp @@ -0,0 +1,20 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// IndexType.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + using nlohmann::json; + + enum class IndexType : int { UINT16, UINT32 }; +} diff --git a/generated/Pass.hpp b/generated/Pass.hpp index 7e4bc2e..96139d0 100644 --- a/generated/Pass.hpp +++ b/generated/Pass.hpp @@ -14,9 +14,15 @@ #include "helper.hpp" #include "DataTransform.hpp" +#include "IndexBuffer.hpp" #include "PassIoBinding.hpp" #include "ModelConfig.hpp" +#include "PipelineState.hpp" +#include "RenderShaderConfig.hpp" +#include "RenderTarget.hpp" #include "ShaderConfig.hpp" +#include "VertexBuffer.hpp" +#include "VertexLayoutEntry.hpp" namespace sgns { enum class PassType : int; @@ -36,13 +42,22 @@ namespace sgns { boost::optional> data_transforms; boost::optional description; boost::optional enabled; + boost::optional index_buffer; boost::optional> inputs; boost::optional model; std::string name; ClassMemberConstraints name_constraint; boost::optional> outputs; + boost::optional estimated_gpu_memory_bytes; + boost::optional max_output_artifact_bytes; + boost::optional per_pass_deadline_ms; + boost::optional pipeline_state; + boost::optional render_shader; + boost::optional render_target; boost::optional shader; PassType type; + boost::optional vertex_buffer; + boost::optional> vertex_layout; public: /** @@ -60,6 +75,12 @@ namespace sgns { boost::optional get_enabled() const { return enabled; } void set_enabled(boost::optional value) { this->enabled = value; } + /** + * Index buffer binding + index type for render passes + */ + boost::optional get_index_buffer() const { return index_buffer; } + void set_index_buffer(boost::optional value) { this->index_buffer = value; } + /** * Input bindings for non-model passes */ @@ -86,7 +107,43 @@ namespace sgns { void set_outputs(boost::optional> value) { this->outputs = value; } /** - * Shader configuration for compute/render passes + * Estimated GPU memory needed for this pass in bytes. 0 means no estimate provided. + */ + boost::optional get_estimated_gpu_memory_bytes() const { return estimated_gpu_memory_bytes; } + void set_estimated_gpu_memory_bytes(boost::optional value) { this->estimated_gpu_memory_bytes = value; } + + /** + * Maximum output artifact size in bytes before the pass is considered budget-exceeded. 0 means no budget. + */ + boost::optional get_max_output_artifact_bytes() const { return max_output_artifact_bytes; } + void set_max_output_artifact_bytes(boost::optional value) { this->max_output_artifact_bytes = value; } + + /** + * Per-pass wall-clock deadline in milliseconds. 0 means no deadline. + */ + boost::optional get_per_pass_deadline_ms() const { return per_pass_deadline_ms; } + void set_per_pass_deadline_ms(boost::optional value) { this->per_pass_deadline_ms = value; } + + /** + * Fixed-function pipeline state for render passes + */ + boost::optional get_pipeline_state() const { return pipeline_state; } + void set_pipeline_state(boost::optional value) { this->pipeline_state = value; } + + /** + * Multi-stage (vertex+fragment) shader configuration for render passes + */ + boost::optional get_render_shader() const { return render_shader; } + void set_render_shader(boost::optional value) { this->render_shader = value; } + + /** + * Offscreen framebuffer (color+depth) config for render passes + */ + boost::optional get_render_target() const { return render_target; } + void set_render_target(boost::optional value) { this->render_target = value; } + + /** + * Shader configuration for compute passes */ boost::optional get_shader() const { return shader; } void set_shader(boost::optional value) { this->shader = value; } @@ -97,5 +154,18 @@ namespace sgns { const PassType & get_type() const { return type; } PassType & get_mutable_type() { return type; } void set_type(const PassType & value) { this->type = value; } + + /** + * Buffer binding supplying vertex attribute data referenced by vertex_layout (D-16 + * Amendment) + */ + boost::optional get_vertex_buffer() const { return vertex_buffer; } + void set_vertex_buffer(boost::optional value) { this->vertex_buffer = value; } + + /** + * Vertex attribute layout for render passes + */ + boost::optional> get_vertex_layout() const { return vertex_layout; } + void set_vertex_layout(boost::optional> value) { this->vertex_layout = value; } }; } diff --git a/generated/PipelineState.hpp b/generated/PipelineState.hpp new file mode 100644 index 0000000..a84a3ee --- /dev/null +++ b/generated/PipelineState.hpp @@ -0,0 +1,63 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// PipelineState.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + enum class CullMode : int; + enum class DepthTest : int; + enum class FrontFace : int; + enum class Topology : int; +} + +namespace sgns { + /** + * Fixed-function pipeline state for render passes + * + * Curated, minimal v1 fixed-function pipeline state subset (D-13); depth compare op is + * fixed at 'less', not schema-configurable (D-14) + */ + + using nlohmann::json; + + /** + * Fixed-function pipeline state for render passes + * + * Curated, minimal v1 fixed-function pipeline state subset (D-13); depth compare op is + * fixed at 'less', not schema-configurable (D-14) + */ + class PipelineState { + public: + PipelineState() = default; + virtual ~PipelineState() = default; + + private: + boost::optional cull_mode; + boost::optional depth_test; + boost::optional front_face; + boost::optional topology; + + public: + boost::optional get_cull_mode() const { return cull_mode; } + void set_cull_mode(boost::optional value) { this->cull_mode = value; } + + boost::optional get_depth_test() const { return depth_test; } + void set_depth_test(boost::optional value) { this->depth_test = value; } + + boost::optional get_front_face() const { return front_face; } + void set_front_face(boost::optional value) { this->front_face = value; } + + boost::optional get_topology() const { return topology; } + void set_topology(boost::optional value) { this->topology = value; } + }; +} diff --git a/generated/RenderShaderConfig.hpp b/generated/RenderShaderConfig.hpp new file mode 100644 index 0000000..00a6c64 --- /dev/null +++ b/generated/RenderShaderConfig.hpp @@ -0,0 +1,52 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// RenderShaderConfig.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +#include "ShaderStage.hpp" +#include "RenderShaderUniform.hpp" + +namespace sgns { + /** + * Multi-stage (vertex+fragment) shader configuration for render passes + */ + + using nlohmann::json; + + /** + * Multi-stage (vertex+fragment) shader configuration for render passes + */ + class RenderShaderConfig { + public: + RenderShaderConfig() = default; + virtual ~RenderShaderConfig() = default; + + private: + std::vector stages; + boost::optional> uniforms; + + public: + /** + * Ordered shader stages (vertex, fragment) making up this render pass's pipeline + */ + const std::vector & get_stages() const { return stages; } + std::vector & get_mutable_stages() { return stages; } + void set_stages(const std::vector & value) { this->stages = value; } + + /** + * Uniform variable declarations, shared across all stages + */ + boost::optional> get_uniforms() const { return uniforms; } + void set_uniforms(boost::optional> value) { this->uniforms = value; } + }; +} diff --git a/generated/RenderShaderUniform.hpp b/generated/RenderShaderUniform.hpp new file mode 100644 index 0000000..35b999a --- /dev/null +++ b/generated/RenderShaderUniform.hpp @@ -0,0 +1,44 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// RenderShaderUniform.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + enum class DataType : int; +} + +namespace sgns { + using nlohmann::json; + + class RenderShaderUniform { + public: + RenderShaderUniform() = default; + virtual ~RenderShaderUniform() = default; + + private: + boost::optional source; + boost::optional type; + nlohmann::json value; + + public: + boost::optional get_source() const { return source; } + void set_source(boost::optional value) { this->source = value; } + + boost::optional get_type() const { return type; } + void set_type(boost::optional value) { this->type = value; } + + const nlohmann::json & get_value() const { return value; } + nlohmann::json & get_mutable_value() { return value; } + void set_value(const nlohmann::json & value) { this->value = value; } + }; +} diff --git a/generated/RenderTarget.hpp b/generated/RenderTarget.hpp new file mode 100644 index 0000000..df9de1e --- /dev/null +++ b/generated/RenderTarget.hpp @@ -0,0 +1,89 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// RenderTarget.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + enum class ColorFormat : int; + enum class DepthFormat : int; +} + +namespace sgns { + /** + * Offscreen framebuffer (color+depth) config for render passes + * + * Offscreen render-target/framebuffer config - all fields required, no schema defaults + */ + + using nlohmann::json; + + /** + * Offscreen framebuffer (color+depth) config for render passes + * + * Offscreen render-target/framebuffer config - all fields required, no schema defaults + */ + class RenderTarget { + public: + RenderTarget() : + clear_depth_constraint(boost::none, boost::none, boost::none, 1, boost::none, boost::none, boost::none), + height_constraint(1, boost::none, boost::none, boost::none, boost::none, boost::none, boost::none), + width_constraint(1, boost::none, boost::none, boost::none, boost::none, boost::none, boost::none) + {} + virtual ~RenderTarget() = default; + + private: + std::vector clear_color; + double clear_depth; + ClassMemberConstraints clear_depth_constraint; + ColorFormat color_format; + DepthFormat depth_format; + int64_t height; + ClassMemberConstraints height_constraint; + int64_t width; + ClassMemberConstraints width_constraint; + + public: + /** + * RGBA clear color + */ + const std::vector & get_clear_color() const { return clear_color; } + std::vector & get_mutable_clear_color() { return clear_color; } + void set_clear_color(const std::vector & value) { this->clear_color = value; } + + const double & get_clear_depth() const { return clear_depth; } + double & get_mutable_clear_depth() { return clear_depth; } + void set_clear_depth(const double & value) { CheckConstraint("clear_depth", clear_depth_constraint, value); this->clear_depth = value; } + + /** + * Color attachment format + */ + const ColorFormat & get_color_format() const { return color_format; } + ColorFormat & get_mutable_color_format() { return color_format; } + void set_color_format(const ColorFormat & value) { this->color_format = value; } + + /** + * Depth attachment format + */ + const DepthFormat & get_depth_format() const { return depth_format; } + DepthFormat & get_mutable_depth_format() { return depth_format; } + void set_depth_format(const DepthFormat & value) { this->depth_format = value; } + + const int64_t & get_height() const { return height; } + int64_t & get_mutable_height() { return height; } + void set_height(const int64_t & value) { CheckConstraint("height", height_constraint, value); this->height = value; } + + const int64_t & get_width() const { return width; } + int64_t & get_mutable_width() { return width; } + void set_width(const int64_t & value) { CheckConstraint("width", width_constraint, value); this->width = value; } + }; +} diff --git a/generated/SGNSProcMain.hpp b/generated/SGNSProcMain.hpp index 785c94a..1675cb6 100644 --- a/generated/SGNSProcMain.hpp +++ b/generated/SGNSProcMain.hpp @@ -23,6 +23,8 @@ #include "Params.hpp" #include "DataTransformType.hpp" #include "DataTransform.hpp" +#include "IndexType.hpp" +#include "IndexBuffer.hpp" #include "PassIoBinding.hpp" #include "ModelFormat.hpp" #include "ModelNode.hpp" @@ -30,10 +32,25 @@ #include "OptimizerType.hpp" #include "OptimizerConfig.hpp" #include "ModelConfig.hpp" -#include "ShaderType.hpp" -#include "Uniform.hpp" +#include "CullMode.hpp" +#include "DepthTest.hpp" +#include "FrontFace.hpp" +#include "Topology.hpp" +#include "PipelineState.hpp" +#include "Stage.hpp" +#include "ShaderSourceType.hpp" +#include "ShaderStage.hpp" +#include "RenderShaderUniform.hpp" +#include "RenderShaderConfig.hpp" +#include "ColorFormat.hpp" +#include "DepthFormat.hpp" +#include "RenderTarget.hpp" +#include "ShaderUniform.hpp" #include "ShaderConfig.hpp" #include "PassType.hpp" +#include "VertexBuffer.hpp" +#include "VertexLayoutFormat.hpp" +#include "VertexLayoutEntry.hpp" #include "Pass.hpp" #include "SgnsProcessing.hpp" namespace sgns { diff --git a/generated/ShaderConfig.hpp b/generated/ShaderConfig.hpp index 3290081..6aa5cca 100644 --- a/generated/ShaderConfig.hpp +++ b/generated/ShaderConfig.hpp @@ -13,21 +13,21 @@ #include #include "helper.hpp" -#include "Uniform.hpp" +#include "ShaderUniform.hpp" namespace sgns { - enum class ShaderType : int; + enum class ShaderSourceType : int; } namespace sgns { /** - * Shader configuration for compute/render passes + * Shader configuration for compute passes */ using nlohmann::json; /** - * Shader configuration for compute/render passes + * Shader configuration for compute passes */ class ShaderConfig { public: @@ -37,8 +37,8 @@ namespace sgns { private: boost::optional entry_point; std::string source; - boost::optional type; - boost::optional> uniforms; + boost::optional type; + boost::optional> uniforms; public: boost::optional get_entry_point() const { return entry_point; } @@ -51,13 +51,13 @@ namespace sgns { std::string & get_mutable_source() { return source; } void set_source(const std::string & value) { this->source = value; } - boost::optional get_type() const { return type; } - void set_type(boost::optional value) { this->type = value; } + boost::optional get_type() const { return type; } + void set_type(boost::optional value) { this->type = value; } /** * Uniform variable declarations */ - boost::optional> get_uniforms() const { return uniforms; } - void set_uniforms(boost::optional> value) { this->uniforms = value; } + boost::optional> get_uniforms() const { return uniforms; } + void set_uniforms(boost::optional> value) { this->uniforms = value; } }; } diff --git a/generated/ShaderSourceType.hpp b/generated/ShaderSourceType.hpp new file mode 100644 index 0000000..ac6045a --- /dev/null +++ b/generated/ShaderSourceType.hpp @@ -0,0 +1,27 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// ShaderSourceType.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + /** + * Shader source language, validated before it ever reaches the driver + */ + + using nlohmann::json; + + /** + * Shader source language, validated before it ever reaches the driver + */ + enum class ShaderSourceType : int { GLSL, SPIRV }; +} diff --git a/generated/ShaderStage.hpp b/generated/ShaderStage.hpp new file mode 100644 index 0000000..d66f158 --- /dev/null +++ b/generated/ShaderStage.hpp @@ -0,0 +1,57 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// ShaderStage.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + enum class Stage : int; + enum class ShaderSourceType : int; +} + +namespace sgns { + using nlohmann::json; + + class ShaderStage { + public: + ShaderStage() = default; + virtual ~ShaderStage() = default; + + private: + boost::optional entry_point; + std::string source; + Stage stage; + ShaderSourceType type; + + public: + boost::optional get_entry_point() const { return entry_point; } + void set_entry_point(boost::optional value) { this->entry_point = value; } + + /** + * Shader source path or URI parameter for this stage + */ + const std::string & get_source() const { return source; } + std::string & get_mutable_source() { return source; } + void set_source(const std::string & value) { this->source = value; } + + /** + * Which pipeline stage this shader source targets + */ + const Stage & get_stage() const { return stage; } + Stage & get_mutable_stage() { return stage; } + void set_stage(const Stage & value) { this->stage = value; } + + const ShaderSourceType & get_type() const { return type; } + ShaderSourceType & get_mutable_type() { return type; } + void set_type(const ShaderSourceType & value) { this->type = value; } + }; +} diff --git a/generated/Uniform.hpp b/generated/ShaderUniform.hpp similarity index 86% rename from generated/Uniform.hpp rename to generated/ShaderUniform.hpp index e8e06cc..f662e98 100644 --- a/generated/Uniform.hpp +++ b/generated/ShaderUniform.hpp @@ -5,7 +5,7 @@ // // Then include this file, and then do // -// Uniform.hpp data = nlohmann::json::parse(jsonString); +// ShaderUniform.hpp data = nlohmann::json::parse(jsonString); #pragma once @@ -20,10 +20,10 @@ namespace sgns { namespace sgns { using nlohmann::json; - class Uniform { + class ShaderUniform { public: - Uniform() = default; - virtual ~Uniform() = default; + ShaderUniform() = default; + virtual ~ShaderUniform() = default; private: boost::optional source; diff --git a/generated/Stage.hpp b/generated/Stage.hpp new file mode 100644 index 0000000..f2d9fb8 --- /dev/null +++ b/generated/Stage.hpp @@ -0,0 +1,27 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// Stage.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + /** + * Which pipeline stage this shader source targets + */ + + using nlohmann::json; + + /** + * Which pipeline stage this shader source targets + */ + enum class Stage : int { FRAGMENT, VERTEX }; +} diff --git a/generated/Topology.hpp b/generated/Topology.hpp new file mode 100644 index 0000000..0f98247 --- /dev/null +++ b/generated/Topology.hpp @@ -0,0 +1,20 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// Topology.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + using nlohmann::json; + + enum class Topology : int { LINE_LIST, POINT_LIST, TRIANGLE_LIST }; +} diff --git a/generated/VertexBuffer.hpp b/generated/VertexBuffer.hpp new file mode 100644 index 0000000..737d92b --- /dev/null +++ b/generated/VertexBuffer.hpp @@ -0,0 +1,53 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// VertexBuffer.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + /** + * Buffer binding supplying vertex attribute data referenced by vertex_layout (D-16 + * Amendment) + * + * Buffer binding that supplies vertex attribute data for vertex_layout entries, using the + * same prefix-notation convention as pass_io_binding + */ + + using nlohmann::json; + + /** + * Buffer binding supplying vertex attribute data referenced by vertex_layout (D-16 + * Amendment) + * + * Buffer binding that supplies vertex attribute data for vertex_layout entries, using the + * same prefix-notation convention as pass_io_binding + */ + class VertexBuffer { + public: + VertexBuffer() : + source_constraint(boost::none, boost::none, boost::none, boost::none, boost::none, boost::none, std::string("^(input|output|internal|parameter):[a-zA-Z][a-zA-Z0-9_]*$")) + {} + virtual ~VertexBuffer() = default; + + private: + std::string source; + ClassMemberConstraints source_constraint; + + public: + /** + * Data source using prefix notation + */ + const std::string & get_source() const { return source; } + std::string & get_mutable_source() { return source; } + void set_source(const std::string & value) { CheckConstraint("source", source_constraint, value); this->source = value; } + }; +} diff --git a/generated/VertexLayoutEntry.hpp b/generated/VertexLayoutEntry.hpp new file mode 100644 index 0000000..6781974 --- /dev/null +++ b/generated/VertexLayoutEntry.hpp @@ -0,0 +1,59 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// VertexLayoutEntry.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + enum class VertexLayoutFormat : int; +} + +namespace sgns { + using nlohmann::json; + + class VertexLayoutEntry { + public: + VertexLayoutEntry() : + offset_constraint(boost::none, boost::none, boost::none, boost::none, boost::none, boost::none, boost::none) + {} + virtual ~VertexLayoutEntry() = default; + + private: + VertexLayoutFormat format; + std::string name; + int64_t offset; + ClassMemberConstraints offset_constraint; + + public: + /** + * Vertex attribute component format + */ + const VertexLayoutFormat & get_format() const { return format; } + VertexLayoutFormat & get_mutable_format() { return format; } + void set_format(const VertexLayoutFormat & value) { this->format = value; } + + /** + * Vertex attribute name + */ + const std::string & get_name() const { return name; } + std::string & get_mutable_name() { return name; } + void set_name(const std::string & value) { this->name = value; } + + /** + * Byte offset within the vertex; stride is auto-computed from the tightly-packed sum of + * attribute sizes, not schema-configurable + */ + const int64_t & get_offset() const { return offset; } + int64_t & get_mutable_offset() { return offset; } + void set_offset(const int64_t & value) { CheckConstraint("offset", offset_constraint, value); this->offset = value; } + }; +} diff --git a/generated/VertexLayoutFormat.hpp b/generated/VertexLayoutFormat.hpp new file mode 100644 index 0000000..946dd56 --- /dev/null +++ b/generated/VertexLayoutFormat.hpp @@ -0,0 +1,27 @@ +// To parse this JSON data, first install +// +// Boost http://www.boost.org +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// VertexLayoutFormat.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "helper.hpp" + +namespace sgns { + /** + * Vertex attribute component format + */ + + using nlohmann::json; + + /** + * Vertex attribute component format + */ + enum class VertexLayoutFormat : int { FLOAT16, FLOAT32, INT32 }; +} diff --git a/gnus-processing-schema.json b/gnus-processing-schema.json index 0ec8d87..ee251d7 100644 --- a/gnus-processing-schema.json +++ b/gnus-processing-schema.json @@ -178,7 +178,34 @@ }, "shader": { "$ref": "#/definitions/shader_config", - "description": "Shader configuration for compute/render passes" + "description": "Shader configuration for compute passes" + }, + "render_shader": { + "$ref": "#/definitions/render_shader_config", + "description": "Multi-stage (vertex+fragment) shader configuration for render passes" + }, + "render_target": { + "$ref": "#/definitions/render_target", + "description": "Offscreen framebuffer (color+depth) config for render passes" + }, + "vertex_layout": { + "type": "array", + "description": "Vertex attribute layout for render passes", + "items": { + "$ref": "#/definitions/vertex_layout_entry" + } + }, + "vertex_buffer": { + "$ref": "#/definitions/vertex_buffer", + "description": "Buffer binding supplying vertex attribute data referenced by vertex_layout (D-16 Amendment)" + }, + "index_buffer": { + "$ref": "#/definitions/index_buffer", + "description": "Index buffer binding + index type for render passes" + }, + "pipeline_state": { + "$ref": "#/definitions/pipeline_state", + "description": "Fixed-function pipeline state for render passes" }, "data_transforms": { "type": "array", @@ -200,6 +227,24 @@ "items": { "$ref": "#/definitions/pass_io_binding" } + }, + "estimated_gpu_memory_bytes": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Estimated GPU memory needed for this pass in bytes. 0 means no estimate provided." + }, + "max_output_artifact_bytes": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Maximum output artifact size in bytes before the pass is considered budget-exceeded. 0 means no budget." + }, + "per_pass_deadline_ms": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Per-pass wall-clock deadline in milliseconds. 0 means no deadline." } }, "allOf": [ @@ -213,11 +258,19 @@ }, { "if": { - "properties": { "type": { "enum": ["compute", "render"] } } + "properties": { "type": { "const": "compute" } } }, "then": { "required": ["shader"] } + }, + { + "if": { + "properties": { "type": { "const": "render" } } + }, + "then": { + "required": ["render_shader", "render_target", "vertex_buffer"] + } } ] }, @@ -323,6 +376,11 @@ } ] }, + "shader_source_type": { + "type": "string", + "description": "Shader source language, validated before it ever reaches the driver", + "enum": ["glsl", "spirv"] + }, "shader_config": { "type": "object", "required": ["source"], @@ -332,8 +390,7 @@ "description": "Shader source path or URI parameter" }, "type": { - "type": "string", - "enum": ["glsl", "hlsl", "metal", "spirv"], + "$ref": "#/definitions/shader_source_type", "default": "glsl" }, "entry_point": { @@ -354,6 +411,167 @@ } } }, + "render_shader_config": { + "type": "object", + "description": "Multi-stage (vertex+fragment) shader configuration for render passes", + "required": ["stages"], + "properties": { + "stages": { + "type": "array", + "description": "Ordered shader stages (vertex, fragment) making up this render pass's pipeline", + "minItems": 1, + "items": { + "$ref": "#/definitions/shader_stage" + } + }, + "uniforms": { + "type": "object", + "description": "Uniform variable declarations, shared across all stages", + "additionalProperties": { + "type": "object", + "properties": { + "type": { "$ref": "#/definitions/data_type" }, + "value": {}, + "source": { "type": "string" } + } + } + } + } + }, + "shader_stage": { + "type": "object", + "required": ["stage", "type", "source"], + "properties": { + "stage": { + "type": "string", + "description": "Which pipeline stage this shader source targets", + "enum": ["vertex", "fragment"] + }, + "type": { + "$ref": "#/definitions/shader_source_type", + "default": "glsl" + }, + "source": { + "type": "string", + "description": "Shader source path or URI parameter for this stage" + }, + "entry_point": { + "type": "string", + "default": "main" + } + } + }, + "render_target": { + "type": "object", + "description": "Offscreen render-target/framebuffer config - all fields required, no schema defaults", + "required": ["color_format", "depth_format", "width", "height", "clear_color", "clear_depth"], + "properties": { + "color_format": { + "type": "string", + "description": "Color attachment format", + "enum": ["RGBA8", "RGB8"] + }, + "depth_format": { + "type": "string", + "description": "Depth attachment format", + "enum": ["D32_SFLOAT", "D24_UNORM_S8_UINT"] + }, + "width": { + "type": "integer", + "minimum": 1 + }, + "height": { + "type": "integer", + "minimum": 1 + }, + "clear_color": { + "type": "array", + "description": "RGBA clear color", + "items": { "type": "number" }, + "minItems": 4, + "maxItems": 4 + }, + "clear_depth": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "vertex_layout_entry": { + "type": "object", + "required": ["name", "format", "offset"], + "properties": { + "name": { + "type": "string", + "description": "Vertex attribute name" + }, + "format": { + "type": "string", + "description": "Vertex attribute component format", + "enum": ["FLOAT32", "FLOAT16", "INT32"] + }, + "offset": { + "type": "integer", + "description": "Byte offset within the vertex; stride is auto-computed from the tightly-packed sum of attribute sizes, not schema-configurable", + "minimum": 0 + } + } + }, + "vertex_buffer": { + "type": "object", + "description": "Buffer binding that supplies vertex attribute data for vertex_layout entries, using the same prefix-notation convention as pass_io_binding", + "required": ["source"], + "properties": { + "source": { + "type": "string", + "description": "Data source using prefix notation", + "pattern": "^(input|output|internal|parameter):[a-zA-Z][a-zA-Z0-9_]*$" + } + } + }, + "index_buffer": { + "type": "object", + "description": "Index buffer binding for render passes; index_type is schema-configurable per D-17", + "properties": { + "index_type": { + "type": "string", + "enum": ["uint16", "uint32"], + "default": "uint32" + }, + "source": { + "type": "string", + "description": "Data source using prefix notation", + "pattern": "^(input|output|internal|parameter):[a-zA-Z][a-zA-Z0-9_]*$" + } + } + }, + "pipeline_state": { + "type": "object", + "description": "Curated, minimal v1 fixed-function pipeline state subset (D-13); depth compare op is fixed at 'less', not schema-configurable (D-14)", + "properties": { + "topology": { + "type": "string", + "enum": ["triangle_list", "line_list", "point_list"], + "default": "triangle_list" + }, + "cull_mode": { + "type": "string", + "enum": ["none", "front", "back"], + "default": "back" + }, + "front_face": { + "type": "string", + "enum": ["cw", "ccw"], + "default": "ccw" + }, + "depth_test": { + "type": "string", + "enum": ["enabled", "disabled"], + "default": "enabled" + } + } + }, "pass_io_binding": { "type": "object", "required": ["name"], diff --git a/include/artifacts/artifact_serializer.hpp b/include/artifacts/artifact_serializer.hpp new file mode 100644 index 0000000..8c60ac0 --- /dev/null +++ b/include/artifacts/artifact_serializer.hpp @@ -0,0 +1,68 @@ +/** + * Deterministic binary serialization for Artifact and ExecutionManifest structs. + * + * Fixed-field, little-endian, fixed-offset binary layout (D-04, D-05, D-06). + * Every multi-byte value at a known, fixed offset; all integers in native + * little-endian byte order; variable-length data (strings, arrays) capped at + * maximum sizes with inline storage. + * + * Byte-identical output across two runs with identical inputs (ARTF-05). + * + * @brief Artifact and manifest binary serialization + */ +#ifndef SGPROCMGR_ARTIFACT_SERIALIZER_HPP +#define SGPROCMGR_ARTIFACT_SERIALIZER_HPP + +#include +#include +#include "artifacts/artifact_types.hpp" +#include "artifacts/execution_manifest.hpp" + +namespace sgns::sgprocessing +{ + + /// Fixed total size of a serialized Artifact in bytes. + /// resourceName[256] + artifactId[32] + passId[256] + outputBinding[256] + /// + dataType[64] + format[64] + width[4] + height[4] + depth[4] + byteSize[8] + /// + mediaType[128] + contentHash[32] + chunkHashCount[4] + chunkHashes[1024*32] + static constexpr size_t ARTIFACT_SERIALIZED_SIZE = 33880; + + /// Fixed total size of a serialized ExecutionManifest in bytes. + /// 5 * MAX_IDENTIFIER[256] + 6 * SHA256_HASH_SIZE[32] + inputArtifactCount[4] + /// + inputArtifactHashes[64*32] + outputArtifactCount[4] + outputArtifactHashes[64*32] + /// + startTimeUsec[8] + endTimeUsec[8] + terminalState[1] + gpuMemoryUsedBytes[8] + /// + outputBytesProduced[8] + wallClockUsec[8] + manifestHash[32] + static constexpr size_t MANIFEST_SERIALIZED_SIZE = 5649; + + /// Serialize an Artifact to a fixed-size binary blob (ARTF-05). + /// @return Vector of exactly ARTIFACT_SERIALIZED_SIZE bytes. + std::vector SerializeArtifact( const Artifact &artifact ); + + /// Deserialize a binary blob back into an Artifact struct. + /// @return true on success; false if input size != ARTIFACT_SERIALIZED_SIZE. + bool DeserializeArtifact( const std::vector &bytes, Artifact &out ); + + /// Serialize an ExecutionManifest to a fixed-size binary blob (ARTF-05). + /// + /// CRITICAL (D-04): The manifestHash field is zeroed before serialization + /// and restored afterward so it does NOT participate in its own hash computation. + /// + /// @return Vector of exactly MANIFEST_SERIALIZED_SIZE bytes. + std::vector SerializeManifest( const ExecutionManifest &manifest ); + + /// Deserialize a binary blob back into an ExecutionManifest struct. + /// @return true on success; false if input size != MANIFEST_SERIALIZED_SIZE. + bool DeserializeManifest( const std::vector &bytes, ExecutionManifest &out ); + + /// Compute the manifest self-hash: SHA-256 of serialized manifest bytes + /// with the manifestHash field zeroed (handled internally by SerializeManifest). + /// @return 32-byte SHA-256 hash. + inline std::vector ComputeManifestHash( const ExecutionManifest &manifest ) + { + auto bytes = SerializeManifest( manifest ); + return sgns::sgprocmanagersha::sha256( bytes.data(), bytes.size() ); + } + +} // namespace sgns::sgprocessing + +#endif // SGPROCMGR_ARTIFACT_SERIALIZER_HPP diff --git a/include/artifacts/artifact_types.hpp b/include/artifacts/artifact_types.hpp new file mode 100644 index 0000000..f1865f9 --- /dev/null +++ b/include/artifacts/artifact_types.hpp @@ -0,0 +1,98 @@ +/** + * Artifact type system for Phase 08: Structured Artifacts & Execution Manifests. + * + * Defines the TerminalState enum and Artifact struct — the data contracts + * for typed output records with content-hash-based identity, resource metadata, + * and per-chunk SHA-256 hashes. No protobuf — plain C++ structs per D-04/D-05. + * + * @brief Artifact and terminal state data types + */ +#ifndef SGPROCMGR_ARTIFACT_TYPES_HPP +#define SGPROCMGR_ARTIFACT_TYPES_HPP + +#include +#include +#include "util/sha256.hpp" + +namespace sgns::sgprocessing +{ + + // Fixed-size constants shared between Artifact and ExecutionManifest. + static constexpr size_t SHA256_HASH_SIZE = 32; ///< SHA-256 digest size in bytes + static constexpr size_t MAX_RESOURCE_NAME = 256; ///< Max bytes for resource/pass/binding name strings (D-06) + static constexpr size_t MAX_MEDIA_TYPE = 128; ///< Max bytes for media type string (D-06) + + /// Terminal execution outcome for the manifest (D-15). + /// Explicit uint8_t underlying type — maps directly to a single byte + /// in the binary serialized manifest layout. + enum class TerminalState : uint8_t + { + Success = 0, ///< Normal completion — all outputs valid + Cancelled = 1, ///< Cancellation token triggered (07 D-01/D-05) + Timeout = 2, ///< Deadline expired (07 D-02/D-09) + BudgetExceeded = 3, ///< Output byte budget exceeded (07 D-08/EXEC-03) + Error = 4 ///< All other failures (D-15: no error string in manifest) + }; + + /// Typed output artifact record (ARTF-01, ARTF-02, ARTF-03). + /// + /// Every artifact carries: identity (resource name + content-hash-based ID), + /// format metadata (data type, format, dimensions, byte size, media type), + /// and per-chunk SHA-256 hashes from the producing processor. + /// + /// Content-hash-based identity (D-01): artifactId = SHA-256 of raw artifact bytes. + /// Same bytes → same ID across any execution — enables deduplication and caching. + struct Artifact + { + // ── Artifact identity (ARTF-01) ──────────────────────────────── + + char resourceName[MAX_RESOURCE_NAME]; ///< Human-readable output name, null-terminated + uint8_t artifactId[SHA256_HASH_SIZE]; ///< SHA-256 of raw artifact bytes (D-01: content-addressable ID) + char passId[MAX_RESOURCE_NAME]; ///< Producing pass identity from schema + char outputBinding[MAX_RESOURCE_NAME]; ///< Output binding reference (e.g. "output:render_target") + + // ── Artifact format metadata (ARTF-02) ───────────────────────── + + char dataType[64]; ///< DataType string (e.g. "TEXTURE2_D", "TENSOR", "STRING") + char format[64]; ///< InputFormat string (e.g. "FLOAT32", "RGBA8", "INT8") + uint32_t width = 0; ///< 0 if not applicable (e.g. string output) + uint32_t height = 0; ///< 0 if not applicable + uint32_t depth = 0; ///< 0 if not applicable (1 for 2D textures by convention) + uint64_t byteSize = 0; ///< Size of raw artifact bytes in bytes + char mediaType[MAX_MEDIA_TYPE]; ///< e.g. "application/octet-stream", "image/png" + + // ── Artifact hashes (ARTF-03, D-07, D-08, D-09) ─────────────── + + uint8_t contentHash[SHA256_HASH_SIZE]; ///< SHA-256 of raw artifact bytes only (D-03/D-07) + uint32_t chunkHashCount = 0; ///< How many chunk hashes the job produced (D-09); 0 if no chunking + uint8_t chunkHashes[1024][SHA256_HASH_SIZE] = {}; ///< Per-chunk SHA-256 hashes from processor output (D-08); max 1024 (D-06) + }; + + // ── Free-standing helpers (namespace scope, inline — avoid header bloat) ── + + /// Compute artifact identity from raw bytes (D-01, D-03). + /// Delegates to sgprocmanagersha::sha256 for the actual hash. + /// Fills both contentHash and artifactId in-place. + inline void ComputeArtifactIdentity( Artifact &artifact, const uint8_t *rawBytes, size_t byteCount ) + { + auto hash = sgns::sgprocmanagersha::sha256( rawBytes, byteCount ); + std::memcpy( artifact.contentHash, hash.data(), SHA256_HASH_SIZE ); + std::memcpy( artifact.artifactId, hash.data(), SHA256_HASH_SIZE ); + } + + /// Add a chunk hash to the artifact's chunk hash list (D-08). + /// @return true on success, false if chunkHashCount >= 1024 (overflow guard). + inline bool AddChunkHash( Artifact &artifact, const uint8_t hash[SHA256_HASH_SIZE] ) + { + if ( artifact.chunkHashCount >= 1024 ) + { + return false; + } + std::memcpy( artifact.chunkHashes[artifact.chunkHashCount], hash, SHA256_HASH_SIZE ); + ++artifact.chunkHashCount; + return true; + } + +} // namespace sgns::sgprocessing + +#endif // SGPROCMGR_ARTIFACT_TYPES_HPP diff --git a/include/artifacts/execution_manifest.hpp b/include/artifacts/execution_manifest.hpp new file mode 100644 index 0000000..1fb322a --- /dev/null +++ b/include/artifacts/execution_manifest.hpp @@ -0,0 +1,87 @@ +/** + * Execution manifest for Phase 08: Structured Artifacts & Execution Manifests. + * + * Defines the ExecutionManifest struct — a self-contained, fixed-field record + * that captures everything needed for hashing, signing, caching, and verification + * of an execution run (ARTF-04, D-13). All identity fields are inline; inapplicable + * identities use the sentinel zero-hash convention (D-14). + * + * @brief Execution manifest data contract + */ +#ifndef SGPROCMGR_EXECUTION_MANIFEST_HPP +#define SGPROCMGR_EXECUTION_MANIFEST_HPP + +#include +#include "artifacts/artifact_types.hpp" + +namespace sgns::sgprocessing +{ + + // Manifest-specific size constants (D-06). + static constexpr size_t MAX_ARTIFACT_REFS = 64; ///< Max input/output artifact hash references + static constexpr size_t MAX_IDENTIFIER = 256; ///< Max bytes for execution/attempt/task/subtask/pass ID strings + + /// Self-contained execution manifest (ARTF-04, D-13). + /// + /// Captures: execution identifiers, executor & compatibility identities, + /// input/output artifact hash references, timing, terminal state, + /// resource-use summary, and a manifest self-hash (computed by + /// artifact_serializer.hpp — see ComputeManifestHash). + /// + /// Sentinel zero-hash convention (D-14): modelIdentity, tokenizerIdentity, + /// adapterIdentity, shaderIdentity, and quantizationIdentity are all-zero + /// when the corresponding resource was not used in this execution. + struct ExecutionManifest + { + // ── Execution identifiers (ARTF-04 first group) ──────────────── + + char executionId[MAX_IDENTIFIER]; ///< Execution-scoped ID + char attemptId[MAX_IDENTIFIER]; ///< Retry-attempt ID + char taskId[MAX_IDENTIFIER]; ///< Task ID from job definition + char subtaskId[MAX_IDENTIFIER]; ///< Subtask ID from processing pipeline + char passId[MAX_RESOURCE_NAME]; ///< Producing pass identity + + // ── Executor & compatibility identities (D-13, D-14) ─────────── + + uint8_t executorIdentity[SHA256_HASH_SIZE]; ///< From CapabilitySnapshot::identityHash (Phase 06 D-08); never zero + uint8_t modelIdentity[SHA256_HASH_SIZE]; ///< SHA-256 of model bytes; all zeros if no model (D-14) + uint8_t tokenizerIdentity[SHA256_HASH_SIZE]; ///< All zeros if no tokenizer (D-14) + uint8_t adapterIdentity[SHA256_HASH_SIZE]; ///< All zeros if no adapter (D-14) + uint8_t shaderIdentity[SHA256_HASH_SIZE]; ///< SHA-256 of compiled SPIR-V bytes; all zeros if no shader (D-14) + uint8_t quantizationIdentity[SHA256_HASH_SIZE]; ///< All zeros if no quantization (D-14) + + // ── Input / output artifact hash references (D-13) ───────────── + + uint32_t inputArtifactCount = 0; + uint8_t inputArtifactHashes[MAX_ARTIFACT_REFS][SHA256_HASH_SIZE]; + + uint32_t outputArtifactCount = 0; + uint8_t outputArtifactHashes[MAX_ARTIFACT_REFS][SHA256_HASH_SIZE]; + + // ── Timing (D-13) ────────────────────────────────────────────── + + int64_t startTimeUsec = 0; ///< Microseconds since Unix epoch, captured before StartProcessing() + int64_t endTimeUsec = 0; ///< Microseconds since Unix epoch, captured after StartProcessing() returns + + // ── Terminal state (D-13, D-15) ──────────────────────────────── + + TerminalState terminalState = TerminalState::Success; + + // ── Resource-use summary (D-13) ──────────────────────────────── + + uint64_t gpuMemoryUsedBytes = 0; ///< Peak GPU memory during execution; 0 if not tracked + uint64_t outputBytesProduced = 0; ///< Sum of all output artifact byte sizes + uint64_t wallClockUsec = 0; ///< endTimeUsec - startTimeUsec; computed at manifest assembly time + + // ── Manifest self-hash ───────────────────────────────────────── + // + // Computed by SerializeManifest() in artifact_serializer.hpp as + // SHA-256 of the serialized manifest bytes with this field zeroed + // (to avoid self-referential hashing). Write all zeros here before + // calling SerializeManifest / ComputeManifestHash. + uint8_t manifestHash[SHA256_HASH_SIZE]; + }; + +} // namespace sgns::sgprocessing + +#endif // SGPROCMGR_EXECUTION_MANIFEST_HPP diff --git a/include/capability/capability_types.hpp b/include/capability/capability_types.hpp new file mode 100644 index 0000000..8661005 --- /dev/null +++ b/include/capability/capability_types.hpp @@ -0,0 +1,79 @@ +/** + * Capability validation type system for Phase 06. + * + * Defines the data contracts (UnmetRequirement, CanExecuteResult, CapabilitySnapshot) + * that all capability validation checks build against. No protobuf — plain C++ structs + * per D-05. + * + * @brief Capability validation data types + */ +#ifndef SGPROCMGR_CAPABILITY_TYPES_HPP +#define SGPROCMGR_CAPABILITY_TYPES_HPP + +#include +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ + + /// Hash functor for PassType keys. + /// Used by CapabilitySnapshot::checkpointSupport and elsewhere. + struct PassTypeHash + { + size_t operator()( PassType p ) const { return static_cast( p ); } + }; + + /// Category of unmet requirement for structured capability rejection (D-06). + /// Follows same enum-prefix convention as ProcessingErrorStage in processing_processor.hpp. + enum class UnmetRequirementCategory + { + VULKAN = 0, ///< Vulkan device/feature/limit insufficiency + MNN = 1, ///< MNN model format or quantization not supported + PASS_TYPE = 2, ///< No executor registered for the requested PassType + RESOURCE = 3 ///< GPU memory or disk space insufficient + }; + + /// A single unmet capability requirement with category tag and human-readable detail. + /// Pattern follows ProcessingError in processing_processor.hpp (D-06). + struct UnmetRequirement + { + UnmetRequirementCategory category = UnmetRequirementCategory::RESOURCE; + std::string detail; ///< Human-readable reason, e.g. "maxImageDimension2D: need 16384, have 8192" + }; + + /// Declared capability of a registered executor (D-11). + struct ExecutorCapability + { + PassType passType; ///< PassType this executor handles + std::vector supportedModelFormats; ///< e.g. ".mnn", ".caffemodel" + std::vector supportedQuantizations;///< e.g. "FP32", "FP16", "INT8" + std::string backend; ///< "VULKAN" (per D-13, all MNN on Vulkan after Phase 01.1) + }; + + /// Full capability snapshot built once at startup (D-08, D-09). + /// Cached for all subsequent CanExecute calls (D-12). + struct CapabilitySnapshot + { + VkPhysicalDeviceProperties vulkanProps{}; ///< From vkGetPhysicalDeviceProperties() (D-14); zero-initialized so deviceName is a valid empty C-string when no device was found + VkPhysicalDeviceMemoryProperties memProps{}; ///< From vkGetPhysicalDeviceMemoryProperties() (D-15) + std::vector executorCaps; ///< From registry query (D-11) + uint64_t availableDiskBytes = 0; ///< From platform syscall (D-16); 0 = query failed (degraded) + std::vector identityHash; ///< SHA-256 of serialized snapshot (D-08) + std::unordered_map checkpointSupport; ///< Per-PassType checkpoint support flag (D-20) + }; + + /// Result of a CanExecute check (D-05, D-07). + struct CanExecuteResult + { + bool executable = false; ///< True if all capability checks pass + std::string executorId; ///< Populated only when executable==true; hex prefix of identityHash (D-08) + std::vector unmet; ///< Populated only when executable==false; one entry per failing check + }; + +} // namespace sgns::sgprocessing + +#endif // SGPROCMGR_CAPABILITY_TYPES_HPP diff --git a/include/capability/capability_validator.hpp b/include/capability/capability_validator.hpp new file mode 100644 index 0000000..482a102 --- /dev/null +++ b/include/capability/capability_validator.hpp @@ -0,0 +1,86 @@ +/** + * CapabilityValidator — pre-execution capability gate for SGProcessingManager. + * + * Constructed internally by ProcessingManager (D-01, D-04). Builds a capability + * snapshot once at startup (D-09, D-12), then provides CanExecute() to validate + * jobs against the cached snapshot before claiming work from the network (D-02). + * + * Uses PIMPL pattern to keep Vulkan headers out of transitive includes. + * + * @brief Pre-execution capability validation gate + */ +#ifndef SGPROCMGR_CAPABILITY_VALIDATOR_HPP +#define SGPROCMGR_CAPABILITY_VALIDATOR_HPP + +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ + + // Forward declaration — ProcessingManager provides the factory map. + class ProcessingProcessor; + + /// Callback type for async CanExecute (D-03). + using CanExecuteCallback = std::function; + + /// Pre-execution capability validation gate. + /// + /// Built once at startup by ProcessingManager::Init(), then used by the scheduler + /// to validate jobs before claiming them from the network. All checks run against + /// the cached CapabilitySnapshot — no I/O needed at check time. + class CapabilityValidator + { + public: + CapabilityValidator(); + ~CapabilityValidator(); + + // Non-copyable, non-movable (owns PIMPL) + CapabilityValidator( const CapabilityValidator & ) = delete; + CapabilityValidator &operator=( const CapabilityValidator & ) = delete; + CapabilityValidator( CapabilityValidator && ) = delete; + CapabilityValidator &operator=( CapabilityValidator && ) = delete; + + /// Build the capability snapshot once at startup (D-09, D-12). + /// Called by ProcessingManager::Init() after all Register* calls. + /// Acquires VulkanInitMutex internally (D-10). + /// + /// @param passFactories — reference to ProcessingManager's m_passFactories, + /// used to enumerate registered PassType→executor mappings (D-11) + /// @param mnnProcessorCount — number of registered MNN processors (DataType-keyed) + /// @param ensureVulkanDevice — callable that ensures Vulkan device exists and + /// returns the VkPhysicalDevice handle + void BuildSnapshot( + const std::unordered_map()>, + PassTypeHash> &passFactories, + size_t mnnProcessorCount, + std::function ensureVulkanDevice ); + + /// Access the cached snapshot. Returns nullptr before BuildSnapshot(). + const CapabilitySnapshot *GetSnapshot() const; + + /// Validate whether a job can be executed on this node (CAP-02..05). + /// Checks PassType registration, Vulkan limits, MNN model compatibility, + /// GPU memory, and disk space against the cached snapshot. + /// @param pass — the job definition to validate + /// @param callback — invoked with CanExecuteResult (D-03 async pattern) + void CanExecute( const sgns::Pass &pass, CanExecuteCallback callback ); + +#ifdef SGPROCMGR_TEST_FRIEND + /// Test-only: replace the cached snapshot with a mock. + /// Only available when SGPROCMGR_TEST_FRIEND is defined. + void SetSnapshotForTest( CapabilitySnapshot snap ); +#endif + + private: + struct Impl; + std::unique_ptr m_impl; + }; + +} // namespace sgns::sgprocessing + +#endif // SGPROCMGR_CAPABILITY_VALIDATOR_HPP diff --git a/include/execution/execution_context.hpp b/include/execution/execution_context.hpp new file mode 100644 index 0000000..e7fb65c --- /dev/null +++ b/include/execution/execution_context.hpp @@ -0,0 +1,163 @@ +#pragma once +/** + * Execution context types for Phase 07: Cancellable Execution Context. + * + * Defines CancellationToken (callback-based cooperative cancellation), + * ExecutionContext (bundles cancel token, progress callback, deadline, budgets), + * ProgressEvent (stage-boundary progress event), and standardized stage enums + * per processor type (RenderStage, MNNStage). + * + * @brief Execution context data contracts + */ +#ifndef SGPROCMGR_EXECUTION_CONTEXT_HPP +#define SGPROCMGR_EXECUTION_CONTEXT_HPP + +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ + + /// Standardized pipeline stages for RenderProcessor (D-12). + /// Matches the four coarse checkpoints in D-04 for RenderProcessor. + enum class RenderStage + { + COMPILE = 0, ///< Shader compilation + BUILD_PIPELINE = 1, ///< Pipeline creation + DRAW = 2, ///< Draw submission + READBACK = 3 ///< Readback from framebuffer + }; + + /// Standardized pipeline stages for MNN processors (D-12). + /// Matches the coarse checkpoints in D-04 for MNN processors. + enum class MNNStage + { + LOAD_MODEL = 0, ///< Model file loaded / MNN interpreter created + CREATE_SESSION = 1, ///< MNN session created + RUN = 2, ///< Inference executed + READ_OUTPUT = 3 ///< Output tensor read + }; + + /// Minimal progress event fired at every stage boundary (D-11, D-13). + /// Carries pass_id, stage name, and percent (0–100 float). + struct ProgressEvent + { + std::string pass_id; ///< Pass name from schema + RenderStage render_stage = RenderStage::COMPILE; ///< Populated for render passes + MNNStage mnn_stage = MNNStage::LOAD_MODEL; ///< Populated for MNN passes + float percent = 0.0f; ///< 0.0–100.0, cumulative progress estimate + + /// Factory for render pass progress events. + static ProgressEvent ForRender( std::string passId, RenderStage stage, float pct ) + { + ProgressEvent ev; + ev.pass_id = std::move( passId ); + ev.render_stage = stage; + ev.percent = pct; + return ev; + } + + /// Factory for MNN pass progress events. + static ProgressEvent ForMNN( std::string passId, MNNStage stage, float pct ) + { + ProgressEvent ev; + ev.pass_id = std::move( passId ); + ev.mnn_stage = stage; + ev.percent = pct; + return ev; + } + }; + + /// Callback-based cooperative cancellation token (D-01, D-02, D-05). + /// + /// Thread-safe: Cancel() may be called from deadline timer thread while + /// IsCancelled() is read on the processing thread. Uses std::atomic + /// with acquire/release ordering. + struct CancellationToken + { + CancellationToken() = default; + CancellationToken( const CancellationToken & ) = delete; + CancellationToken &operator=( const CancellationToken & ) = delete; + CancellationToken( CancellationToken && ) = delete; + CancellationToken &operator=( CancellationToken && ) = delete; + + /// Invokes the registered cancel callback (if set) and sets the cancelled flag. + /// The callback is invoked synchronously, at most once. + void Cancel() + { + bool expected = false; + if ( m_cancelled.compare_exchange_strong( expected, true, + std::memory_order_release, std::memory_order_acquire ) ) + { + if ( m_cancelCallback ) + { + m_cancelCallback(); + } + } + } + + /// Returns true after Cancel() has been called. + bool IsCancelled() const + { + return m_cancelled.load( std::memory_order_acquire ); + } + + /// Register the cancel callback. Called by ProcessingManager before + /// passing the token to the processor. + void SetCallback( std::function callback ) + { + m_cancelCallback = std::move( callback ); + } + + private: + std::function m_cancelCallback; + std::atomic m_cancelled{ false }; + }; + + /// Bundles everything a processor needs for an execution (D-01, D-02, D-06, D-08, D-10). + /// + /// One ExecutionContext per job (D-02), shared across all passes in the job's pass graph. + struct ExecutionContext + { + ExecutionContext() = default; + ExecutionContext( const ExecutionContext & ) = delete; + ExecutionContext &operator=( const ExecutionContext & ) = delete; + ExecutionContext( ExecutionContext && ) = delete; + ExecutionContext &operator=( ExecutionContext && ) = delete; + + CancellationToken cancelToken; ///< Per-job cancellation token (D-02) + std::function progressCallback; ///< Processor calls at stage boundaries (D-10) + + /// Capture-only hook: fires with (quantized bytes, pre-quantize bytes) at each + /// processor's existing hash call site. Stays unset (nullptr) in production job + /// execution; only tools/capture/capture_harness.cpp (Wave 3) ever sets this. + /// Mirrors progressCallback's opt-in injection pattern via ExecutionContext, + /// not a new StartProcessing() parameter. + std::function &quantizedBytes, + const std::vector &preQuantizeBytes )> rawOutputCapture; + + uint64_t deadlineMs = 0; ///< Per-pass wall-clock deadline in ms; 0 = no deadline (D-08) + uint64_t gpuMemoryBudget = 0; ///< Estimated GPU memory in bytes; 0 = no budget (D-08) + uint64_t maxOutputArtifactBytes = 0; ///< Max output artifact size in bytes; 0 = no budget (D-08) + + /// Returns a fully no-op ExecutionContext as a heap-allocated unique_ptr. + /// Used in tests. ExecutionContext is non-copyable, non-movable due to + /// CancellationToken containing std::atomic. + static std::unique_ptr NoOp() + { + auto ctx = std::make_unique(); + ctx->cancelToken.SetCallback( []() {} ); + ctx->progressCallback = []( const ProgressEvent & ) {}; + // Deliberately left unset (nullptr), unlike progressCallback: rawOutputCapture + // is opt-in only, so production and test callers that never set it explicitly + // pay zero capture-path cost. Do not "fix" this to match progressCallback. + return ctx; + } + }; + +} // namespace sgns::sgprocessing + +#endif // SGPROCMGR_EXECUTION_CONTEXT_HPP diff --git a/include/processingbase/ProcessingManager.hpp b/include/processingbase/ProcessingManager.hpp index 82fa2eb..c766d2b 100644 --- a/include/processingbase/ProcessingManager.hpp +++ b/include/processingbase/ProcessingManager.hpp @@ -21,6 +21,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -30,6 +35,30 @@ namespace sgns::sgprocessing // Move enum to namespace level using ProcessingProcessor = sgns::sgprocessing::ProcessingProcessor; + /// Executor registry entry wrapping a processor factory and checkpoint support flag (D-20). + struct ExecutorRegistryEntry + { + std::function()> factory; + bool supports_checkpointing = false; + }; + + /// Structured output from Process() — typed artifact records + execution manifest (Phase 08, ARTF-01/02/04). + struct ProcessOutput + { + std::vector artifacts; ///< One Artifact per output buffer (ARTF-01, ARTF-02) + ExecutionManifest manifest; ///< Full execution manifest (ARTF-04) + std::vector combinedHash; ///< SHA-256 of serialized manifest (for backward compat) + + // Backward-compatible accessors — delegate to combinedHash so existing callers + // that treat the Process() return as std::vector continue to compile (D-10). + size_t size() const { return combinedHash.size(); } + bool empty() const { return combinedHash.empty(); } + auto begin() const { return combinedHash.begin(); } + auto end() const { return combinedHash.end(); } + auto begin() { return combinedHash.begin(); } + auto end() { return combinedHash.end(); } + }; + class ProcessingManager { public: @@ -42,26 +71,76 @@ namespace sgns::sgprocessing NO_PROCESSOR = 4, MISSING_INPUT = 5, INPUT_UNAVAIL = 6, + SHADER_COMPILE_FAILED = 7, + SPIRV_VALIDATION_FAILED = 8, + PROCESSING_FAILED = 9, + MODEL_MISSING = 10, + MODEL_FORMAT_UNSUPPORTED = 11, + RENDER_SHADER_MISSING = 12, + UNKNOWN_PASS_TYPE = 13, }; static outcome::result> Create( const std::string &jsondata ); outcome::result ParseBlockSize(); outcome::result CheckProcessValidity(); - outcome::result> Process( std::shared_ptr ioc, - std::vector> &chunkhashes, - sgns::ModelNode &model, - std::vector &output_locations ); - - /** Register an available processor - * @param name - Name of processor - * @param factoryFunction - Pointer to processor - */ + outcome::result Process( std::shared_ptr ioc, + std::vector> &chunkhashes, + sgns::ModelNode &model, + std::vector &output_locations ); + + /** Process() overload accepting a caller-owned ExecutionContext (Gap 2 / TEST-07). + * Lets a caller cancel mid-execution via `externalExecCtx.cancelToken.Cancel()` + * from another thread, or pre-set `deadlineMs`/`gpuMemoryBudget`/ + * `maxOutputArtifactBytes` before calling. Per-pass schema-derived budgets are + * still applied as defaults, but only when the corresponding field is still `0` + * (unset) on entry — an explicit caller-supplied nonzero value is never + * overwritten. Delegates to the same ProcessInternal() implementation as the + * legacy 4-arg overload above, so behavior is otherwise identical. + * @param ioc — Boost.Asio io_context used for IPFS/file IO + * @param chunkhashes — chunk hashes for the input data + * @param model — model node describing the input source + * @param output_locations — populated with save locations for produced outputs + * @param externalExecCtx — caller-owned ExecutionContext; not copied or reset + */ + outcome::result Process( std::shared_ptr ioc, + std::vector> &chunkhashes, + sgns::ModelNode &model, + std::vector &output_locations, + ExecutionContext &externalExecCtx ); + + /** Pre-execution capability gate (D-02, D-19). + * Validates whether this node can execute the given pass — checks PassType + * registration, Vulkan limits, MNN model compatibility, GPU memory, and disk + * space against the cached startup snapshot. Caller's responsibility to call + * this before Process(); Process() trusts the caller validated. + * @param pass — the job pass definition to validate + * @param callback — invoked with CanExecuteResult + */ + void CanExecute( const sgns::Pass &pass, + sgns::sgprocessing::CanExecuteCallback callback ); + + /** Register an available processor keyed by DataType + * @param name - DataType cast to int + * @param factoryFunction - Pointer to processor + */ void RegisterProcessorFactory( const int &name, std::function()> factoryFunction ) { m_processorFactories[name] = std::move( factoryFunction ); } + /** Register an available processor keyed by PassType + * @param type - PassType enum + * @param factoryFunction - Pointer to processor + * @param supportsCheckpointing - Whether this executor supports checkpoint/resume (D-20) + */ + void RegisterPassProcessorFactory( PassType type, + std::function()> factoryFunction, + bool supportsCheckpointing = false ) + { + m_passFactories[type] = { std::move( factoryFunction ), supportsCheckpointing }; + } + /** Get Processing Data item which can be used to access any processing data, inputs, or params. */ sgns::SgnsProcessing GetProcessingData(); @@ -112,6 +191,16 @@ namespace sgns::sgprocessing std::string url, std::shared_ptr> results ); + /** Shared implementation for both public Process() overloads (Gap 2 / TEST-07). + * @param execCtx — either a freshly-constructed local context (from the 4-arg + * overload) or a caller-owned one (from the 5-arg overload). + */ + outcome::result ProcessInternal( std::shared_ptr ioc, + std::vector> &chunkhashes, + sgns::ModelNode &model, + std::vector &output_locations, + ExecutionContext &execCtx ); + bool SetProcessorByName( const int &name ) { auto factoryFunction = m_processorFactories.find( name ); @@ -124,12 +213,28 @@ namespace sgns::sgprocessing return false; } + bool SetProcessorByPassType( PassType type ) + { + auto factoryFunction = m_passFactories.find( type ); + if ( factoryFunction != m_passFactories.end() ) + { + m_processor = factoryFunction->second.factory(); + return true; + } + std::cerr << "Unknown pass type: " << static_cast( type ) << std::endl; + return false; + } + sgns::sgprocmanager::Logger m_logger = sgns::sgprocmanager::createLogger( "SGProcessingManager" ); sgns::SgnsProcessing processing_; std::unique_ptr m_processor; - std::unordered_map()>> m_processorFactories; - std::unordered_map m_inputMap; + std::unordered_map()>> m_processorFactories; + std::unordered_map m_passFactories; + std::unordered_map m_inputMap; + std::unique_ptr m_capabilityValidator; }; } +OUTCOME_HPP_DECLARE_ERROR_2( sgns::sgprocessing, ProcessingManager::Error ); + #endif diff --git a/include/processingbase/vulkan_init_guard.hpp b/include/processingbase/vulkan_init_guard.hpp new file mode 100644 index 0000000..f6b95fd --- /dev/null +++ b/include/processingbase/vulkan_init_guard.hpp @@ -0,0 +1,23 @@ +#pragma once +#include + +namespace sgns::sgprocessing +{ + // Process-wide, header-declared synchronization primitive guarding every + // Vulkan instance/device-creation call site in this process: every MNN + // processor's createSession() call that requests MNN_FORWARD_VULKAN, + // plus RenderProcessor's lazy-init path. This is a pattern, not a fixed + // count -- grep `MNN_FORWARD_VULKAN` across + // SGProcessingManager/src/processors/*.cpp for the current, authoritative + // call-site count; re-verify it whenever a processor migrates backends, + // since a stale hardcoded number here has already undercounted the real + // total once (see 01.1-03-SUMMARY.md). Acquire via std::lock_guard at + // each call site, once per Vulkan-init call, for the lifetime of the + // process -- this must be acquired repeatedly (a run-once primitive + // would be the wrong tool here). + inline std::mutex &VulkanInitMutex() + { + static std::mutex vulkan_init_mutex; // magic static -- thread-safe init, C++11+ + return vulkan_init_mutex; + } +} diff --git a/include/processors/processing_processor.hpp b/include/processors/processing_processor.hpp index b77badc..f9461a7 100644 --- a/include/processors/processing_processor.hpp +++ b/include/processors/processing_processor.hpp @@ -8,20 +8,60 @@ #include #include +#include #include #include #include #include #include +#include namespace sgns::sgprocessing { + /// Per-stage failure classification for structured processor errors (D-25/D-26). + /// Plain, non-outcome::result enum -- StartProcessing()'s return type stays the + /// concrete ProcessingResult, so the OUTCOME_HPP_DECLARE_ERROR_2 machinery is + /// unnecessary here. + enum class ProcessingErrorStage + { + UNSPECIFIED = 0, + CONTEXT_INIT_FAILED, + RESOURCE_RESOLUTION, + BUFFER_ALLOCATION, + IMAGE_ALLOCATION, + FORMAT_UNSUPPORTED, + SHADER_MODULE_CREATION, + PIPELINE_CREATION, + RENDER_PASS_CREATION, + DRAW_SUBMISSION, + READBACK, + DATA_TRANSFORM_UNSUPPORTED, + CANCELLED = 12, ///< Processor cancelled via CancellationToken (D-05: distinct error code) + TIMED_OUT = 13, ///< Per-pass deadline expired (D-02, D-05) + BUDGET_EXCEEDED = 14 ///< Output artifact size exceeded max_output_artifact_bytes (EXEC-03) + }; + + /// Structured, per-stage processor failure detail (D-25/D-26). Carries the + /// failing VkResult/context as a plain message string. + struct ProcessingError + { + ProcessingErrorStage stage = ProcessingErrorStage::UNSPECIFIED; + std::string message; + }; + struct ProcessingResult { std::vector hash; std::shared_ptr, std::vector>>> output_buffers; /// Output locations for each saved result (file paths, IPFS CIDs, URLs, etc.) std::vector output_locations; + /// Structured per-stage failure detail (D-25/D-26). Empty/unset on success. + std::optional error; + + /// Migration adapter: convert new ProcessOutput to legacy ProcessingResult (D-10). + /// This is a TEMPORARY adapter — removed before Phase 08 ships per D-10/D-12. + /// Forward-declared to avoid circular dependency with ProcessingManager.hpp. + static ProcessingResult FromProcessOutput( const struct ProcessOutput &output ); }; class ProcessingProcessor @@ -29,16 +69,21 @@ namespace sgns::sgprocessing public: virtual ~ProcessingProcessor() = default; - /** Start processing data - * @param result - Reference to result item to set hashes to - * @param task - Reference to task to get image split data - * @param subTask - Reference to subtask to get chunk data from - */ + /** Start processing data with ExecutionContext (D-19, D-21). + * Pure virtual — all processors must implement this 6-argument overload. + * @param chunkhashes - Hashes of input chunks + * @param proc - Input/output declaration + * @param imageData - Image data buffer + * @param modelFile - Model file buffer + * @param parameters - Processing parameters + * @param execCtx - Execution context (cancel token, progress, budgets, deadline) + */ virtual ProcessingResult StartProcessing( std::vector> &chunkhashes, const sgns::IoDeclaration &proc, std::vector &imageData, std::vector &modelFile, - const std::vector *parameters ) = 0; + const std::vector *parameters, + const ExecutionContext &execCtx ) = 0; /** Set data for processor * @param buffers - Data containing file name and data pair lists. @@ -50,9 +95,41 @@ namespace sgns::sgprocessing */ virtual float GetProgress() const { return m_progress; } + /// Invokes every entry in m_teardownFns in reverse order (LIFO), + /// wrapping each in try/catch to guarantee noexcept-safe execution (D-17). + /// Public so ProcessingManager can call it in catch blocks (D-17). + void RunTeardown() + { + for ( auto it = m_teardownFns.rbegin(); it != m_teardownFns.rend(); ++it ) + { + try + { + ( *it )(); + } + catch ( ... ) + { + // D-17: Teardown functions must be noexcept-safe since + // stack unwinding may leave Vulkan objects in unknown + // state. Log failure and continue. + } + } + m_teardownFns.clear(); + } + protected: + /// Appends a teardown action to the LIFO teardown stack (D-14). + /// MNN processors register session cleanup; RenderProcessor registers + /// Vulkan object destruction. Protected — only subclasses push to stack. + void PushTeardown( std::function fn ) + { + m_teardownFns.push_back( std::move( fn ) ); + } + std::atomic m_progress{0.0f}; // Progress percentage sgns::sgprocmanager::Logger m_logger = sgns::sgprocmanager::createLogger( "SGProcessor" ); + + private: + std::vector> m_teardownFns; ///< LIFO teardown stack (D-14) }; } diff --git a/include/processors/processing_processor_mnn_audio.hpp b/include/processors/processing_processor_mnn_audio.hpp index 4de9c42..65c7be0 100644 --- a/include/processors/processing_processor_mnn_audio.hpp +++ b/include/processors/processing_processor_mnn_audio.hpp @@ -40,7 +40,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &imageData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; /** Set data for processor * @param buffers - Data containing file name and data pair lists. diff --git a/include/processors/processing_processor_mnn_bool.hpp b/include/processors/processing_processor_mnn_bool.hpp index 8320d66..ec16d8f 100644 --- a/include/processors/processing_processor_mnn_bool.hpp +++ b/include/processors/processing_processor_mnn_bool.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &boolData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_buffer.hpp b/include/processors/processing_processor_mnn_buffer.hpp index 25c2cb9..437e11b 100644 --- a/include/processors/processing_processor_mnn_buffer.hpp +++ b/include/processors/processing_processor_mnn_buffer.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &bufferData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_float.hpp b/include/processors/processing_processor_mnn_float.hpp index 21325f7..05571ee 100644 --- a/include/processors/processing_processor_mnn_float.hpp +++ b/include/processors/processing_processor_mnn_float.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &floatData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_image.hpp b/include/processors/processing_processor_mnn_image.hpp index ab623f9..144430b 100644 --- a/include/processors/processing_processor_mnn_image.hpp +++ b/include/processors/processing_processor_mnn_image.hpp @@ -58,7 +58,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &imageData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; /** Set data for processor * @param buffers - Data containing file name and data pair lists. diff --git a/include/processors/processing_processor_mnn_int.hpp b/include/processors/processing_processor_mnn_int.hpp index c1055b6..d303f85 100644 --- a/include/processors/processing_processor_mnn_int.hpp +++ b/include/processors/processing_processor_mnn_int.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &intData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_mat2.hpp b/include/processors/processing_processor_mnn_mat2.hpp index 7f8aa29..800998b 100644 --- a/include/processors/processing_processor_mnn_mat2.hpp +++ b/include/processors/processing_processor_mnn_mat2.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &mat2Data, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_mat3.hpp b/include/processors/processing_processor_mnn_mat3.hpp index bb1fce2..96a4a30 100644 --- a/include/processors/processing_processor_mnn_mat3.hpp +++ b/include/processors/processing_processor_mnn_mat3.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &mat3Data, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_mat4.hpp b/include/processors/processing_processor_mnn_mat4.hpp index 8573137..cac1d85 100644 --- a/include/processors/processing_processor_mnn_mat4.hpp +++ b/include/processors/processing_processor_mnn_mat4.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &mat4Data, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_ml.hpp b/include/processors/processing_processor_mnn_ml.hpp index 0d56834..0bedd46 100644 --- a/include/processors/processing_processor_mnn_ml.hpp +++ b/include/processors/processing_processor_mnn_ml.hpp @@ -40,7 +40,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &imageData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; /** Set data for processor * @param buffers - Data containing file name and data pair lists. diff --git a/include/processors/processing_processor_mnn_string.hpp b/include/processors/processing_processor_mnn_string.hpp index edc9d5d..730d73f 100644 --- a/include/processors/processing_processor_mnn_string.hpp +++ b/include/processors/processing_processor_mnn_string.hpp @@ -40,7 +40,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &textData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: /** Run MNN processing on text/string diff --git a/include/processors/processing_processor_mnn_tensor.hpp b/include/processors/processing_processor_mnn_tensor.hpp index 777982b..2f71d0b 100644 --- a/include/processors/processing_processor_mnn_tensor.hpp +++ b/include/processors/processing_processor_mnn_tensor.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &tensorData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_texture1d.hpp b/include/processors/processing_processor_mnn_texture1d.hpp index 5a4eca9..a90b3af 100644 --- a/include/processors/processing_processor_mnn_texture1d.hpp +++ b/include/processors/processing_processor_mnn_texture1d.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &signalData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &signalData, diff --git a/include/processors/processing_processor_mnn_texturecube.hpp b/include/processors/processing_processor_mnn_texturecube.hpp index 47d66a7..2b53833 100644 --- a/include/processors/processing_processor_mnn_texturecube.hpp +++ b/include/processors/processing_processor_mnn_texturecube.hpp @@ -21,7 +21,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &cubeData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: std::unique_ptr Process( const std::vector &inputData, diff --git a/include/processors/processing_processor_mnn_vec2.hpp b/include/processors/processing_processor_mnn_vec2.hpp index 44c47ac..a48beff 100644 --- a/include/processors/processing_processor_mnn_vec2.hpp +++ b/include/processors/processing_processor_mnn_vec2.hpp @@ -17,7 +17,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &vec2Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: diff --git a/include/processors/processing_processor_mnn_vec3.hpp b/include/processors/processing_processor_mnn_vec3.hpp index 588270c..b626dc8 100644 --- a/include/processors/processing_processor_mnn_vec3.hpp +++ b/include/processors/processing_processor_mnn_vec3.hpp @@ -17,7 +17,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &vec3Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: diff --git a/include/processors/processing_processor_mnn_vec4.hpp b/include/processors/processing_processor_mnn_vec4.hpp index 48ddb21..d980987 100644 --- a/include/processors/processing_processor_mnn_vec4.hpp +++ b/include/processors/processing_processor_mnn_vec4.hpp @@ -17,7 +17,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &vec4Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: diff --git a/include/processors/processing_processor_mnn_volume.hpp b/include/processors/processing_processor_mnn_volume.hpp index 2d7f9d9..97654a8 100644 --- a/include/processors/processing_processor_mnn_volume.hpp +++ b/include/processors/processing_processor_mnn_volume.hpp @@ -40,7 +40,8 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &volumeData, std::vector &modelFile, - const std::vector *parameters ) override; + const std::vector *parameters, + const ExecutionContext &execCtx ) override; private: /** Run MNN processing on volume data diff --git a/include/processors/processing_processor_render.hpp b/include/processors/processing_processor_render.hpp new file mode 100644 index 0000000..dd5afb3 --- /dev/null +++ b/include/processors/processing_processor_render.hpp @@ -0,0 +1,282 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include "processing_processor.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ + class RenderProcessor : public ProcessingProcessor + { + public: + RenderProcessor() {} + ~RenderProcessor() override = default; + + ProcessingResult StartProcessing( std::vector> &chunkhashes, + const sgns::IoDeclaration &proc, + std::vector &imageData, + std::vector &modelFile, + const std::vector *parameters, + const ExecutionContext &execCtx ) override; + + /// Device-type filter (DISCRETE_GPU/INTEGRATED_GPU only). Public so + /// vulkan_gpu_probe.cpp's HasUsableVulkanDevice() can reuse the exact + /// same predicate instead of duplicating it (avoids drift risk). + static bool IsAcceptable( VkPhysicalDeviceType type ); + + /// Lazy-init Vulkan instance/device (idempotent, double-checked locking). + /// Public so CapabilityValidator::BuildSnapshot can ensure the device exists + /// before querying its properties (D-10). + bool InitializeContext(); + + /// Returns the physical device handle after InitializeContext() has succeeded. + /// Returns VK_NULL_HANDLE if context not yet initialized. + VkPhysicalDevice GetPhysicalDevice() const { return m_physicalDevice; } + + private: + /// One parsed SPIR-V shader stage, inverted from ProcessingManager.cpp's + /// SerializeCompiledStages( stages, entryPoints ) wire format (plan 03-01). + struct ParsedStage + { + sgns::Stage stage; + std::string entry_point; + std::vector spirv; + }; + + /// Result of resolving a render pass's declared uniforms (D-29/D-30): + /// packed bytes plus the push-constant-vs-descriptor-set decision. + struct ResolvedUniforms + { + std::vector packedBytes; + bool pushConstant = true; + }; + + static VkDeviceSize LargestDeviceLocalHeap( VkPhysicalDevice device ); + + /// Exact byte-for-byte inverse of ProcessingManager.cpp's + /// SerializeCompiledStages( stages, entryPoints ). Bounds-checks every + /// read against modelFile.size() -- never reads past the end of a + /// malformed/truncated buffer. + static bool ParseCompiledStages( const std::vector &modelFile, + std::vector &outStages, + ProcessingResult &errorOut ); + + /// Exact byte-for-byte inverse of ProcessingManager.cpp's + /// SerializeRenderPassConfig(...). This is the ONLY method that ever + /// produces sgns::RenderTarget/PipelineState/VertexLayoutEntry/uniform-map + /// instances inside RenderProcessor, and the only source of + /// outDataTransformCount -- RenderProcessor has no other path to a + /// Pass/RenderShaderConfig object at all. + static bool ParseRenderPassConfig( + const std::vector &imageData, + sgns::RenderTarget &outTarget, + boost::optional &outPipelineState, + std::vector &outVertexLayout, + boost::optional> &outUniforms, + std::vector &outVertexBytes, + bool &outHasIndex, + sgns::IndexType &outIndexType, + std::vector &outIndexBytes, + uint32_t &outDataTransformCount, + ProcessingResult &errorOut ); + + /// Resolves each declared uniform's value -- either a literal + /// RenderShaderUniform.value or a parameter:-sourced value read from + /// `parameters` -- and packs the resolved bytes per each uniform's + /// declared DataType, 16-byte-aligned per uniform (std430-avoidance, + /// see 03-03-PLAN.md objective). Sets pushConstant per D-29's fixed + /// 128-byte threshold and D-30's all-or-nothing rule. + static bool ResolveUniforms( + const boost::optional> &uniforms, + const std::vector *parameters, + ResolvedUniforms &outResolved, + ProcessingResult &errorOut ); + + /// Constructs a ProcessingResult populated via the given stage/message + /// (D-25/D-26). Does NOT call RunTeardown() itself -- every caller must + /// call RunTeardown() immediately before or after, per this plan's + /// Task 2 convention. Static -- needs no instance state, so the + /// also-static ParseCompiledStages()/ParseRenderPassConfig()/ + /// ResolveUniforms() can call it directly, alongside the non-static + /// CheckFormatSupport()/CreateBufferDedicated()/CreateImageDedicated(). + static ProcessingResult MakeError( sgns::sgprocessing::ProcessingErrorStage stage, const std::string &message ); + + /// Queries vkGetPhysicalDeviceFormatProperties and checks that + /// requiredFeature is present in optimalTilingFeatures (RESEARCH.md + /// Pitfall 7) -- fails with a structured FORMAT_UNSUPPORTED error + /// naming the specific format, rather than letting image/render-pass + /// creation fail with an opaque VkResult or misbehave silently. + bool CheckFormatSupport( VkFormat format, VkFormatFeatureFlagBits requiredFeature, ProcessingResult &errorOut ); + + /// Allocates a VkBuffer with its own dedicated VkDeviceMemory + /// allocation (D-18/D-19), sized exactly to the buffer's memory + /// requirements -- no sub-allocation. Registers automatic teardown + /// via PushTeardown() on success; destroys the buffer itself (but not + /// via the teardown stack, since it isn't registered yet) on a + /// partial-failure path (D-24). + bool CreateBufferDedicated( VkDeviceSize size, + VkBufferUsageFlags usage, + VkMemoryPropertyFlags properties, + VkBuffer &outBuffer, + VkDeviceMemory &outMemory, + ProcessingResult &errorOut ); + + /// Allocates a VkImage with its own dedicated VkDeviceMemory + /// allocation (D-18/D-19), identical in shape to CreateBufferDedicated. + bool CreateImageDedicated( const VkImageCreateInfo &imageInfo, + VkMemoryPropertyFlags properties, + VkImage &outImage, + VkDeviceMemory &outMemory, + ProcessingResult &errorOut ); + + /// Sane, conservative maximum render_target width/height (Security Domain + /// V5's DoS concern -- the schema only enforces minimum:1, no maximum). 8192 + /// is a generous-but-bounded default; no specific value is mandated by + /// REQUIREMENTS.md/CONTEXT.md. + static constexpr uint32_t kMaxRenderDimension = 8192; + + /// Builds the offscreen VkRenderPass (color+depth, explicit CLEAR load ops + /// on both, VK_SAMPLE_COUNT_1_BIT unconditionally per DETV-02). Bounds-checks + /// target.get_width()/get_height() against kMaxRenderDimension and format- + /// support-checks both formats via CheckFormatSupport() (RESEARCH.md Pitfall + /// 7) before creating anything. Sets m_renderWidth/m_renderHeight for + /// plan 03-04 Task 2's BuildPipeline() to consume for its fixed viewport. + bool BuildRenderPass( const sgns::RenderTarget &target, ProcessingResult &errorOut ); + + /// Builds the offscreen VkFramebuffer: a color+depth VkImage/VkImageView + /// pair (each image via CreateImageDedicated(), DEVICE_LOCAL) referencing + /// m_renderPass. Must be called after BuildRenderPass() succeeds. + bool BuildFramebuffer( const sgns::RenderTarget &target, ProcessingResult &errorOut ); + + /// Builds the complete graphics pipeline: one VkShaderModule/shader-stage + /// per parsed stage (real per-stage entry point, never a hard-coded + /// "main"), fixed (never dynamic) pipeline state from pipelineState (or + /// schema-documented defaults when absent), the auto-computed vertex + /// input binding/attributes from vertexLayout, and a pipeline layout + /// branching on uniforms.pushConstant (D-29/D-30's all-or-nothing rule). + bool BuildPipeline( const std::vector &stages, + const std::vector &vertexLayout, + const boost::optional &pipelineState, + const ResolvedUniforms &uniforms, + ProcessingResult &errorOut ); + + /// Validates vertexBytes.size() % stride == 0 and (if hasIndex) + /// indexBytes.size() % index-type-byte-size == 0 BEFORE any buffer is + /// created (closes T-03-03-02 -- out-of-bounds vkCmdDraw(Indexed) read), + /// then uploads vertex/index/uniform bytes into dedicated HOST_VISIBLE| + /// HOST_COHERENT buffers via direct vkMapMemory/memcpy/vkUnmapMemory (D-20/ + /// D-21 -- no staging+device-local path, no manual flush). Only allocates + /// m_uniformBuffer (descriptor-set path) when uniforms.pushConstant is + /// false; the push-constant path needs no VkBuffer (bytes copied directly + /// from ResolvedUniforms::packedBytes at record time via + /// RecordAndSubmit()/vkCmdPushConstants). + bool UploadBuffers( const std::vector &vertexBytes, + bool hasIndex, + sgns::IndexType indexType, + const std::vector &indexBytes, + uint32_t stride, + const ResolvedUniforms &uniforms, + ProcessingResult &errorOut ); + + /// Records and submits ONE command buffer: begin render pass (clears from + /// target.get_clear_color()/get_clear_depth()) -> bind pipeline/vertex/ + /// index buffers -> push constants or bind descriptor set -> draw(Indexed) + /// -> end render pass -> (Pitfall 4) record the vkCmdCopyImageToBuffer + /// readback copy INTO THIS SAME command buffer, immediately after + /// vkCmdEndRenderPass and before vkEndCommandBuffer -- no second command + /// buffer/submission, no extra image-layout-transition barrier (the render pass's + /// color attachment finalLayout is already VK_IMAGE_LAYOUT_TRANSFER_SRC_ + /// OPTIMAL, plan 03-04) -- then vkQueueSubmit and a synchronous + /// vkDeviceWaitIdle (D-23). Allocates m_stagingBuffer/m_stagingMemory + /// (the readback destination Readback() later maps) as part of recording + /// this copy. + bool RecordAndSubmit( const sgns::RenderTarget &target, ProcessingResult &errorOut ); + + /// Maps m_stagingBuffer (already populated by RecordAndSubmit()'s + /// vkCmdCopyImageToBuffer + vkDeviceWaitIdle) and copies its bytes into + /// outBytes -- no vkInvalidateMappedMemoryRanges call (HOST_COHERENT, + /// D-20). Must be called after RecordAndSubmit() succeeds. + bool Readback( const sgns::RenderTarget &target, std::vector &outBytes, ProcessingResult &errorOut ); + + /// Bytes per pixel for a given color attachment format. RGBA8 -> 4, + /// RGB8 -> 3. + static uint32_t ColorFormatByteSize( sgns::ColorFormat fmt ); + + static VkFormat ToVkFormat( sgns::ColorFormat fmt ); + static VkFormat ToVkFormat( sgns::DepthFormat fmt ); + static VkFormat ToVkFormat( sgns::VertexLayoutFormat fmt ); + static VkPrimitiveTopology ToVkTopology( sgns::Topology t ); + static VkCullModeFlags ToVkCullMode( sgns::CullMode c ); + static VkFrontFace ToVkFrontFace( sgns::FrontFace f ); + static VkBool32 ToVkBool( sgns::DepthTest d ); + + /// Byte size of a single scalar vertex-attribute component (this plan's + /// documented scalar-component reading of vertex_layout -- see + /// 03-04-PLAN.md's objective). FLOAT32/INT32 -> 4, FLOAT16 -> 2. + static uint32_t VertexFormatByteSize( sgns::VertexLayoutFormat f ); + + VkInstance m_instance{VK_NULL_HANDLE}; + VkPhysicalDevice m_physicalDevice{VK_NULL_HANDLE}; + VkDevice m_device{VK_NULL_HANDLE}; + VkQueue m_queue{VK_NULL_HANDLE}; + /// Graphics queue family index InitializeContext() resolved for m_queue -- + /// stored so RecordAndSubmit()'s VkCommandPool creation reuses the same + /// already-selected graphics queue family instead of re-running device + /// queue-family selection. + uint32_t m_queueFamilyIndex{0}; + bool m_contextInitialized{false}; + + /// render_target width/height, set by BuildRenderPass() after its bounds + /// check succeeds -- consumed by Task 2's BuildPipeline() for its fixed + /// (never a runtime-settable pipeline attribute, per D-22) viewport/ + /// scissor, since VkGraphicsPipelineCreateInfo requires a concrete + /// VkPipelineViewportStateCreateInfo when no dynamic viewport/scissor + /// state is used. + uint32_t m_renderWidth{0}; + uint32_t m_renderHeight{0}; + + VkRenderPass m_renderPass{VK_NULL_HANDLE}; + VkFramebuffer m_framebuffer{VK_NULL_HANDLE}; + VkImage m_colorImage{VK_NULL_HANDLE}, m_depthImage{VK_NULL_HANDLE}; + VkImageView m_colorView{VK_NULL_HANDLE}, m_depthView{VK_NULL_HANDLE}; + VkDeviceMemory m_colorMemory{VK_NULL_HANDLE}, m_depthMemory{VK_NULL_HANDLE}; + + VkPipelineLayout m_pipelineLayout{VK_NULL_HANDLE}; + VkPipeline m_pipeline{VK_NULL_HANDLE}; + VkDescriptorSetLayout m_descriptorSetLayout{VK_NULL_HANDLE}; + VkDescriptorPool m_descriptorPool{VK_NULL_HANDLE}; + VkDescriptorSet m_descriptorSet{VK_NULL_HANDLE}; + + VkBuffer m_vertexBuffer{VK_NULL_HANDLE}, m_indexBuffer{VK_NULL_HANDLE}; + VkBuffer m_uniformBuffer{VK_NULL_HANDLE}, m_stagingBuffer{VK_NULL_HANDLE}; + VkDeviceMemory m_vertexMemory{VK_NULL_HANDLE}, m_indexMemory{VK_NULL_HANDLE}; + VkDeviceMemory m_uniformMemory{VK_NULL_HANDLE}, m_stagingMemory{VK_NULL_HANDLE}; + + VkCommandPool m_commandPool{VK_NULL_HANDLE}; + VkCommandBuffer m_commandBuffer{VK_NULL_HANDLE}; + + bool m_hasIndexBuffer{false}; + sgns::IndexType m_indexType{sgns::IndexType::UINT32}; + uint32_t m_vertexCount{0}; + uint32_t m_indexCount{0}; + + /// Set by UploadBuffers() from the ResolvedUniforms passed into it -- + /// RecordAndSubmit()'s declared signature (target, errorOut) carries no + /// uniform data of its own, so the push-constant bytes/decision must be + /// stored here for RecordAndSubmit()'s vkCmdPushConstants call. The + /// descriptor-set path needs no equivalent member: m_descriptorSet + /// (already built by BuildPipeline()) is bound directly. + bool m_usePushConstant{false}; + std::vector m_pushConstantBytes; + }; +} diff --git a/include/processors/vulkan_gpu_probe.hpp b/include/processors/vulkan_gpu_probe.hpp new file mode 100644 index 0000000..0456ca3 --- /dev/null +++ b/include/processors/vulkan_gpu_probe.hpp @@ -0,0 +1,17 @@ +#pragma once + +namespace sgns::sgprocessing +{ + /// Runtime probe answering "does this host have at least one usable Vulkan + /// device?", mirroring RenderProcessor::IsAcceptable's DISCRETE_GPU/ + /// INTEGRATED_GPU device-type filter (D-32). Builds and immediately tears + /// down its own throwaway VkInstance -- it never creates a VkDevice and + /// never touches RenderProcessor's own Vulkan state. + /// + /// Callers MUST treat a `false` return as "skip GPU-dependent work" (e.g. + /// via GTEST_SKIP()), never as a hard error -- a GPU-less host is an + /// expected, valid environment (D-34), not a failure condition. + /// + /// Never throws. + bool HasUsableVulkanDevice(); +} diff --git a/include/shaders/shader_compiler.hpp b/include/shaders/shader_compiler.hpp new file mode 100644 index 0000000..70c6798 --- /dev/null +++ b/include/shaders/shader_compiler.hpp @@ -0,0 +1,72 @@ +#ifndef SGPROCESSINGMANAGER_SHADER_COMPILER_HPP +#define SGPROCESSINGMANAGER_SHADER_COMPILER_HPP + +#include +#include +#include +#include + +#include +#include +#include + +namespace sgns::sgprocessing +{ + /** + * A single compiled-and-validated shader stage: the SPIR-V words that + * survived the mandatory spirv-val gate, plus which pipeline stage they + * target. + */ + struct CompiledShaderStage + { + std::vector spirv; + sgns::Stage stage; + }; + + /** + * Standalone, Vulkan-device-free GLSL->SPIR-V compiler + mandatory + * SPIRV-Tools validation gate. No VkInstance/VkDevice/VkPhysicalDevice + * member anywhere -- pure CPU-side text/bytecode transformation and + * validation, fully unit-testable in isolation from any Vulkan context. + * + * Every code path -- GLSL-compiled or directly-submitted SPIR-V -- passes + * through spvtools::SpirvTools::Validate() before CompileAndValidate() + * can ever return success. shaderc's CompileGlslToSpv() does NOT run + * spirv-val internally; "shaderc compiled it" must never be conflated + * with "SPIRV-Tools validated it." + */ + class ShaderCompiler + { + public: + enum class Error + { + COMPILE_FAILED = 1, + VALIDATION_FAILED = 2 + }; + + /** + * Compile (if GLSL) and unconditionally validate a single shader + * stage's source bytes. + * + * @param source_bytes raw source bytes -- GLSL text if type == GLSL, + * raw SPIR-V bytes if type == SPIRV + * @param stage which pipeline stage this shader targets + * @param type whether source_bytes is GLSL text or raw SPIR-V bytes + * @param entry_point the shader's entry point function name (only + * meaningful for the GLSL path) + * @return validated SPIR-V words + stage on success, or a structured + * Error on failure -- never throws + */ + outcome::result CompileAndValidate( const std::vector &source_bytes, + sgns::Stage stage, + sgns::ShaderSourceType type, + const std::string &entry_point ); + + private: + sgns::sgprocmanager::Logger m_logger = sgns::sgprocmanager::createLogger( "ShaderCompiler" ); + }; +} + +OUTCOME_HPP_DECLARE_ERROR_2( sgns::sgprocessing, ShaderCompiler::Error ); + +#endif // SGPROCESSINGMANAGER_SHADER_COMPILER_HPP diff --git a/include/util/diff_utils.hpp b/include/util/diff_utils.hpp new file mode 100644 index 0000000..a57cc99 --- /dev/null +++ b/include/util/diff_utils.hpp @@ -0,0 +1,146 @@ +#ifndef SGPROCMGR_DIFF_UTILS_HPP +#define SGPROCMGR_DIFF_UTILS_HPP + +#include +#include +#include + +#include "Parameter.hpp" +#include "ParameterType.hpp" + +namespace sgns::sgprocmanagerdiff +{ + /// Relative-delta denominator floor -- avoids divide-by-zero near + /// zero-valued float elements. Extracted verbatim from capture_diff.cpp's + /// former unnamed-namespace constant (Phase 10, Plan 10-05) so both the + /// CLI tool and this shared library read the identical value. + constexpr float kRelativeDeltaEpsilonFloor = 1e-6f; + + /// D-04 fallback: fixed float relative-delta threshold used by + /// ComputeFloat32Diff's exceeding-count check and by + /// IsFloatChunkWithinTolerance when no valid "quantScale" is declared. + /// Exported (not file-local) so capture_diff.cpp and any runtime + /// validator consumer read the exact same symbol -- guarantees zero + /// behavioral drift between the offline CLI tool and Plan 15-02's + /// runtime comparison. + constexpr double kDefaultFloatRelativeThreshold = 1e-4; + + /// D-04 fallback: fixed byte absolute-delta threshold used by + /// ComputeUint8Diff's exceeding-count check and by + /// IsByteChunkWithinTolerance when no valid "byteQuantMode" is declared. + constexpr int kDefaultByteAbsoluteThreshold = 1; + + /// Whole-buffer per-element divergence summary (DIFF-01/DIFF-02). + /// Extracted verbatim from capture_diff.cpp's former unnamed-namespace + /// struct of the same name/shape. + struct ElementDiffStats + { + size_t elementCount = 0; + double maxAbsDelta = 0.0; + double maxRelDelta = 0.0; + int64_t maxUlpDistance = 0; + double percentExceedingThreshold = 0.0; + bool sizeMismatch = false; + }; + + /// Standard ordered-integer bit-reinterpretation technique for float ULP + /// distance. Extracted verbatim from capture_diff.cpp. + int64_t OrderedFloatBits( float f ); + + /// Extracted verbatim from capture_diff.cpp. + int64_t UlpDistanceFloat( float a, float b ); + + /// Computes per-element float32 divergence stats between two raw byte + /// buffers (each buffer's size must be a multiple of sizeof(float)). + /// Extracted verbatim from capture_diff.cpp's former unnamed-namespace + /// function of the same name -- behavior-neutral relocation, byte-for-byte + /// identical output to the pre-extraction version on the same inputs. + ElementDiffStats ComputeFloat32Diff( const std::vector &a, const std::vector &b ); + + /// Computes per-element uint8 divergence stats between two raw byte + /// buffers. Extracted verbatim from capture_diff.cpp's former + /// unnamed-namespace function of the same name. + ElementDiffStats ComputeUint8Diff( const std::vector &a, const std::vector &b ); + + /// Element-type hint for a chunk's raw output data, used to decide which + /// tolerance-derivation function (float vs. byte) applies. + enum class ChunkElementType + { + FLOAT32, + UINT8 + }; + + /// Phase 15 (XNODE-02): resolves whether a chunk's raw output data should + /// be treated as float32 or uint8 for tolerance-comparison purposes. + /// + /// Returns UINT8 only when a job schema-declares "byteQuantMode" (INT + /// type, integer value in [0, 8]) with a value greater than 0 -- i.e. the + /// byte-quantization path is actually active for this job. Returns + /// FLOAT32 in every other case: "quantScale" declared instead, a + /// "byteQuantMode" of exactly 0 declared (the byte-identity no-op case, + /// per quantization.hpp's own doc comments), or neither declared. This is + /// a documented, deliberate default -- float32 is the more general/common + /// MNN numeric case, and the render byte path has historically been a + /// no-op per Phase 12/14's own doc comments -- not an attempt to solve + /// per-output element-type inference generally. + /// + /// @param parameters Job schema's generic parameters array, or nullptr. + /// @return UINT8 only when byteQuantMode is validly declared with a value + /// > 0; FLOAT32 otherwise. + ChunkElementType ResolveChunkElementTypeHint( const std::vector *parameters ); + + /// Phase 15 (XNODE-02, D-03/D-04): determines whether two float32 chunk + /// buffers are numerically "close enough" to be treated as a tolerant + /// match rather than a genuine cross-node divergence. + /// + /// D-03: when the job validly declares "quantScale" = S, the bound is + /// derived from the quantization grid step (2/S, two grid steps of + /// margin -- the same "one full step of margin" philosophy + /// quantization.cpp's own S=2^15-over-2^14 derivation history documents) + /// and compared against ComputeFloat32Diff's maxAbsDelta. + /// + /// D-04: when no valid "quantScale" is declared, falls back to + /// capture_diff's existing kDefaultFloatRelativeThreshold (1e-4, + /// relative), via ComputeFloat32Diff's own percentExceedingThreshold + /// stat (zero-elements-may-exceed policy, not a percentage-based bar). + /// + /// @param a First chunk's raw float32 bytes. + /// @param b Second chunk's raw float32 bytes. + /// @param parameters Job schema's generic parameters array, or nullptr. + /// @param statsOut Populated with ComputeFloat32Diff's full stats, + /// regardless of the boolean result. + /// @return false immediately (statsOut.sizeMismatch=true) if a/b differ + /// in length; otherwise true iff within the D-03/D-04 tolerance. + bool IsFloatChunkWithinTolerance( const std::vector &a, + const std::vector &b, + const std::vector *parameters, + ElementDiffStats &statsOut ); + + /// Phase 15 (XNODE-02, D-03/D-04): determines whether two uint8 chunk + /// buffers are numerically "close enough" to be treated as a tolerant + /// match rather than a genuine cross-node divergence. + /// + /// D-03: when the job validly declares "byteQuantMode" = N, the bound is + /// derived from the quantization mask width ((1< &a, + const std::vector &b, + const std::vector *parameters, + ElementDiffStats &statsOut ); +} // namespace sgns::sgprocmanagerdiff + +#endif diff --git a/include/util/quantization.hpp b/include/util/quantization.hpp new file mode 100644 index 0000000..6620276 --- /dev/null +++ b/include/util/quantization.hpp @@ -0,0 +1,153 @@ +#ifndef SGPROCMGR_QUANTIZATION_HPP +#define SGPROCMGR_QUANTIZATION_HPP + +#include +#include +#include + +#include "Parameter.hpp" +#include "ParameterType.hpp" + +namespace sgns::sgprocmanagerquant +{ + /// Phase 14 (QUANT-CFG-01/02, D-01/D-02/D-04/D-05): resolves a job + /// schema-declared "quantScale" entry from the generic `parameters` array, + /// mirroring the existing find-by-name-in-parameters convention + /// (ParseLayout / ResolveUniforms). + /// + /// Falls back to the exact v2.1 constant 32768.0f (2^15) -- no warning + /// logged, no job rejection -- when `parameters` is null, no entry named + /// "quantScale" of type FLOAT exists, its declared default value is not a + /// JSON number, or the numeric value is not a strictly positive power of + /// two (D-05's mandatory validation, guaranteeing the exact float + /// round-trip property D-03's round(x*S)/S formula relies on can never be + /// silently violated by a bad schema value). + /// + /// @param parameters Job schema's generic parameters array, or nullptr. + /// @return The validated, schema-declared scale, or 32768.0f on any + /// invalid/missing declaration. + float ResolveQuantScale( const std::vector *parameters ); + + /// Phase 14 (QUANT-CFG-01/02, D-02/D-07/D-08): resolves a job + /// schema-declared "byteQuantMode" entry from the generic `parameters` + /// array, same lookup convention as ResolveQuantScale. + /// + /// Falls back to 0 (the exact v2.1 byte-identity no-op) -- no warning, no + /// job rejection -- when `parameters` is null, no entry named + /// "byteQuantMode" of type INT exists, its declared default value is not + /// a JSON integer, or the integer value falls outside the inclusive range + /// [0, 8]. N=8 (masking all 8 bits) is a valid, non-fallback boundary + /// value by design (D-07/D-08); N=9 and above fall back to 0. + /// + /// @param parameters Job schema's generic parameters array, or nullptr. + /// @return The validated, schema-declared mask-bit count in [0, 8], or 0 + /// on any invalid/missing declaration. + int ResolveByteQuantMode( const std::vector *parameters ); + + + /// Phase 12 real implementation (D-03 through D-09): IEEE-754 special-value + /// canonicalization followed by fixed-precision scale-round-cast quantization. + /// + /// Canonicalization (evaluated strictly before any rounding arithmetic, D-07): + /// - Denormals (both signs) flush to canonical +0.0 (0x00000000), D-07/D-08. + /// - NaN (any payload/sign/signaling bit) canonicalizes to the hardcoded + /// quiet-NaN bit pattern 0x7FC00000 (D-09) -- never + /// std::numeric_limits::quiet_NaN(), since that is not guaranteed + /// to be bit-identical across compilers/platforms. + /// - +Inf / -Inf canonicalize to two *distinct* fixed bit patterns, + /// 0x7F800000 / 0xFF800000 respectively (D-06) -- never collapsed to one + /// value, so a wrong-sign divergence stays visible to SECV-01's + /// counter-test. + /// - -0.0 and +0.0 both collapse to the single canonical zero bit pattern + /// 0x00000000, sign discarded (D-08). + /// + /// Rounding (only reached once every canonicalization branch above has been + /// evaluated and found not to apply): q = round(x * S) / S, with + /// S = 2^15 (32768.0f) -- a power-of-two scale factor for exact float + /// round-tripping. The tolerance is a single fixed absolute epsilon (D-04) + /// -- not magnitude-adaptive, not relative/ULP-based, and not + /// schema-configurable. + /// + /// Original Phase 12 derivation (D-05): S = 2^20 (1048576.0f), grid step + /// ~9.5367431640625e-07, chosen for a ~9.14x margin over Phase 11's + /// measured cross-machine (Mac vs Windows) MNN float32 divergence: + /// maxAbsDelta ≈ 1.043081283569336e-07, maxRelDelta ≈ 7.269731577252969e-05, + /// maxUlpDistance = 768 (512-element float32 MNN fixture; see + /// 11-CAPTURE-RESULTS.md). + /// + /// Phase 13 Plan 13-04 gap-closure revision (this constant's current + /// value): Phase 13's own fresh re-validation (13-SCOPE-BOUNDARY.md) + /// measured a post-quantization maxAbsDelta of exactly 9.5367431640625e-07 + /// -- one full old-grid step -- with 12 of 15 MNN chunk hashes still + /// diverging cross-hardware at the old S=2^20 grid, direct evidence the + /// original ~9x margin was insufficient against per-element + /// grid-boundary tie-break divergence for this fixture's real data. + /// + /// A local binary search over power-of-two S values (13-04-PLAN.md Task 1, + /// revised approach) against processing_conformance_security_test's + /// Secv01CounterTest.MnnCorruptedModelStillDiverges bracketed a hard + /// boundary: S=2^15 (grid step 3.0517578125e-05) passes -- the corrupted + /// MNN model's artifactId still diverges from the correct model's, as + /// SECV-01 requires -- while S=2^14 (grid step 6.103515625e-05) FAILS + /// deterministically (the corrupted model's post-quantization artifactId + /// collides bit-for-bit with the correct model's, confirmed by re-running + /// twice, not flaky). S=2^15 was chosen over S=2^14 specifically to keep + /// one full power-of-two step of margin above this confirmed failure + /// boundary rather than sitting at the exact edge (floating-point + /// behavior can vary subtly build-to-build). S=2^15's grid step is 32x + /// the original S=2^20 grid step and ~292x Phase 11's originally-measured + /// maxAbsDelta -- substantially reducing (not mathematically eliminating) + /// the per-element grid-boundary tie-break collision probability for this + /// fixture's real values, while every SECV-01 case (corrupted MNN model, + /// wrong render shader constant) still passes. + /// + /// This is a probabilistic engineering mitigation, not a one-shot + /// guaranteed solution: a fixed rounding grid cannot mathematically + /// guarantee zero cross-hardware divergence for arbitrary per-element + /// deltas that happen to land arbitrarily close to a rounding boundary -- + /// it only reduces the probability of that happening for this fixture's + /// actual values. See 13-SCOPE-BOUNDARY.md's Refit section (Plan 13-05) + /// for the fresh empirical cross-machine outcome this constant change is + /// validated against. + /// + /// Phase 14 (QUANT-CFG-01/02): `S` is now schema-configurable via the + /// caller-resolved `scale` argument, produced by calling + /// ResolveQuantScale() once per StartProcessing() invocation. 32768.0f + /// remains the exact fallback when nothing valid is schema-declared, and + /// the D-03 round(x*S)/S formula plus the D-06..D-09 canonicalization + /// branch order above are entirely unchanged by this addition -- this + /// paragraph documents schema-configurability, it does not revise or + /// contradict the S=2^15 derivation history above it. + /// + /// @param data Pointer to a float buffer to quantize in place. + /// @param count Number of float elements in the buffer. + /// @param scale The resolved scale S to use (see ResolveQuantScale()). + void QuantizeFloatBuffer( float *data, size_t count, float scale ); + + /// Phase 12 deliberate identity pass-through for the render uint8 path. + /// + /// This is a considered design decision for this phase, not an inherited + /// Phase 10 placeholder: Phase 11's empirical render fixture data + /// (256-element uint8 RGBA/RGB pixel output, Mac vs Windows) showed + /// contentHashMatch: true with maxAbsDelta/maxRelDelta/maxUlpDistance all + /// 0.0 -- no observed cross-hardware divergence in the uint8 render path + /// this milestone's fixtures exercise (see 11-CAPTURE-RESULTS.md). Applying + /// a lossy tolerance-band here with no empirical justification would only + /// enlarge the space of results indistinguishable from a correct one, so + /// this stays byte-identity until new fixture data shows otherwise. + /// + /// Phase 14 (QUANT-CFG-01/02, D-06/D-07): the mask is now schema- + /// configurable via the caller-resolved `maskBits` argument, produced by + /// calling ResolveByteQuantMode() once per StartProcessing() invocation. + /// `maskBits <= 0` (D-07's N=0/absent case) remains the exact v2.1 + /// byte-identity no-op; otherwise the low `maskBits` bits of every byte + /// are cleared (D-06's bit-masking technique, `value &= ~((1< + $ + $ +) + +target_link_libraries(SGArtifacts + PUBLIC + sgprocmanagersha +) + +sgnus_install(SGArtifacts) diff --git a/src/artifacts/artifact_serializer.cpp b/src/artifacts/artifact_serializer.cpp new file mode 100644 index 0000000..c4ee121 --- /dev/null +++ b/src/artifacts/artifact_serializer.cpp @@ -0,0 +1,317 @@ +/** + * Implementation of deterministic binary serialization for Artifact and + * ExecutionManifest structs. Fixed-field, little-endian layout per D-05. + * + * Follows the same memcpy-based approach as SerializeRenderPassConfig() in + * ProcessingManager.cpp: pre-allocate zero-filled vector, then copy each + * field at its documented fixed offset. + */ + +#include "artifacts/artifact_serializer.hpp" +#include +#include + +namespace sgns::sgprocessing +{ + + // ──────────────────────────────────────────────────────────────────── + // Artifact Serialization + // ──────────────────────────────────────────────────────────────────── + + std::vector SerializeArtifact( const Artifact &artifact ) + { + // Fixed-field layout offsets for Artifact (D-05). + // All fields at known offsets; multi-byte values in native little-endian. + static constexpr size_t OFF_resourceName = 0; + static constexpr size_t OFF_artifactId = 256; + static constexpr size_t OFF_passId = 288; + static constexpr size_t OFF_outputBinding = 544; + static constexpr size_t OFF_dataType = 800; + static constexpr size_t OFF_format = 864; + static constexpr size_t OFF_width = 928; + static constexpr size_t OFF_height = 932; + static constexpr size_t OFF_depth = 936; + static constexpr size_t OFF_byteSize = 940; + static constexpr size_t OFF_mediaType = 948; + static constexpr size_t OFF_contentHash = 1076; + static constexpr size_t OFF_chunkHashCount = 1108; + static constexpr size_t OFF_chunkHashes = 1112; + + // Pre-allocate zero-filled buffer at fixed total size + std::vector out( ARTIFACT_SERIALIZED_SIZE, 0 ); + + // --- String fields: null-padded copy up to MAX-1 --- + auto copyStr = [&]( size_t offset, const char *src, size_t maxLen ) + { + size_t len = std::min( std::strlen( src ), maxLen - 1 ); + std::memcpy( out.data() + offset, src, len ); + // Rest is already zero from pre-allocation + }; + + copyStr( OFF_resourceName, artifact.resourceName, MAX_RESOURCE_NAME ); + copyStr( OFF_passId, artifact.passId, MAX_RESOURCE_NAME ); + copyStr( OFF_outputBinding, artifact.outputBinding, MAX_RESOURCE_NAME ); + copyStr( OFF_dataType, artifact.dataType, 64 ); + copyStr( OFF_format, artifact.format, 64 ); + copyStr( OFF_mediaType, artifact.mediaType, MAX_MEDIA_TYPE ); + + // --- Fixed-size byte arrays --- + std::memcpy( out.data() + OFF_artifactId, artifact.artifactId, SHA256_HASH_SIZE ); + std::memcpy( out.data() + OFF_contentHash, artifact.contentHash, SHA256_HASH_SIZE ); + + // --- Integer fields (native little-endian, no byte swap needed on x86/ARM) --- + std::memcpy( out.data() + OFF_width, &artifact.width, sizeof( uint32_t ) ); + std::memcpy( out.data() + OFF_height, &artifact.height, sizeof( uint32_t ) ); + std::memcpy( out.data() + OFF_depth, &artifact.depth, sizeof( uint32_t ) ); + std::memcpy( out.data() + OFF_byteSize, &artifact.byteSize, sizeof( uint64_t ) ); + std::memcpy( out.data() + OFF_chunkHashCount, &artifact.chunkHashCount, sizeof( uint32_t ) ); + + // --- Chunk hashes: only chunkHashCount * 32 meaningful bytes --- + std::memcpy( out.data() + OFF_chunkHashes, artifact.chunkHashes, + artifact.chunkHashCount * SHA256_HASH_SIZE ); + // Rest stays zero from pre-allocation + + return out; + } + + bool DeserializeArtifact( const std::vector &bytes, Artifact &out ) + { + if ( bytes.size() != ARTIFACT_SERIALIZED_SIZE ) + { + return false; + } + + static constexpr size_t OFF_resourceName = 0; + static constexpr size_t OFF_artifactId = 256; + static constexpr size_t OFF_passId = 288; + static constexpr size_t OFF_outputBinding = 544; + static constexpr size_t OFF_dataType = 800; + static constexpr size_t OFF_format = 864; + static constexpr size_t OFF_width = 928; + static constexpr size_t OFF_height = 932; + static constexpr size_t OFF_depth = 936; + static constexpr size_t OFF_byteSize = 940; + static constexpr size_t OFF_mediaType = 948; + static constexpr size_t OFF_contentHash = 1076; + static constexpr size_t OFF_chunkHashCount = 1108; + static constexpr size_t OFF_chunkHashes = 1112; + + // Zero the output struct + out = Artifact{}; + + // Copy string fields + std::memcpy( out.resourceName, bytes.data() + OFF_resourceName, MAX_RESOURCE_NAME ); + out.resourceName[MAX_RESOURCE_NAME - 1] = '\0'; // force null terminator + std::memcpy( out.passId, bytes.data() + OFF_passId, MAX_RESOURCE_NAME ); + out.passId[MAX_RESOURCE_NAME - 1] = '\0'; + std::memcpy( out.outputBinding, bytes.data() + OFF_outputBinding, MAX_RESOURCE_NAME ); + out.outputBinding[MAX_RESOURCE_NAME - 1] = '\0'; + std::memcpy( out.dataType, bytes.data() + OFF_dataType, 64 ); + out.dataType[63] = '\0'; + std::memcpy( out.format, bytes.data() + OFF_format, 64 ); + out.format[63] = '\0'; + std::memcpy( out.mediaType, bytes.data() + OFF_mediaType, MAX_MEDIA_TYPE ); + out.mediaType[MAX_MEDIA_TYPE - 1] = '\0'; + + // Copy fixed-size byte arrays + std::memcpy( out.artifactId, bytes.data() + OFF_artifactId, SHA256_HASH_SIZE ); + std::memcpy( out.contentHash, bytes.data() + OFF_contentHash, SHA256_HASH_SIZE ); + + // Copy integers + std::memcpy( &out.width, bytes.data() + OFF_width, sizeof( uint32_t ) ); + std::memcpy( &out.height, bytes.data() + OFF_height, sizeof( uint32_t ) ); + std::memcpy( &out.depth, bytes.data() + OFF_depth, sizeof( uint32_t ) ); + std::memcpy( &out.byteSize, bytes.data() + OFF_byteSize, sizeof( uint64_t ) ); + std::memcpy( &out.chunkHashCount, bytes.data() + OFF_chunkHashCount, sizeof( uint32_t ) ); + + // Clamp chunk hash count + if ( out.chunkHashCount > 1024 ) + { + out.chunkHashCount = 1024; + } + + // Copy chunk hashes + std::memcpy( out.chunkHashes, bytes.data() + OFF_chunkHashes, + out.chunkHashCount * SHA256_HASH_SIZE ); + + return true; + } + + // ──────────────────────────────────────────────────────────────────── + // ExecutionManifest Serialization + // ──────────────────────────────────────────────────────────────────── + + std::vector SerializeManifest( const ExecutionManifest &manifest ) + { + // Fixed-field layout offsets for ExecutionManifest (D-05). + static constexpr size_t OFF_executionId = 0; + static constexpr size_t OFF_attemptId = 256; + static constexpr size_t OFF_taskId = 512; + static constexpr size_t OFF_subtaskId = 768; + static constexpr size_t OFF_passId = 1024; + static constexpr size_t OFF_executorIdentity = 1280; + static constexpr size_t OFF_modelIdentity = 1312; + static constexpr size_t OFF_tokenizerIdentity = 1344; + static constexpr size_t OFF_adapterIdentity = 1376; + static constexpr size_t OFF_shaderIdentity = 1408; + static constexpr size_t OFF_quantizationIdentity = 1440; + static constexpr size_t OFF_inputArtifactCount = 1472; + static constexpr size_t OFF_inputArtifactHashes = 1476; + static constexpr size_t OFF_outputArtifactCount = 3524; + static constexpr size_t OFF_outputArtifactHashes = 3528; + static constexpr size_t OFF_startTimeUsec = 5576; + static constexpr size_t OFF_endTimeUsec = 5584; + static constexpr size_t OFF_terminalState = 5592; + static constexpr size_t OFF_gpuMemoryUsedBytes = 5593; + static constexpr size_t OFF_outputBytesProduced = 5601; + static constexpr size_t OFF_wallClockUsec = 5609; + static constexpr size_t OFF_manifestHash = 5617; + + // Pre-allocate zero-filled buffer + std::vector out( MANIFEST_SERIALIZED_SIZE, 0 ); + + // --- CRITICAL: Save and zero manifestHash before serialization (D-04) --- + uint8_t savedManifestHash[SHA256_HASH_SIZE]; + std::memcpy( savedManifestHash, manifest.manifestHash, SHA256_HASH_SIZE ); + // We need to zero the manifestHash in a mutable copy — cast away const + // because SerializeManifest logically must mutate the hash field for correctness. + std::memset( const_cast( manifest.manifestHash ), 0, SHA256_HASH_SIZE ); + + auto copyStr = [&]( size_t offset, const char *src, size_t maxLen ) + { + size_t len = std::min( std::strlen( src ), maxLen - 1 ); + std::memcpy( out.data() + offset, src, len ); + }; + + // --- Identifier strings --- + copyStr( OFF_executionId, manifest.executionId, MAX_IDENTIFIER ); + copyStr( OFF_attemptId, manifest.attemptId, MAX_IDENTIFIER ); + copyStr( OFF_taskId, manifest.taskId, MAX_IDENTIFIER ); + copyStr( OFF_subtaskId, manifest.subtaskId, MAX_IDENTIFIER ); + copyStr( OFF_passId, manifest.passId, MAX_RESOURCE_NAME ); + + // --- Identity hashes --- + std::memcpy( out.data() + OFF_executorIdentity, manifest.executorIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.data() + OFF_modelIdentity, manifest.modelIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.data() + OFF_tokenizerIdentity, manifest.tokenizerIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.data() + OFF_adapterIdentity, manifest.adapterIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.data() + OFF_shaderIdentity, manifest.shaderIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.data() + OFF_quantizationIdentity, manifest.quantizationIdentity, SHA256_HASH_SIZE ); + + // --- Input artifact refs --- + std::memcpy( out.data() + OFF_inputArtifactCount, &manifest.inputArtifactCount, sizeof( uint32_t ) ); + uint32_t inCount = std::min( manifest.inputArtifactCount, static_cast( MAX_ARTIFACT_REFS ) ); + std::memcpy( out.data() + OFF_inputArtifactHashes, manifest.inputArtifactHashes, + inCount * SHA256_HASH_SIZE ); + + // --- Output artifact refs --- + std::memcpy( out.data() + OFF_outputArtifactCount, &manifest.outputArtifactCount, sizeof( uint32_t ) ); + uint32_t outCount = std::min( manifest.outputArtifactCount, static_cast( MAX_ARTIFACT_REFS ) ); + std::memcpy( out.data() + OFF_outputArtifactHashes, manifest.outputArtifactHashes, + outCount * SHA256_HASH_SIZE ); + + // --- Timing --- + std::memcpy( out.data() + OFF_startTimeUsec, &manifest.startTimeUsec, sizeof( int64_t ) ); + std::memcpy( out.data() + OFF_endTimeUsec, &manifest.endTimeUsec, sizeof( int64_t ) ); + + // --- Terminal state --- + out[OFF_terminalState] = static_cast( manifest.terminalState ); + + // --- Resource summary --- + std::memcpy( out.data() + OFF_gpuMemoryUsedBytes, &manifest.gpuMemoryUsedBytes, sizeof( uint64_t ) ); + std::memcpy( out.data() + OFF_outputBytesProduced, &manifest.outputBytesProduced, sizeof( uint64_t ) ); + std::memcpy( out.data() + OFF_wallClockUsec, &manifest.wallClockUsec, sizeof( uint64_t ) ); + + // manifestHash field stays zeroed (we're serializing with hash excluded) + + // --- Restore manifestHash --- + std::memcpy( const_cast( manifest.manifestHash ), savedManifestHash, SHA256_HASH_SIZE ); + + return out; + } + + bool DeserializeManifest( const std::vector &bytes, ExecutionManifest &out ) + { + if ( bytes.size() != MANIFEST_SERIALIZED_SIZE ) + { + return false; + } + + static constexpr size_t OFF_executionId = 0; + static constexpr size_t OFF_attemptId = 256; + static constexpr size_t OFF_taskId = 512; + static constexpr size_t OFF_subtaskId = 768; + static constexpr size_t OFF_passId = 1024; + static constexpr size_t OFF_executorIdentity = 1280; + static constexpr size_t OFF_modelIdentity = 1312; + static constexpr size_t OFF_tokenizerIdentity = 1344; + static constexpr size_t OFF_adapterIdentity = 1376; + static constexpr size_t OFF_shaderIdentity = 1408; + static constexpr size_t OFF_quantizationIdentity = 1440; + static constexpr size_t OFF_inputArtifactCount = 1472; + static constexpr size_t OFF_inputArtifactHashes = 1476; + static constexpr size_t OFF_outputArtifactCount = 3524; + static constexpr size_t OFF_outputArtifactHashes = 3528; + static constexpr size_t OFF_startTimeUsec = 5576; + static constexpr size_t OFF_endTimeUsec = 5584; + static constexpr size_t OFF_terminalState = 5592; + static constexpr size_t OFF_gpuMemoryUsedBytes = 5593; + static constexpr size_t OFF_outputBytesProduced = 5601; + static constexpr size_t OFF_wallClockUsec = 5609; + static constexpr size_t OFF_manifestHash = 5617; + + out = ExecutionManifest{}; + + // Identifier strings + std::memcpy( out.executionId, bytes.data() + OFF_executionId, MAX_IDENTIFIER ); + out.executionId[MAX_IDENTIFIER - 1] = '\0'; + std::memcpy( out.attemptId, bytes.data() + OFF_attemptId, MAX_IDENTIFIER ); + out.attemptId[MAX_IDENTIFIER - 1] = '\0'; + std::memcpy( out.taskId, bytes.data() + OFF_taskId, MAX_IDENTIFIER ); + out.taskId[MAX_IDENTIFIER - 1] = '\0'; + std::memcpy( out.subtaskId, bytes.data() + OFF_subtaskId, MAX_IDENTIFIER ); + out.subtaskId[MAX_IDENTIFIER - 1] = '\0'; + std::memcpy( out.passId, bytes.data() + OFF_passId, MAX_RESOURCE_NAME ); + out.passId[MAX_RESOURCE_NAME - 1] = '\0'; + + // Identity hashes + std::memcpy( out.executorIdentity, bytes.data() + OFF_executorIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.modelIdentity, bytes.data() + OFF_modelIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.tokenizerIdentity, bytes.data() + OFF_tokenizerIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.adapterIdentity, bytes.data() + OFF_adapterIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.shaderIdentity, bytes.data() + OFF_shaderIdentity, SHA256_HASH_SIZE ); + std::memcpy( out.quantizationIdentity, bytes.data() + OFF_quantizationIdentity, SHA256_HASH_SIZE ); + + // Input artifact refs + std::memcpy( &out.inputArtifactCount, bytes.data() + OFF_inputArtifactCount, sizeof( uint32_t ) ); + if ( out.inputArtifactCount > MAX_ARTIFACT_REFS ) + out.inputArtifactCount = static_cast( MAX_ARTIFACT_REFS ); + std::memcpy( out.inputArtifactHashes, bytes.data() + OFF_inputArtifactHashes, + out.inputArtifactCount * SHA256_HASH_SIZE ); + + // Output artifact refs + std::memcpy( &out.outputArtifactCount, bytes.data() + OFF_outputArtifactCount, sizeof( uint32_t ) ); + if ( out.outputArtifactCount > MAX_ARTIFACT_REFS ) + out.outputArtifactCount = static_cast( MAX_ARTIFACT_REFS ); + std::memcpy( out.outputArtifactHashes, bytes.data() + OFF_outputArtifactHashes, + out.outputArtifactCount * SHA256_HASH_SIZE ); + + // Timing + std::memcpy( &out.startTimeUsec, bytes.data() + OFF_startTimeUsec, sizeof( int64_t ) ); + std::memcpy( &out.endTimeUsec, bytes.data() + OFF_endTimeUsec, sizeof( int64_t ) ); + + // Terminal state + out.terminalState = static_cast( bytes[OFF_terminalState] ); + + // Resource summary + std::memcpy( &out.gpuMemoryUsedBytes, bytes.data() + OFF_gpuMemoryUsedBytes, sizeof( uint64_t ) ); + std::memcpy( &out.outputBytesProduced, bytes.data() + OFF_outputBytesProduced, sizeof( uint64_t ) ); + std::memcpy( &out.wallClockUsec, bytes.data() + OFF_wallClockUsec, sizeof( uint64_t ) ); + + // Manifest hash + std::memcpy( out.manifestHash, bytes.data() + OFF_manifestHash, SHA256_HASH_SIZE ); + + return true; + } + +} // namespace sgns::sgprocessing diff --git a/src/capability/CMakeLists.txt b/src/capability/CMakeLists.txt new file mode 100644 index 0000000..1520fde --- /dev/null +++ b/src/capability/CMakeLists.txt @@ -0,0 +1,30 @@ +# CapabilityValidator — pre-execution capability gate (Phase 06) +# Validates Vulkan device limits, MNN model compatibility, PassType registration, +# GPU memory, and disk space against a cached startup snapshot. + +add_library(SGCapability STATIC + capability_validator.cpp + ../../include/capability/capability_validator.hpp + ../../include/capability/capability_types.hpp +) + +target_compile_definitions(SGCapability PUBLIC SGPROCMGR_TEST_FRIEND) + +target_include_directories(SGCapability PUBLIC + $ + $ + $ + $ + $ +) + +target_link_libraries(SGCapability + PUBLIC + sgprocmanagerlogger + sgprocmanagersha + sgprocmanagertypes + Vulkan::Vulkan + OpenSSL::Crypto +) + +sgnus_install(SGCapability) diff --git a/src/capability/capability_validator.cpp b/src/capability/capability_validator.cpp new file mode 100644 index 0000000..d47f82d --- /dev/null +++ b/src/capability/capability_validator.cpp @@ -0,0 +1,495 @@ +/** + * CapabilityValidator implementation — BuildSnapshot and CanExecute. + * + * BuildSnapshot: queries Vulkan device properties, MNN executor registry, + * disk space, and computes the deterministic executor identity hash. + * CanExecute: validates jobs against the cached snapshot across all five + * check categories (PassType, Vulkan, MNN, GPU memory, disk space). + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace sgns::sgprocessing +{ + + // ========================================================================= + // PIMPL + // ========================================================================= + + struct CapabilityValidator::Impl + { + CapabilitySnapshot snapshot; + bool snapshotBuilt = false; + }; + + // ========================================================================= + // Anonymous-namespace helpers + // ========================================================================= + + namespace + { + + std::string JoinStrings( const std::vector &items, const std::string &sep ) + { + std::ostringstream oss; + for ( size_t i = 0; i < items.size(); ++i ) + { + if ( i > 0 ) oss << sep; + oss << items[i]; + } + return oss.str(); + } + + std::string FormatBytes( uint64_t bytes ) + { + const char *units[] = { "B", "KB", "MB", "GB", "TB" }; + int unit = 0; + double val = static_cast( bytes ); + while ( val >= 1024.0 && unit < 4 ) { val /= 1024.0; ++unit; } + char tmp[64]; + if ( unit == 0 ) + std::snprintf( tmp, sizeof( tmp ), "%llu %s", + static_cast( bytes ), units[unit] ); + else + std::snprintf( tmp, sizeof( tmp ), "%.1f %s", val, units[unit] ); + return tmp; + } + + std::string PassTypeToString( PassType pt ) + { + switch ( pt ) + { + case PassType::COMPUTE: return "COMPUTE"; + case PassType::DATA_TRANSFORM: return "DATA_TRANSFORM"; + case PassType::INFERENCE: return "INFERENCE"; + case PassType::RENDER: return "RENDER"; + case PassType::RETRAIN: return "RETRAIN"; + } + return std::to_string( static_cast( pt ) ); + } + + std::string ListAvailablePassTypes( const std::vector &caps ) + { + std::vector names; + for ( const auto &cap : caps ) + names.push_back( PassTypeToString( cap.passType ) + " (" + + std::to_string( static_cast( cap.passType ) ) + ")" ); + if ( names.empty() ) return ""; + return JoinStrings( names, ", " ); + } + + uint64_t QueryAvailableDiskBytes( const std::string &path ) + { +#ifdef _WIN32 + ULARGE_INTEGER freeBytesAvailable; + if ( GetDiskFreeSpaceExA( path.empty() ? "." : path.c_str(), + &freeBytesAvailable, nullptr, nullptr ) ) + return freeBytesAvailable.QuadPart; + return 0; +#else + struct statvfs stat; + if ( statvfs( path.empty() ? "." : path.c_str(), &stat ) == 0 ) + return static_cast( stat.f_bavail ) * stat.f_frsize; + return 0; +#endif + } + + uint64_t BytesPerPixel( sgns::ColorFormat fmt ) + { + switch ( fmt ) + { + case sgns::ColorFormat::RGBA8: return 4; + case sgns::ColorFormat::RGB8: return 3; + default: return 4; + } + } + + uint64_t BytesPerPixel( sgns::DepthFormat fmt ) + { + switch ( fmt ) + { + case sgns::DepthFormat::D32_SFLOAT: return 4; + case sgns::DepthFormat::D24_UNORM_S8_UINT: return 4; + default: return 4; + } + } + + std::vector CollectExecutorCapabilities( + const std::unordered_map()>, + PassTypeHash> &passFactories, + size_t /*mnnProcessorCount*/ ) + { + std::vector caps; + for ( const auto &[passType, factory] : passFactories ) + { + (void)factory; + ExecutorCapability cap; + cap.passType = passType; + if ( passType == PassType::RENDER ) + { + cap.backend = "VULKAN"; + } + else + { + cap.backend = "VULKAN"; + cap.supportedModelFormats = { "MNN" }; + cap.supportedQuantizations = { "FP32", "FP16", "INT8" }; + } + caps.push_back( std::move( cap ) ); + } + return caps; + } + + const ExecutorCapability *FindExecutorCap( const CapabilitySnapshot &snap, + PassType pt ) + { + for ( const auto &cap : snap.executorCaps ) + if ( cap.passType == pt ) return ∩ + return nullptr; + } + + std::string DeriveExecutorId( const std::vector &identityHash ) + { + if ( identityHash.empty() ) return "sgproc-0000000000000000"; + std::ostringstream oss; + oss << "sgproc-"; + size_t n = (std::min)( size_t( 8 ), identityHash.size() ); + for ( size_t i = 0; i < n; ++i ) + { + char hex[3]; + std::snprintf( hex, sizeof( hex ), "%02x", identityHash[i] ); + oss << hex; + } + return oss.str(); + } + + std::string ModelFormatToString( sgns::ModelFormat fmt ) + { + switch ( fmt ) + { + case sgns::ModelFormat::MNN: return "MNN"; + case sgns::ModelFormat::ONNX: return "ONNX"; + case sgns::ModelFormat::PY_TORCH: return "PY_TORCH"; + case sgns::ModelFormat::TENSOR_FLOW: return "TENSOR_FLOW"; + default: return "UNKNOWN"; + } + } + + } // anonymous namespace + + // ========================================================================= + // Construction / destruction + // ========================================================================= + + CapabilityValidator::CapabilityValidator() + : m_impl( std::make_unique() ) {} + + CapabilityValidator::~CapabilityValidator() = default; + + // ========================================================================= + // BuildSnapshot (D-09, D-10, D-11, D-12, D-16) + // ========================================================================= + + void CapabilityValidator::BuildSnapshot( + const std::unordered_map()>, + PassTypeHash> &passFactories, + size_t mnnProcessorCount, + std::function ensureVulkanDevice ) + { + CapabilitySnapshot snapshot; + + // Vulkan device query (D-10, D-14) + // NOTE: ensureVulkanDevice() internally calls RenderProcessor::InitializeContext(), + // which acquires VulkanInitMutex() itself via a double-check locking pattern. + // Holding the mutex here while calling ensureVulkanDevice() would cause a + // self-deadlock on the same thread. Only lock around the vkGetPhysicalDevice* + // queries — the device is kept alive by the static RenderProcessor inside the + // lambda, so it's safe to read its properties without the mutex. + { + VkPhysicalDevice device = ensureVulkanDevice(); + if ( device != VK_NULL_HANDLE ) + { + std::lock_guard lock( VulkanInitMutex() ); + vkGetPhysicalDeviceProperties( device, &snapshot.vulkanProps ); + vkGetPhysicalDeviceMemoryProperties( device, &snapshot.memProps ); + } + } + + // MNN executor capability collection (D-11) + snapshot.executorCaps = CollectExecutorCapabilities( passFactories, mnnProcessorCount ); + + // Disk space query (D-16) + snapshot.availableDiskBytes = QueryAvailableDiskBytes( "." ); + + // Executor identity hash (D-08) + { + std::vector hashInput; + auto appendBytes = [&hashInput]( const void *data, size_t size ) + { + const auto *bytes = static_cast( data ); + hashInput.insert( hashInput.end(), bytes, bytes + size ); + }; + + appendBytes( &snapshot.vulkanProps.deviceID, + sizeof( snapshot.vulkanProps.deviceID ) ); + appendBytes( &snapshot.vulkanProps.driverVersion, + sizeof( snapshot.vulkanProps.driverVersion ) ); + appendBytes( &snapshot.vulkanProps.vendorID, + sizeof( snapshot.vulkanProps.vendorID ) ); + appendBytes( snapshot.vulkanProps.deviceName, + std::strlen( snapshot.vulkanProps.deviceName ) ); + + for ( uint32_t i = 0; i < snapshot.memProps.memoryHeapCount; ++i ) + { + appendBytes( &snapshot.memProps.memoryHeaps[i].size, + sizeof( snapshot.memProps.memoryHeaps[i].size ) ); + appendBytes( &snapshot.memProps.memoryHeaps[i].flags, + sizeof( snapshot.memProps.memoryHeaps[i].flags ) ); + } + + for ( const auto &cap : snapshot.executorCaps ) + { + auto pt = static_cast( cap.passType ); + appendBytes( &pt, sizeof( pt ) ); + appendBytes( cap.backend.data(), cap.backend.size() ); + for ( const auto &fmt : cap.supportedModelFormats ) + appendBytes( fmt.data(), fmt.size() ); + for ( const auto &q : cap.supportedQuantizations ) + appendBytes( q.data(), q.size() ); + } + + snapshot.identityHash = sgns::sgprocmanagersha::sha256( + hashInput.data(), hashInput.size() ); + } + + m_impl->snapshot = std::move( snapshot ); + m_impl->snapshotBuilt = true; + } + + const CapabilitySnapshot *CapabilityValidator::GetSnapshot() const + { + if ( !m_impl->snapshotBuilt ) return nullptr; + return &m_impl->snapshot; + } + + // ========================================================================= + // CanExecute — all five validation categories (CAP-02..05) + // ========================================================================= + + void CapabilityValidator::CanExecute( const sgns::Pass &pass, + CanExecuteCallback callback ) + { + CanExecuteResult result; + std::vector unmet; + + if ( !m_impl->snapshotBuilt ) + { + result.executable = false; + result.unmet.push_back( + { UnmetRequirementCategory::RESOURCE, + "CapabilityValidator not initialized" } ); + callback( result ); + return; + } + + const auto &snapshot = m_impl->snapshot; + PassType passType = pass.get_type(); + + // —— Step 1: PassType registration check (CAP-04/D-04) —— + const ExecutorCapability *executorCap = FindExecutorCap( snapshot, passType ); + if ( !executorCap ) + { + unmet.push_back( + { UnmetRequirementCategory::PASS_TYPE, + "No executor registered for PassType " + + PassTypeToString( passType ) + " (" + + std::to_string( static_cast( passType ) ) + ")" + + ". Available: [" + + ListAvailablePassTypes( snapshot.executorCaps ) + "]" } ); + result.executable = false; + result.unmet = std::move( unmet ); + callback( result ); + return; + } + + // —— Step 2: Vulkan feature/limit check (CAP-02/D-14) —— + if ( passType == PassType::RENDER ) + { + const auto &limits = snapshot.vulkanProps.limits; + + if ( snapshot.vulkanProps.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU + && snapshot.vulkanProps.deviceType != VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ) + { + unmet.push_back( + { UnmetRequirementCategory::VULKAN, + "Device type not acceptable (need DISCRETE_GPU or INTEGRATED_GPU)" } ); + } + + if ( auto rt = pass.get_render_target() ) + { + uint32_t w = static_cast( rt->get_width() ); + uint32_t h = static_cast( rt->get_height() ); + + if ( w > limits.maxImageDimension2D ) + unmet.push_back( + { UnmetRequirementCategory::VULKAN, + "maxImageDimension2D: need " + std::to_string( w ) + + ", have " + std::to_string( limits.maxImageDimension2D ) } ); + if ( h > limits.maxImageDimension2D ) + unmet.push_back( + { UnmetRequirementCategory::VULKAN, + "maxImageDimension2D: need " + std::to_string( h ) + + ", have " + std::to_string( limits.maxImageDimension2D ) } ); + } + + if ( limits.maxColorAttachments < 1 ) + unmet.push_back( + { UnmetRequirementCategory::VULKAN, + "maxColorAttachments: need 1, have " + + std::to_string( limits.maxColorAttachments ) } ); + + if ( limits.maxMemoryAllocationCount < 4 ) + unmet.push_back( + { UnmetRequirementCategory::VULKAN, + "maxMemoryAllocationCount: need 4, have " + + std::to_string( limits.maxMemoryAllocationCount ) } ); + } + + if ( !unmet.empty() ) + { + result.executable = false; + result.unmet = std::move( unmet ); + callback( result ); + return; + } + + // —— Step 2b: MNN model format check (CAP-03/D-11) —— + if ( passType == PassType::INFERENCE || passType == PassType::RETRAIN ) + { + if ( auto model = pass.get_model() ) + { + std::string fmtStr = ModelFormatToString( model->get_format() ); + bool formatSupported = false; + for ( const auto &sf : executorCap->supportedModelFormats ) + { + if ( sf == fmtStr ) { formatSupported = true; break; } + } + if ( !formatSupported ) + unmet.push_back( + { UnmetRequirementCategory::MNN, + "Model format " + fmtStr + " not supported. Supported: [" + + JoinStrings( executorCap->supportedModelFormats, ", " ) + + "]" } ); + } + } + + if ( !unmet.empty() ) + { + result.executable = false; + result.unmet = std::move( unmet ); + callback( result ); + return; + } + + // —— Step 3: GPU memory estimation (CAP-05/D-15) —— + if ( passType == PassType::RENDER ) + { + uint64_t estimatedGpuMem = 0; + + if ( auto rt = pass.get_render_target() ) + { + uint64_t w = static_cast( rt->get_width() ); + uint64_t h = static_cast( rt->get_height() ); + uint64_t colorBytes = BytesPerPixel( rt->get_color_format() ); + uint64_t depthBytes = BytesPerPixel( rt->get_depth_format() ); + estimatedGpuMem += w * h * ( colorBytes + depthBytes ); + } + + estimatedGpuMem += 64ULL * 1024 * 1024; // pipeline overhead + + uint64_t largestHeap = 0; + for ( uint32_t i = 0; i < snapshot.memProps.memoryHeapCount; ++i ) + { + if ( snapshot.memProps.memoryHeaps[i].flags + & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ) + { + largestHeap = (std::max)( largestHeap, + snapshot.memProps.memoryHeaps[i].size ); + } + } + + if ( largestHeap > 0 && estimatedGpuMem > largestHeap ) + unmet.push_back( + { UnmetRequirementCategory::RESOURCE, + "Estimated GPU memory " + FormatBytes( estimatedGpuMem ) + + " exceeds largest device-local heap " + + FormatBytes( largestHeap ) } ); + } + + // —— Step 4: Disk space check (CAP-05/D-16) —— + if ( snapshot.availableDiskBytes > 0 ) + { + uint64_t estimatedOutputSize = 0; + + if ( passType == PassType::RENDER ) + { + if ( auto rt = pass.get_render_target() ) + { + uint64_t w = static_cast( rt->get_width() ); + uint64_t h = static_cast( rt->get_height() ); + estimatedOutputSize = w * h + * BytesPerPixel( rt->get_color_format() ); + } + } + + if ( estimatedOutputSize > snapshot.availableDiskBytes ) + unmet.push_back( + { UnmetRequirementCategory::RESOURCE, + "Estimated output size " + FormatBytes( estimatedOutputSize ) + + " exceeds available disk space " + + FormatBytes( snapshot.availableDiskBytes ) } ); + } + + // —— Build final result —— + if ( !unmet.empty() ) + { + result.executable = false; + result.unmet = std::move( unmet ); + } + else + { + result.executable = true; + result.executorId = DeriveExecutorId( snapshot.identityHash ); + } + + callback( result ); + } + +#ifdef SGPROCMGR_TEST_FRIEND + void CapabilityValidator::SetSnapshotForTest( CapabilitySnapshot snap ) + { + m_impl->snapshot = std::move( snap ); + m_impl->snapshotBuilt = true; + } +#endif + +} // namespace sgns::sgprocessing diff --git a/src/execution/CMakeLists.txt b/src/execution/CMakeLists.txt new file mode 100644 index 0000000..2a1de76 --- /dev/null +++ b/src/execution/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(SGExecution INTERFACE) +target_include_directories(SGExecution INTERFACE ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(SGExecution INTERFACE SGProcessors) diff --git a/src/processingbase/CMakeLists.txt b/src/processingbase/CMakeLists.txt index feec32c..8c28bd6 100644 --- a/src/processingbase/CMakeLists.txt +++ b/src/processingbase/CMakeLists.txt @@ -23,6 +23,9 @@ target_link_libraries( AsyncIOManager SGProcessors DataSplitter + SGShaderCompiler + SGCapability + SGArtifacts ) sgnus_install(ProcessingBase) diff --git a/src/processingbase/ProcessingManager.cpp b/src/processingbase/ProcessingManager.cpp index f4d1b9a..b8b4a4e 100644 --- a/src/processingbase/ProcessingManager.cpp +++ b/src/processingbase/ProcessingManager.cpp @@ -3,6 +3,15 @@ #include #include "FileManager.hpp" #include "URLStringUtil.h" +#include "shaders/shader_compiler.hpp" + +#include +#include +#include +#include +#include +#include +#include "artifacts/artifact_serializer.hpp" OUTCOME_CPP_DEFINE_CATEGORY_3( sgns::sgprocessing, ProcessingManager::Error, e ) { @@ -20,6 +29,20 @@ OUTCOME_CPP_DEFINE_CATEGORY_3( sgns::sgprocessing, ProcessingManager::Error, e ) return "Input missing"; case sgns::sgprocessing::ProcessingManager::Error::INPUT_UNAVAIL: return "Could not get input from source"; + case sgns::sgprocessing::ProcessingManager::Error::SHADER_COMPILE_FAILED: + return "Shader source failed to compile"; + case sgns::sgprocessing::ProcessingManager::Error::SPIRV_VALIDATION_FAILED: + return "SPIR-V failed validation"; + case sgns::sgprocessing::ProcessingManager::Error::PROCESSING_FAILED: + return "Processor failed to produce a valid result"; + case sgns::sgprocessing::ProcessingManager::Error::MODEL_MISSING: + return "Inference or retrain pass is missing required model configuration"; + case sgns::sgprocessing::ProcessingManager::Error::MODEL_FORMAT_UNSUPPORTED: + return "Model format is not supported for execution (only MNN format is executable)"; + case sgns::sgprocessing::ProcessingManager::Error::RENDER_SHADER_MISSING: + return "Render pass is missing required shader configuration"; + case sgns::sgprocessing::ProcessingManager::Error::UNKNOWN_PASS_TYPE: + return "Job definition references an unrecognized or unregistered pass type"; } return "Unknown error"; } @@ -54,6 +77,293 @@ namespace sgns::sgprocessing } return !extension.empty(); } + + /** + * Packs validated per-stage SPIR-V into a single byte buffer. + * + * PROVISIONAL WIRE FORMAT -- this is this plan's own choice, not a + * negotiated Phase-3 contract. Phase 3's RenderProcessor has not been + * designed yet and does not currently consume mainbuffers->first for + * render passes at all; Phase 3's planning may revise this format + * once RenderProcessor's actual pipeline-construction needs are + * known. + * + * Layout (all integers little-endian, native uint32_t width): + * uint32_t stage_count + * per stage: + * uint32_t stage_tag (static_cast(sgns::Stage)) + * uint32_t entry_point_len (number of following raw UTF-8 bytes) + * entry_point_len raw UTF-8 bytes (no null terminator) + * uint32_t word_count (number of following uint32_t SPIR-V words) + * word_count * uint32_t spirv_words + */ + std::vector SerializeCompiledStages( + const std::vector &stages, + const std::vector &entryPoints ) + { + std::vector out; + + if ( entryPoints.size() != stages.size() ) + { + // Invariant of this plan's own call site -- a mismatch indicates a + // caller bug, not malformed job-supplied input. + return out; + } + + auto appendU32 = [&out]( uint32_t value ) + { + size_t offset = out.size(); + out.resize( offset + sizeof( uint32_t ) ); + std::memcpy( out.data() + offset, &value, sizeof( uint32_t ) ); + }; + + appendU32( static_cast( stages.size() ) ); + for ( size_t i = 0; i < stages.size(); ++i ) + { + const auto &compiled = stages[i]; + appendU32( static_cast( compiled.stage ) ); + + const std::string &entryPoint = entryPoints[i]; + appendU32( static_cast( entryPoint.size() ) ); + if ( !entryPoint.empty() ) + { + size_t offset = out.size(); + out.resize( offset + entryPoint.size() ); + std::memcpy( out.data() + offset, entryPoint.data(), entryPoint.size() ); + } + + appendU32( static_cast( compiled.spirv.size() ) ); + if ( !compiled.spirv.empty() ) + { + size_t offset = out.size(); + out.resize( offset + compiled.spirv.size() * sizeof( uint32_t ) ); + std::memcpy( out.data() + offset, + compiled.spirv.data(), + compiled.spirv.size() * sizeof( uint32_t ) ); + } + } + + return out; + } + + /** + * Packs render_target/pipeline_state/vertex_layout/uniforms alongside the + * independently-resolved vertex/index buffer bytes into the single wire-format + * buffer GetCidForProc() places into mainbuffers->second for a render pass. + * + * This is the ONLY channel any of this Pass-level data has to reach + * RenderProcessor -- StartProcessing()'s fixed signature never carries the + * Pass or RenderShaderConfig object itself (D-25's no-signature-change + * constraint). Plan 03-03's RenderProcessor parser must be the exact inverse + * of this function. + * + * Layout (all integers little-endian, native uint32_t width; clear_color/ + * clear_depth narrowed from the schema's double to float on write): + * uint32_t width + * uint32_t height + * uint32_t color_format_tag (static_cast(ColorFormat)) + * uint32_t depth_format_tag (static_cast(DepthFormat)) + * float clear_color[4] + * float clear_depth + * uint8_t has_pipeline_state + * if has_pipeline_state: + * uint8_t has_topology + [uint32_t topology_tag] + * uint8_t has_cull_mode + [uint32_t cull_mode_tag] + * uint8_t has_front_face + [uint32_t front_face_tag] + * uint8_t has_depth_test + [uint32_t depth_test_tag] + * uint32_t vertex_layout_count + * per entry: + * uint32_t name_len + name bytes (raw UTF-8, no null terminator) + * uint32_t format_tag (static_cast(VertexLayoutFormat)) + * uint32_t offset + * uint8_t has_uniforms + * if has_uniforms: + * uint32_t uniform_count + * per entry (std::map's natural key-sorted iteration order): + * uint32_t name_len + name bytes + * uint8_t has_source + [uint32_t source_len + source bytes] + * uint8_t has_type + [uint32_t type_tag (static_cast(DataType))] + * uint32_t value_json_len + value bytes (nlohmann::json::dump() UTF-8; + * empty string if get_value().is_null()) + * uint32_t vertex_len + vertex bytes + * uint8_t has_index + * if has_index: + * uint32_t index_type_tag (static_cast(IndexType)) + * uint32_t index_len + index bytes + * uint32_t data_transform_count + */ + std::vector SerializeRenderPassConfig( + const sgns::RenderTarget &target, + const boost::optional &pipelineState, + const std::vector &vertexLayout, + const boost::optional> &uniforms, + const std::vector &vertexBytes, + bool hasIndexBuffer, + sgns::IndexType indexType, + const std::vector &indexBytes, + uint32_t dataTransformCount ) + { + std::vector out; + + auto appendBytes = [&out]( const char *data, size_t size ) + { + if ( size > 0 ) + { + size_t offset = out.size(); + out.resize( offset + size ); + std::memcpy( out.data() + offset, data, size ); + } + }; + auto appendU32 = [&out]( uint32_t value ) + { + size_t offset = out.size(); + out.resize( offset + sizeof( uint32_t ) ); + std::memcpy( out.data() + offset, &value, sizeof( uint32_t ) ); + }; + auto appendU8 = [&out]( uint8_t value ) { out.push_back( static_cast( value ) ); }; + auto appendF32 = [&out]( float value ) + { + size_t offset = out.size(); + out.resize( offset + sizeof( float ) ); + std::memcpy( out.data() + offset, &value, sizeof( float ) ); + }; + auto appendString = [&]( const std::string &value ) + { + appendU32( static_cast( value.size() ) ); + appendBytes( value.data(), value.size() ); + }; + + appendU32( static_cast( target.get_width() ) ); + appendU32( static_cast( target.get_height() ) ); + appendU32( static_cast( target.get_color_format() ) ); + appendU32( static_cast( target.get_depth_format() ) ); + + const auto &clearColor = target.get_clear_color(); + for ( size_t i = 0; i < 4; ++i ) + { + appendF32( i < clearColor.size() ? static_cast( clearColor[i] ) : 0.0f ); + } + appendF32( static_cast( target.get_clear_depth() ) ); + + if ( pipelineState ) + { + appendU8( 1 ); + const auto &ps = pipelineState.value(); + + if ( ps.get_topology() ) + { + appendU8( 1 ); + appendU32( static_cast( ps.get_topology().value() ) ); + } + else + { + appendU8( 0 ); + } + + if ( ps.get_cull_mode() ) + { + appendU8( 1 ); + appendU32( static_cast( ps.get_cull_mode().value() ) ); + } + else + { + appendU8( 0 ); + } + + if ( ps.get_front_face() ) + { + appendU8( 1 ); + appendU32( static_cast( ps.get_front_face().value() ) ); + } + else + { + appendU8( 0 ); + } + + if ( ps.get_depth_test() ) + { + appendU8( 1 ); + appendU32( static_cast( ps.get_depth_test().value() ) ); + } + else + { + appendU8( 0 ); + } + } + else + { + appendU8( 0 ); + } + + appendU32( static_cast( vertexLayout.size() ) ); + for ( const auto &entry : vertexLayout ) + { + appendString( entry.get_name() ); + appendU32( static_cast( entry.get_format() ) ); + appendU32( static_cast( entry.get_offset() ) ); + } + + if ( uniforms ) + { + appendU8( 1 ); + const auto &uniformMap = uniforms.value(); + appendU32( static_cast( uniformMap.size() ) ); + // std::map iterates in key-sorted order already -- matches plan + // 03-03's ResolveUniforms iteration-order decision. + for ( const auto &uniformEntry : uniformMap ) + { + appendString( uniformEntry.first ); + const auto &uniform = uniformEntry.second; + + if ( uniform.get_source() ) + { + appendU8( 1 ); + appendString( uniform.get_source().value() ); + } + else + { + appendU8( 0 ); + } + + if ( uniform.get_type() ) + { + appendU8( 1 ); + appendU32( static_cast( uniform.get_type().value() ) ); + } + else + { + appendU8( 0 ); + } + + std::string valueJson = + uniform.get_value().is_null() ? std::string() : uniform.get_value().dump(); + appendString( valueJson ); + } + } + else + { + appendU8( 0 ); + } + + appendU32( static_cast( vertexBytes.size() ) ); + appendBytes( vertexBytes.data(), vertexBytes.size() ); + + if ( hasIndexBuffer ) + { + appendU8( 1 ); + appendU32( static_cast( indexType ) ); + appendU32( static_cast( indexBytes.size() ) ); + appendBytes( indexBytes.data(), indexBytes.size() ); + } + else + { + appendU8( 0 ); + } + + appendU32( dataTransformCount ); + + return out; + } } ProcessingManager::~ProcessingManager() {} @@ -101,18 +411,108 @@ namespace sgns::sgprocessing [] { return std::make_unique(); } ); RegisterProcessorFactory( static_cast( DataType::TEXTURE_CUBE ), [] { return std::make_unique(); } ); + RegisterPassProcessorFactory( PassType::RENDER, + [] { return std::make_unique(); }, + false /* supports_checkpointing */ ); + + // Build capability snapshot after all executors are registered (D-01, D-09) + m_capabilityValidator = std::make_unique(); + { + // Extract factory functions from ExecutorRegistryEntry for BuildSnapshot + std::unordered_map()>, PassTypeHash> factoriesOnly; + std::unordered_map checkpointFlags; + for ( auto &entry : m_passFactories ) + { + factoriesOnly[entry.first] = entry.second.factory; + checkpointFlags[entry.first] = entry.second.supports_checkpointing; + } + m_capabilityValidator->BuildSnapshot( + factoriesOnly, + m_processorFactories.size(), + []() -> VkPhysicalDevice + { + // Ensure Vulkan device exists via a temporary RenderProcessor + // that lazy-initializes the shared Vulkan context under VulkanInitMutex. + static auto s_renderProc = std::make_unique(); + if ( !s_renderProc->InitializeContext() ) + return VK_NULL_HANDLE; + return s_renderProc->GetPhysicalDevice(); + } ); + // Populate checkpoint support flags onto the snapshot (D-20) + if ( auto *snap = m_capabilityValidator->GetSnapshot() ) + { + // const_cast: GetSnapshot returns const*, but we own the snapshot + // and this is the only place it's populated during Init(). + const_cast( snap )->checkpointSupport = std::move( checkpointFlags ); + } + } //Parse Json //This will check required fields inherently. try { auto data = nlohmann::json::parse( jsondata ); + + // Pre-parse validation: intercept unrecognized passes[].type and + // passes[].model.format raw strings *before* sgns::from_json() runs. + // The quicktype-generated from_json(PassType&)/from_json(ModelFormat&) + // throw a plain std::runtime_error with no field context when a job + // submits an unrecognized enum string, which the generic catch below + // would otherwise collapse into a context-free Error::INVALID_JSON. + // Recognized-but-unsupported formats (e.g. ONNX) are intentionally left + // alone here -- they parse successfully and are rejected afterward by + // CheckProcessValidity()'s explicit MNN-executability check instead. + if ( data.contains( "passes" ) && data[ "passes" ].is_array() ) + { + static const std::set kRecognizedPassTypes = { "compute", "data_transform", + "inference", "render", "retrain" }; + static const std::set kRecognizedModelFormats = { "MNN", "ONNX", "PyTorch", + "TensorFlow" }; + for ( const auto &passEntry : data[ "passes" ] ) + { + if ( !passEntry.is_object() || !passEntry.contains( "type" ) || !passEntry[ "type" ].is_string() ) + { + continue; + } + const std::string passType = passEntry[ "type" ].get(); + if ( kRecognizedPassTypes.find( passType ) == kRecognizedPassTypes.end() ) + { + m_logger->error( "Job definition references an unrecognized pass type: " + passType ); + return outcome::failure( Error::UNKNOWN_PASS_TYPE ); + } + if ( ( passType == "inference" || passType == "retrain" ) && passEntry.contains( "model" ) && + passEntry[ "model" ].is_object() ) + { + const auto &modelEntry = passEntry[ "model" ]; + if ( modelEntry.contains( "format" ) && modelEntry[ "format" ].is_string() ) + { + const std::string modelFormat = modelEntry[ "format" ].get(); + if ( kRecognizedModelFormats.find( modelFormat ) == kRecognizedModelFormats.end() ) + { + m_logger->error( "Job definition references an unsupported model format: " + + modelFormat ); + return outcome::failure( Error::MODEL_FORMAT_UNSUPPORTED ); + } + } + } + } + } + sgns::from_json( data, processing_ ); } catch ( const nlohmann::json::exception &e ) { return outcome::failure( Error::INVALID_JSON ); } + catch ( const std::exception &e ) + { + // quicktype-generated enum from_json functions (e.g. the narrowed + // ShaderSourceType) throw a plain std::runtime_error -- not a + // nlohmann::json::exception subclass -- when a job submits a + // schema-invalid enum value (e.g. legacy "hlsl"/"metal"). Must be + // caught here as well or it propagates uncaught out of Init(). + return outcome::failure( Error::INVALID_JSON ); + } auto isvalid = CheckProcessValidity(); if ( !isvalid ) { @@ -140,7 +540,13 @@ namespace sgns::sgprocessing if ( !pass.get_model() ) { m_logger->error( "Inference json has no model" ); - return outcome::failure( Error::PROCESS_INFO_MISSING ); + return outcome::failure( Error::MODEL_MISSING ); + } + if ( pass.get_model().value().get_format() != ModelFormat::MNN ) + { + m_logger->error( "Inference pass model format is not executable (only MNN is supported), pass: " + + pass.get_name() ); + return outcome::failure( Error::MODEL_FORMAT_UNSUPPORTED ); } break; } @@ -149,7 +555,102 @@ namespace sgns::sgprocessing case PassType::DATA_TRANSFORM: break; case PassType::RENDER: + { + if ( !pass.get_render_shader() ) + { + m_logger->error( "Render pass has no render_shader config" ); + return outcome::failure( Error::RENDER_SHADER_MISSING ); + } + if ( !pass.get_render_target() ) + { + m_logger->error( "Render pass has no render_target config" ); + return outcome::failure( Error::PROCESS_INFO_MISSING ); + } + if ( !pass.get_vertex_buffer() ) + { + m_logger->error( "Render pass has no vertex_buffer binding" ); + return outcome::failure( Error::PROCESS_INFO_MISSING ); + } + if ( !pass.get_vertex_layout() || pass.get_vertex_layout()->empty() ) + { + m_logger->error( "Render pass has no vertex_layout entries" ); + return outcome::failure( Error::PROCESS_INFO_MISSING ); + } + + // Task 2: defensively reject vertex_buffer/index_buffer/uniform + // sources this phase has no real resolution path for + // (output:/internal:/parameter: for buffers; anything but + // parameter: for uniforms) -- fail closed at Create() time + // rather than reaching RenderProcessor unchecked (T-03-02-01, + // T-03-02-02). + auto rejectUnsupportedBufferSourcePrefix = + [this]( const char *fieldName, const std::string &source ) -> outcome::result + { + m_logger->error( + "Render pass {}.source '{}' uses an unsupported prefix -- " + "only input: is resolvable (no cross-pass output:/internal: dependency " + "graph exists; parameter:-sourced raw buffers are not supported)", + fieldName, + source ); + return outcome::failure( Error::PROCESS_INFO_MISSING ); + }; + + { + const auto vertexBufferCfg = pass.get_vertex_buffer().value(); + const std::string vertexSource = vertexBufferCfg.get_source(); + if ( vertexSource.rfind( "input:", 0 ) != 0 ) + { + return rejectUnsupportedBufferSourcePrefix( "vertex_buffer", vertexSource ); + } + } + + if ( pass.get_index_buffer() && pass.get_index_buffer().value().get_source() ) + { + const auto indexBufferCfg = pass.get_index_buffer().value(); + std::string indexSource = indexBufferCfg.get_source().value(); + if ( indexSource.rfind( "input:", 0 ) != 0 ) + { + return rejectUnsupportedBufferSourcePrefix( "index_buffer", indexSource ); + } + } + + { + const auto renderShaderCfg = pass.get_render_shader().value(); + if ( renderShaderCfg.get_uniforms() ) + { + const auto uniformsCfg = renderShaderCfg.get_uniforms().value(); + for ( const auto &uniformEntry : uniformsCfg ) + { + const std::string &uniformName = uniformEntry.first; + const auto &uniform = uniformEntry.second; + + if ( uniform.get_source() ) + { + const std::string &uniformSource = uniform.get_source().value(); + if ( uniformSource.rfind( "parameter:", 0 ) != 0 ) + { + m_logger->error( + "Render pass uniform '{}' has source '{}' with an " + "unsupported prefix -- only parameter: is resolvable for " + "uniform values in this phase", + uniformName, + uniformSource ); + return outcome::failure( Error::PROCESS_INFO_MISSING ); + } + } + else if ( uniform.get_value().is_null() ) + { + m_logger->error( + "Render pass uniform '{}' has neither a source nor a usable " + "value", + uniformName ); + return outcome::failure( Error::PROCESS_INFO_MISSING ); + } + } + } + } break; + } case PassType::RETRAIN: break; default: @@ -646,6 +1147,10 @@ namespace sgns::sgprocessing auto passes = processing_.get_passes(); for ( const auto &pass : passes ) { + if ( !pass.get_model() ) + { + continue; + } auto input_nodes = pass.get_model().value().get_input_nodes(); for ( auto &model : input_nodes ) { @@ -661,10 +1166,33 @@ namespace sgns::sgprocessing return block_total_len; } - outcome::result> ProcessingManager::Process( std::shared_ptr ioc, + outcome::result ProcessingManager::Process( std::shared_ptr ioc, std::vector> &chunkhashes, sgns::ModelNode &model, std::vector &output_locations ) + { + // Legacy 4-arg overload: construct a fresh, internally-owned ExecutionContext + // (unchanged behavior for every existing caller) and delegate to ProcessInternal. + ExecutionContext execCtx; + return ProcessInternal( ioc, chunkhashes, model, output_locations, execCtx ); + } + + outcome::result ProcessingManager::Process( std::shared_ptr ioc, + std::vector> &chunkhashes, + sgns::ModelNode &model, + std::vector &output_locations, + ExecutionContext &externalExecCtx ) + { + // New 5-arg overload: caller owns the ExecutionContext, so cancellation, + // deadline, and budget fields may be pre-set/cancelled from another thread. + return ProcessInternal( ioc, chunkhashes, model, output_locations, externalExecCtx ); + } + + outcome::result ProcessingManager::ProcessInternal( std::shared_ptr ioc, + std::vector> &chunkhashes, + sgns::ModelNode &model, + std::vector &output_locations, + ExecutionContext &execCtx ) { //Get input index auto modelname = model.get_source().value(); @@ -679,145 +1207,470 @@ namespace sgns::sgprocessing return maybe_buffers.error(); } auto buffers = maybe_buffers.value(); - if ( !SetProcessorByName( static_cast( processing_.get_inputs()[index.value()].get_type() ) ) ) + const auto &pass = processing_.get_passes()[index.value()]; + + // Extract budget fields from pass schema (D-06, D-07, D-08) + uint64_t gpuMemoryBudget = pass.get_estimated_gpu_memory_bytes().value_or( 0 ); + uint64_t outputArtifactBudget = pass.get_max_output_artifact_bytes().value_or( 0 ); + uint64_t deadlineMs = pass.get_per_pass_deadline_ms().value_or( 0 ); + + if ( pass.get_type() == PassType::RENDER ) { - return outcome::failure( Error::NO_PROCESSOR ); + if ( !SetProcessorByPassType( PassType::RENDER ) ) + { + return outcome::failure( Error::NO_PROCESSOR ); + } + } + else + { + if ( !SetProcessorByName( static_cast( processing_.get_inputs()[index.value()].get_type() ) ) ) + { + return outcome::failure( Error::NO_PROCESSOR ); + } } const auto maybeParameters = processing_.get_parameters(); const auto *parameters = maybeParameters ? &maybeParameters.value() : nullptr; - auto processResult = m_processor->StartProcessing( chunkhashes, - processing_.get_inputs()[index.value()], - *buffers->second, - *buffers->first, - parameters ); - - const auto &outputs = processing_.get_outputs(); - if ( processResult.output_buffers && !outputs.empty() ) + try { - const auto &bufferNames = processResult.output_buffers->first; - const auto &bufferData = processResult.output_buffers->second; + // Apply schema-derived budgets (D-06, D-07, D-08) only when the incoming + // execCtx still has the field at its "unset" sentinel (0). A caller of the + // 5-arg Process() overload may have pre-set any of these fields explicitly; + // that caller-supplied value is never overwritten. For the legacy 4-arg + // overload's freshly-constructed ExecutionContext, every field starts at 0, + // so this is behavior-neutral — the schema default always applies. + if ( execCtx.gpuMemoryBudget == 0 ) + { + execCtx.gpuMemoryBudget = gpuMemoryBudget; + } + if ( execCtx.maxOutputArtifactBytes == 0 ) + { + execCtx.maxOutputArtifactBytes = outputArtifactBudget; + } + if ( execCtx.deadlineMs == 0 ) + { + execCtx.deadlineMs = deadlineMs; + } + + // Progress callback logs events at stage boundaries (D-10). Only install the + // default logging callback when the caller did not already supply their own + // via the 5-arg Process() overload — otherwise a caller-supplied callback + // (e.g. one capturing ProgressEvents for a test) would be silently discarded. + if ( !execCtx.progressCallback ) + { + execCtx.progressCallback = [this]( const ProgressEvent &ev ) + { + m_logger->info( "Progress: pass={} percent={:.1f}", ev.pass_id, ev.percent ); + }; + } + + // Wire deadline timer (D-05, D-09) + boost::asio::deadline_timer deadlineTimer( *ioc ); + if ( deadlineMs > 0 ) + { + deadlineTimer.expires_from_now( boost::posix_time::milliseconds( deadlineMs ) ); + deadlineTimer.async_wait( [&execCtx]( const boost::system::error_code &ec ) + { + if ( !ec ) + { + // D-09: deadline → unified cancel path + execCtx.cancelToken.Cancel(); + } + } ); + } + + // Register cancel callback: if explicit cancel happens first, cancel the timer + execCtx.cancelToken.SetCallback( [&deadlineTimer]() + { + deadlineTimer.cancel(); + } ); + + // Capture start time before StartProcessing (D-13) + auto startTimeUsec = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() ).count(); + + // Extract executor identity from CapabilityValidator (Phase 06 D-08) + uint8_t executorId[SHA256_HASH_SIZE] = {}; + if ( m_capabilityValidator ) + { + auto *snap = m_capabilityValidator->GetSnapshot(); + if ( snap && snap->identityHash.size() >= SHA256_HASH_SIZE ) + { + std::memcpy( executorId, snap->identityHash.data(), SHA256_HASH_SIZE ); + } + } + + // Call new 6-arg StartProcessing() overload (D-18) + auto processResult = m_processor->StartProcessing( chunkhashes, + processing_.get_inputs()[index.value()], + *buffers->second, + *buffers->first, + parameters, + execCtx ); + + // Cancel deadline timer after StartProcessing returns (whether success or failure) + deadlineTimer.cancel(); + + // Capture end time after StartProcessing returns (D-13) + auto endTimeUsec = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() ).count(); + + // Map terminal error to TerminalState for manifest (D-15) + TerminalState terminalState = TerminalState::Success; + if ( processResult.error ) + { + switch ( processResult.error->stage ) + { + case ProcessingErrorStage::CANCELLED: terminalState = TerminalState::Cancelled; break; + case ProcessingErrorStage::TIMED_OUT: terminalState = TerminalState::Timeout; break; + case ProcessingErrorStage::BUDGET_EXCEEDED: terminalState = TerminalState::BudgetExceeded; break; + default: terminalState = TerminalState::Error; break; + } + } + + // Check terminal conditions before saving (D-15) + if ( processResult.error ) + { + if ( processResult.error->stage == ProcessingErrorStage::CANCELLED ) + { + m_logger->error( "Processing cancelled" ); + return outcome::failure( Error::PROCESSING_FAILED ); + } + if ( processResult.error->stage == ProcessingErrorStage::TIMED_OUT ) + { + m_logger->error( "Processing deadline exceeded" ); + return outcome::failure( Error::PROCESSING_FAILED ); + } + if ( processResult.error->stage == ProcessingErrorStage::BUDGET_EXCEEDED ) + { + m_logger->error( "Processing output budget exceeded" ); + return outcome::failure( Error::PROCESSING_FAILED ); + } + } - if ( !bufferData.empty() ) + if ( processResult.error || processResult.hash.empty() ) { - FileManager::GetInstance().InitializeSingletons(); - bool hasSaves = false; + m_logger->error( "Processing failed: {}", + processResult.error + ? processResult.error->message + : std::string( "processor returned an empty hash with no result (legacy failure sentinel)" ) ); + return outcome::failure( Error::PROCESSING_FAILED ); + } - // Pre-allocate location slots matching the number of outputs - output_locations.clear(); - output_locations.resize( outputs.size() ); + // ── Build ProcessOutput: artifact records + execution manifest (Phase 08) ── + ProcessOutput output{}; + const auto &procInput = processing_.get_inputs()[index.value()]; + const auto &outputs = processing_.get_outputs(); - // Collect save location shared_ptrs for post-ioc collection - std::vector> locationPtrs; - locationPtrs.resize( outputs.size() ); + if ( processResult.output_buffers && !outputs.empty() ) + { + const auto &bufferNames = processResult.output_buffers->first; + const auto &bufferData = processResult.output_buffers->second; - for ( size_t outputIndex = 0; outputIndex < outputs.size(); ++outputIndex ) + // Build one Artifact per output buffer + for ( size_t outIdx = 0; outIdx < outputs.size() && outIdx < bufferData.size(); ++outIdx ) { - const auto &output = outputs[outputIndex]; - const auto &outputUrl = output.get_source_uri_param(); - if ( outputUrl.empty() ) + Artifact art{}; + + // Identity (ARTF-01) + std::strncpy( art.resourceName, outputs[outIdx].get_name().c_str(), MAX_RESOURCE_NAME - 1 ); + std::strncpy( art.passId, pass.get_name().c_str(), MAX_RESOURCE_NAME - 1 ); { - continue; + std::string binding = "output:" + outputs[outIdx].get_name(); + std::strncpy( art.outputBinding, binding.c_str(), MAX_RESOURCE_NAME - 1 ); + } + + // Format metadata (ARTF-02) + { + // Map DataType enum to string + static const char *dataTypeNames[] = { + "BOOL", "BUFFER", "FLOAT", "INT", "MAT2", "MAT3", "MAT4", + "STRING", "TENSOR", "TEXTURE1_D", "TEXTURE2_D", "TEXTURE3_D", + "TEXTURE_CUBE", "VEC2", "VEC3", "VEC4" + }; + int dtIdx = static_cast( procInput.get_type() ); + if ( dtIdx >= 0 && dtIdx < static_cast( sizeof( dataTypeNames ) / sizeof( dataTypeNames[0] ) ) ) + { + std::strncpy( art.dataType, dataTypeNames[dtIdx], 63 ); + } } - if ( !IsUrl( outputUrl ) ) { - m_logger->warn( "Output source_uri_param '{}' is not a URL; skipping save", outputUrl ); - continue; + // Map InputFormat enum to string. procInput.get_format() is optional -- + // e.g. BUFFER-type inputs (such as a render pass's vertex_buffer source) + // may omit "format" entirely, defaulting to INT8 per the same convention + // already applied above in CheckProcessValidity()'s BUFFER case (see the + // "Buffer input missing format; defaulting to INT8" warning). + static const char *formatNames[] = { + "FLOAT16", "FLOAT32", "FP4_ULTRA", "INT16", "INT32", "INT8", "RGB8", "RGBA8" + }; + sgns::InputFormat fmt = procInput.get_format().value_or( sgns::InputFormat::INT8 ); + int fmtIdx = static_cast( fmt ); + if ( fmtIdx >= 0 && fmtIdx < static_cast( sizeof( formatNames ) / sizeof( formatNames[0] ) ) ) + { + std::strncpy( art.format, formatNames[fmtIdx], 63 ); + } + } + if ( procInput.get_dimensions() ) + { + auto dims = procInput.get_dimensions().value(); + if ( dims.get_block_len() ) + art.width = static_cast( dims.get_block_len().value() ); + if ( dims.get_block_line_stride() ) + art.height = static_cast( dims.get_block_line_stride().value() ); + art.depth = 1; // 2D texture convention } + art.byteSize = bufferData[outIdx].size(); + std::strncpy( art.mediaType, "application/octet-stream", MAX_MEDIA_TYPE - 1 ); - const size_t dataIndex = ( bufferData.size() == outputs.size() ) ? outputIndex : 0; - if ( dataIndex >= bufferData.size() ) + // Content hash (ARTF-03, D-01) + ComputeArtifactIdentity( art, + reinterpret_cast( bufferData[outIdx].data() ), + bufferData[outIdx].size() ); + + // Chunk hashes from processor output (D-08) + for ( const auto &ch : chunkhashes ) { - continue; + if ( ch.size() >= SHA256_HASH_SIZE ) + { + AddChunkHash( art, ch.data() ); + } + } + + output.artifacts.push_back( art ); + } + + // ── Assemble ExecutionManifest (ARTF-04, D-13) ── + ExecutionManifest &manifest = output.manifest; + + // Identifiers — use schema name as executionId; attempt/task/subtask + // IDs are not tracked in v2.0 schema (left as empty strings per D-14 sentinel convention) + std::strncpy( manifest.executionId, processing_.get_name().c_str(), MAX_IDENTIFIER - 1 ); + // attemptId, taskId, subtaskId default to empty (zero-initialized) + std::strncpy( manifest.passId, pass.get_name().c_str(), MAX_RESOURCE_NAME - 1 ); + + // Executor identity + std::memcpy( manifest.executorIdentity, executorId, SHA256_HASH_SIZE ); + + // Model identity: SHA-256 of model bytes if model used (D-14) + if ( pass.get_model() ) + { + const auto &modelBytes = *buffers->first; // model file bytes + if ( !modelBytes.empty() ) + { + auto modelHash = sgns::sgprocmanagersha::sha256( + modelBytes.data(), modelBytes.size() ); + std::memcpy( manifest.modelIdentity, modelHash.data(), SHA256_HASH_SIZE ); } + } + + // Shader identity: SHA-256 of SPIR-V bytes if RENDER pass (D-14) + if ( pass.get_type() == PassType::RENDER && pass.get_render_target() ) + { + // The SPIR-V was compiled earlier in GetCidForProc — we compute + // the model identity from the shader bytes stored in buffers->second + // (second is image data, first is model/shader data for render passes) + // For now: shaderIdentity stays zero — SPIR-V bytes not tracked separately. + // Future: populate from compiled SPIR-V cache. + } + + // Output artifact hashes + manifest.outputArtifactCount = static_cast( + std::min( output.artifacts.size(), static_cast( MAX_ARTIFACT_REFS ) ) ); + for ( size_t i = 0; i < manifest.outputArtifactCount; ++i ) + { + std::memcpy( manifest.outputArtifactHashes[i], + output.artifacts[i].artifactId, SHA256_HASH_SIZE ); + } + + // Timing + manifest.startTimeUsec = startTimeUsec; + manifest.endTimeUsec = endTimeUsec; + manifest.wallClockUsec = endTimeUsec - startTimeUsec; + + // Terminal state + manifest.terminalState = terminalState; + + // Resource summary + manifest.outputBytesProduced = 0; + for ( const auto &art : output.artifacts ) + { + manifest.outputBytesProduced += art.byteSize; + } + + // Compute manifest self-hash (D-04) + // Hash a timing-zeroed copy so combinedHash/manifestHash are deterministic + // across separate Process() calls; the live manifest returned to the caller + // keeps its real startTimeUsec/endTimeUsec/wallClockUsec for provenance (ARTF-04). + ExecutionManifest hashInput = manifest; + hashInput.startTimeUsec = 0; + hashInput.endTimeUsec = 0; + hashInput.wallClockUsec = 0; + auto mHash = ComputeManifestHash( hashInput ); + std::memcpy( manifest.manifestHash, mHash.data(), SHA256_HASH_SIZE ); + output.combinedHash = mHash; + } + + // ── Existing FileManager save loop (unchanged) ── + if ( processResult.output_buffers && !outputs.empty() ) + { + const auto &bufferNames = processResult.output_buffers->first; + const auto &bufferData = processResult.output_buffers->second; - const size_t nameIndex = ( bufferNames.size() == outputs.size() ) ? outputIndex : 0; - std::string outputFileName; - if ( !UrlHasExtension( outputUrl ) ) + if ( !bufferData.empty() ) + { + FileManager::GetInstance().InitializeSingletons(); + bool hasSaves = false; + + // Pre-allocate location slots matching the number of outputs + output_locations.clear(); + output_locations.resize( outputs.size() ); + + // Collect save location shared_ptrs for post-ioc collection + std::vector> locationPtrs; + locationPtrs.resize( outputs.size() ); + + for ( size_t outputIndex = 0; outputIndex < outputs.size(); ++outputIndex ) { - std::string baseName; - if ( nameIndex < bufferNames.size() && !bufferNames[nameIndex].empty() ) + const auto &output = outputs[outputIndex]; + const auto &outputUrl = output.get_source_uri_param(); + if ( outputUrl.empty() ) { - baseName = bufferNames[nameIndex]; + continue; } - else + if ( !IsUrl( outputUrl ) ) { - baseName = output.get_name() + ".raw"; + m_logger->warn( "Output source_uri_param '{}' is not a URL; skipping save", outputUrl ); + continue; } - if ( EndsWithSlash( outputUrl ) ) + const size_t dataIndex = ( bufferData.size() == outputs.size() ) ? outputIndex : 0; + if ( dataIndex >= bufferData.size() ) { - outputFileName = baseName; + continue; } - else + + const size_t nameIndex = ( bufferNames.size() == outputs.size() ) ? outputIndex : 0; + std::string outputFileName; + if ( !UrlHasExtension( outputUrl ) ) { - outputFileName = "/" + baseName; + std::string baseName; + if ( nameIndex < bufferNames.size() && !bufferNames[nameIndex].empty() ) + { + baseName = bufferNames[nameIndex]; + } + else + { + baseName = output.get_name() + ".raw"; + } + + if ( EndsWithSlash( outputUrl ) ) + { + outputFileName = baseName; + } + else + { + outputFileName = "/" + baseName; + } } - } - auto saveBuffers = - std::make_shared, std::vector>>>(); - saveBuffers->first.push_back( outputFileName ); - saveBuffers->second.push_back( bufferData[dataIndex] ); + auto saveBuffers = + std::make_shared, std::vector>>>(); + saveBuffers->first.push_back( outputFileName ); + saveBuffers->second.push_back( bufferData[dataIndex] ); - // Create a shared_ptr to capture the save location from the saver - auto saveLocation = std::make_shared(); - locationPtrs[outputIndex] = saveLocation; + // Create a shared_ptr to capture the save location from the saver + auto saveLocation = std::make_shared(); + locationPtrs[outputIndex] = saveLocation; - FileManager::GetInstance().SaveASync( outputUrl, - outcome::success( saveBuffers ), - ioc, - [this, outputUrl]( const FileManager::ResultType &result ) - { - if ( !result ) + FileManager::GetInstance().SaveASync( outputUrl, + outcome::success( saveBuffers ), + ioc, + [this, outputUrl]( const FileManager::ResultType &result ) { - m_logger->error( "Failed to save output to {}: {}", - outputUrl, - result.error().message() ); - } - }, - saveLocation ); - hasSaves = true; - - // Dual-save: persist a local copy when output is IPFS - // This ensures the producing node can re-serve data after restart. - std::string urlPrefix, urlPath, urlExt; - getURLComponents( outputUrl, urlPrefix, urlPath, urlExt ); - if ( urlPrefix == "ipfs" ) - { - auto cacheDir = FileManager::GetInstance().getCacheDir(); - if ( !cacheDir.empty() ) + if ( !result ) + { + m_logger->error( "Failed to save output to {}: {}", + outputUrl, + result.error().message() ); + } + }, + saveLocation ); + hasSaves = true; + + // Dual-save: persist a local copy when output is IPFS + // This ensures the producing node can re-serve data after restart. + std::string urlPrefix, urlPath, urlExt; + getURLComponents( outputUrl, urlPrefix, urlPath, urlExt ); + if ( urlPrefix == "ipfs" ) { - auto localUrl = "file://" + cacheDir + "/results/" + - output.get_name() + outputFileName; - FileManager::GetInstance().SaveASync( - localUrl, - outcome::success( saveBuffers ), - ioc, - nullptr, // no callback needed for local save - nullptr ); // no save_location needed + auto cacheDir = FileManager::GetInstance().getCacheDir(); + if ( !cacheDir.empty() ) + { + auto localUrl = "file://" + cacheDir + "/results/" + + output.get_name() + outputFileName; + FileManager::GetInstance().SaveASync( + localUrl, + outcome::success( saveBuffers ), + ioc, + nullptr, // no callback needed for local save + nullptr ); // no save_location needed + } } } - } - - if ( hasSaves ) - { - ioc->reset(); - ioc->run(); - // After async IO completes, collect the save locations - for ( size_t i = 0; i < locationPtrs.size(); ++i ) + if ( hasSaves ) { - if ( locationPtrs[i] && !locationPtrs[i]->empty() ) + ioc->reset(); + ioc->run(); + + // After async IO completes, collect the save locations + for ( size_t i = 0; i < locationPtrs.size(); ++i ) { - output_locations[i] = *locationPtrs[i]; + if ( locationPtrs[i] && !locationPtrs[i]->empty() ) + { + output_locations[i] = *locationPtrs[i]; + } } } } } + + return output; + } + catch ( const std::exception &e ) + { + m_logger->error( "Process() exception: {}", e.what() ); + if ( m_processor ) + { + m_processor->RunTeardown(); + } + return outcome::failure( Error::PROCESSING_FAILED ); + } + } + + // ── ProcessingResult Migration Adapter (D-10) ────────────────────────── + // Temporary: maps new ProcessOutput back to legacy ProcessingResult shape. + // Removed before Phase 08 ships per D-10/D-12. + + ProcessingResult ProcessingResult::FromProcessOutput( const ProcessOutput &output ) + { + ProcessingResult result; + result.hash = output.combinedHash; + + if ( !output.artifacts.empty() ) + { + auto buffers = std::make_shared, std::vector>>>(); + for ( const auto &art : output.artifacts ) + { + buffers->first.push_back( std::string( art.resourceName ) ); + // Raw bytes not stored in Artifact struct (only hash). + // Callers needing raw bytes must use ProcessOutput directly. + buffers->second.push_back( {} ); + } + result.output_buffers = buffers; } - return processResult.hash; + return result; } outcome::result>, std::shared_ptr>>>> @@ -837,24 +1690,176 @@ namespace sgns::sgprocessing std::make_shared>(), std::make_shared>() ); - std::string modelFile = processing_.get_passes()[index.value()].get_model().value().get_source_uri_param(); + const auto &p = processing_.get_passes()[index.value()]; + const bool isRender = ( p.get_type() == PassType::RENDER && p.get_render_shader() ); - std::string image = processing_.get_inputs()[index.value()].get_source_uri_param(); - m_logger->info( "Model Input URL: {}", modelFile ); - m_logger->info( "Data Input URL: {}", image ); //Init Loaders FileManager::GetInstance().InitializeSingletons(); - //Get Model - string modelURL = modelFile; - GetSubCidForProc( ioc, modelURL, mainbuffers->first ); - string imageUrl = image; - GetSubCidForProc( ioc, imageUrl, mainbuffers->second ); + // Per-stage fetch buffers for the render path -- queued alongside the + // existing image fetch below so the single existing ioc->run() call + // still drains everything in one pass (no new synchronization + // primitive needed). + std::vector>>> stageBuffers; + + // Independently-resolved vertex/index buffer fetch buffers (Task 1 -- + // resolved via the "input:name" prefix, NOT the coincidental single + // model-index input `mainbuffers->second` used to carry today). + std::shared_ptr> vertexBuffer; + std::shared_ptr> indexBuffer; + bool hasIndexBuffer = false; + sgns::IndexType indexType = sgns::IndexType::UINT16; + + if ( isRender ) + { + // NOTE: get_render_shader() returns boost::optional BY VALUE + // (quicktype's standard convention for optional accessors) -- binding `stages` as a + // reference into a chained `.value().get_stages()` call would dangle the moment this + // statement ends, since the temporary optional/RenderShaderConfig backing that + // reference is destroyed at the semicolon. Copy the optional into a named local first + // so its lifetime covers the loop below. + const auto renderShader = p.get_render_shader().value(); + const std::vector &stages = renderShader.get_stages(); + for ( const auto &stage : stages ) + { + auto tempBuffer = std::make_shared>(); + GetSubCidForProc( ioc, stage.get_source(), tempBuffer ); + stageBuffers.emplace_back( stage, tempBuffer ); + } + // For a render pass, mainbuffers->first is populated by + // SerializeCompiledStages() below, not by a raw modelURL fetch -- + // skip the old single GetSubCidForProc(ioc, modelURL, ...) call + // entirely for this pass type. + + // Resolve vertex_buffer.source as an independently-named "input:" + // reference. CheckProcessValidity() already requires vertex_buffer to + // be present and (Task 2) requires its source to start with "input:" -- + // this call-site check is defense-in-depth, not the primary rejection + // point. + const auto vertexBufferCfg = p.get_vertex_buffer().value(); + const std::string vertexSource = vertexBufferCfg.get_source(); + if ( vertexSource.rfind( "input:", 0 ) != 0 ) + { + return outcome::failure( Error::MISSING_INPUT ); + } + auto vertexInputIndex = GetInputIndex( vertexSource ); + if ( !vertexInputIndex ) + { + return outcome::failure( Error::MISSING_INPUT ); + } + std::string vertexUrl = processing_.get_inputs()[vertexInputIndex.value()].get_source_uri_param(); + vertexBuffer = std::make_shared>(); + GetSubCidForProc( ioc, vertexUrl, vertexBuffer ); + + // index_buffer is optional; if present but its source is absent, that's + // a schema-permitted-but-unusable-here state -- treat as no index + // buffer (skip, do not error). + const auto indexBufferOpt = p.get_index_buffer(); + if ( indexBufferOpt && indexBufferOpt.value().get_source() ) + { + const auto indexBufferCfg = indexBufferOpt.value(); + std::string indexSource = indexBufferCfg.get_source().value(); + if ( indexSource.rfind( "input:", 0 ) != 0 ) + { + return outcome::failure( Error::MISSING_INPUT ); + } + auto indexInputIndex = GetInputIndex( indexSource ); + if ( !indexInputIndex ) + { + return outcome::failure( Error::MISSING_INPUT ); + } + std::string indexUrl = processing_.get_inputs()[indexInputIndex.value()].get_source_uri_param(); + indexBuffer = std::make_shared>(); + hasIndexBuffer = true; + indexType = indexBufferCfg.get_index_type().value_or( sgns::IndexType::UINT16 ); + GetSubCidForProc( ioc, indexUrl, indexBuffer ); + } + } + else + { + std::string modelFile = p.get_model().value().get_source_uri_param(); + m_logger->info( "Model Input URL: {}", modelFile ); + + string modelURL = modelFile; + GetSubCidForProc( ioc, modelURL, mainbuffers->first ); + } + + if ( !isRender ) + { + // For a render pass, mainbuffers->second is populated by + // SerializeRenderPassConfig() below, not by this raw single fetch -- + // `index` here is the coincidental pass-index-as-input-index value, + // not any render-specific buffer. + std::string image = processing_.get_inputs()[index.value()].get_source_uri_param(); + m_logger->info( "Data Input URL: {}", image ); + + string imageUrl = image; + GetSubCidForProc( ioc, imageUrl, mainbuffers->second ); + } //Run IO ioc->reset(); ioc->run(); + if ( isRender ) + { + std::vector compiledStages; + std::vector entryPoints; + compiledStages.reserve( stageBuffers.size() ); + entryPoints.reserve( stageBuffers.size() ); + for ( auto &entry : stageBuffers ) + { + const auto &stage = entry.first; + auto &tempBuffer = entry.second; + + std::string entryPoint = stage.get_entry_point().value_or( "main" ); + + sgns::sgprocessing::ShaderCompiler compiler; + auto compileResult = compiler.CompileAndValidate( *tempBuffer, + stage.get_stage(), + stage.get_type(), + entryPoint ); + if ( !compileResult ) + { + if ( compileResult.error() == sgns::sgprocessing::ShaderCompiler::Error::VALIDATION_FAILED ) + { + return outcome::failure( Error::SPIRV_VALIDATION_FAILED ); + } + return outcome::failure( Error::SHADER_COMPILE_FAILED ); + } + compiledStages.push_back( compileResult.value() ); + entryPoints.push_back( std::move( entryPoint ) ); + } + + *mainbuffers->first = SerializeCompiledStages( compiledStages, entryPoints ); + + // Preserve the pre-existing INPUT_UNAVAIL failure semantics: previously + // this pass type's mainbuffers->second WAS the raw vertex/model fetch + // buffer, so an unresolvable source URI surfaced here via the + // mainbuffers->second->size() <= 0 check below. Now mainbuffers->second + // is always populated with a non-empty SerializeRenderPassConfig() + // header regardless of fetch success, so that check alone would no + // longer catch a failed vertex-buffer fetch -- check it explicitly. + if ( vertexBuffer->empty() ) + { + return outcome::failure( Error::INPUT_UNAVAIL ); + } + + static const std::vector kEmptyIndexBytes; + *mainbuffers->second = SerializeRenderPassConfig( p.get_render_target().value(), + p.get_pipeline_state(), + p.get_vertex_layout().value(), + p.get_render_shader().value().get_uniforms(), + *vertexBuffer, + hasIndexBuffer, + indexType, + hasIndexBuffer ? *indexBuffer : kEmptyIndexBytes, + p.get_data_transforms() + ? static_cast( + p.get_data_transforms()->size() ) + : 0u ); + } + if ( mainbuffers == nullptr ) { return outcome::failure( Error::INPUT_UNAVAIL ); @@ -912,6 +1917,22 @@ namespace sgns::sgprocessing "file" ); } + void ProcessingManager::CanExecute( const sgns::Pass &pass, + sgns::sgprocessing::CanExecuteCallback callback ) + { + if ( !m_capabilityValidator ) + { + CanExecuteResult result; + result.executable = false; + result.unmet.push_back( + { UnmetRequirementCategory::RESOURCE, + "CapabilityValidator not initialized" } ); + callback( result ); + return; + } + m_capabilityValidator->CanExecute( pass, std::move( callback ) ); + } + bool ProcessingManager::IsProcessingValid( const std::string &jsondata ) { auto result = Create( jsondata ); diff --git a/src/processors/CMakeLists.txt b/src/processors/CMakeLists.txt index 48990d9..792bc33 100644 --- a/src/processors/CMakeLists.txt +++ b/src/processors/CMakeLists.txt @@ -1,3 +1,14 @@ +# Vulkan validation layers toggle (D-24, VVAL-03) +# ON in Debug builds (development convenience), OFF in Release (production). +# Best-effort: instance creation proceeds even if layers are not found (D-20). +string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_UPPER) +if(CMAKE_BUILD_TYPE_UPPER STREQUAL "DEBUG") + set(_vulkan_val_default ON) +else() + set(_vulkan_val_default OFF) +endif() +option(ENABLE_VULKAN_VALIDATION "Enable Vulkan validation layers for RenderProcessor (best-effort)" ${_vulkan_val_default}) + add_library(SGProcessors STATIC processing_processor_mnn_image.cpp processing_processor_mnn_audio.cpp @@ -17,6 +28,8 @@ add_library(SGProcessors STATIC processing_processor_mnn_texturecube.cpp processing_processor_mnn_texture1d.cpp processing_processor_mnn_volume.cpp + processing_processor_render.cpp + vulkan_gpu_probe.cpp ../../include/processors/processing_processor.hpp ../../include/processors/processing_processor_mnn_audio.hpp ../../include/processors/processing_processor_mnn_image.hpp @@ -36,6 +49,8 @@ add_library(SGProcessors STATIC ../../include/processors/processing_processor_mnn_texturecube.hpp ../../include/processors/processing_processor_mnn_texture1d.hpp ../../include/processors/processing_processor_mnn_volume.hpp + ../../include/processors/processing_processor_render.hpp + ../../include/processors/vulkan_gpu_probe.hpp ) @@ -56,8 +71,11 @@ target_link_libraries( sgprocmanagertypes MNN::MNN Vulkan::Vulkan + vk-bootstrap::vk-bootstrap OpenSSL::Crypto sgprocmanagersha + sgprocmanagerquant + sgprocmanagerdiff ) if(APPLE) @@ -83,4 +101,12 @@ if(APPLE) endif() endif() +# Vulkan validation layers compile definition (D-23, D-24) +if(ENABLE_VULKAN_VALIDATION) + target_compile_definitions(SGProcessors PUBLIC ENABLE_VULKAN_VALIDATION) + message(STATUS "Vulkan validation layers: ENABLED (best-effort)") +else() + message(STATUS "Vulkan validation layers: DISABLED") +endif() + sgnus_install(SGProcessors) diff --git a/src/processors/processing_processor_mnn_audio.cpp b/src/processors/processing_processor_mnn_audio.cpp index 4827370..3f5e585 100644 --- a/src/processors/processing_processor_mnn_audio.cpp +++ b/src/processors/processing_processor_mnn_audio.cpp @@ -15,9 +15,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &imageData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { (void)parameters; + (void)execCtx; std::vector modelFile_bytes; modelFile_bytes.assign(modelFile.begin(), modelFile.end()); diff --git a/src/processors/processing_processor_mnn_bool.cpp b/src/processors/processing_processor_mnn_bool.cpp index 526a563..bfe7058 100644 --- a/src/processors/processing_processor_mnn_bool.cpp +++ b/src/processors/processing_processor_mnn_bool.cpp @@ -1,11 +1,14 @@ #include "processors/processing_processor_mnn_bool.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -183,9 +186,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &boolData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -261,8 +266,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Bool pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Bool pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchLength ), 0.0f ); for ( int i = 0; i < patchLength; ++i ) @@ -311,7 +333,19 @@ namespace sgns::sgprocessing } } - std::vector shahash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + std::vector shahash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); std::string hashString( shahash.begin(), shahash.end() ); chunkhashes.push_back( shahash ); @@ -336,8 +370,29 @@ namespace sgns::sgprocessing } } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + m_progress = 100.0f; + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -354,6 +409,9 @@ namespace sgns::sgprocessing m_logger->info( "Bool processing complete" ); + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -371,16 +429,24 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return std::make_unique(); } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensors = interpreter->getSessionInputAll( session ); if ( inputTensors.empty() ) { diff --git a/src/processors/processing_processor_mnn_buffer.cpp b/src/processors/processing_processor_mnn_buffer.cpp index 2a1ea62..90baab4 100644 --- a/src/processors/processing_processor_mnn_buffer.cpp +++ b/src/processors/processing_processor_mnn_buffer.cpp @@ -1,10 +1,13 @@ #include "processors/processing_processor_mnn_buffer.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -134,9 +137,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &bufferData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -192,8 +197,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Buffer pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Buffer pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchLength ), 0.0f ); for ( int i = 0; i < patchLength; ++i ) @@ -242,7 +264,19 @@ namespace sgns::sgprocessing } } - std::vector shahash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + std::vector shahash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); std::string hashString( shahash.begin(), shahash.end() ); chunkhashes.push_back( shahash ); @@ -250,6 +284,13 @@ namespace sgns::sgprocessing subTaskResultHash = sgprocmanagersha::sha256( combinedHash.c_str(), combinedHash.length() ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + if ( !stitchedOutput.empty() ) { for ( int c = 0; c < outputChannels; ++c ) @@ -269,6 +310,20 @@ namespace sgns::sgprocessing m_progress = 100.0f; + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -285,6 +340,9 @@ namespace sgns::sgprocessing m_logger->info( "Buffer processing complete" ); + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -302,16 +360,24 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return std::make_unique(); } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensors = interpreter->getSessionInputAll( session ); if ( inputTensors.empty() ) { diff --git a/src/processors/processing_processor_mnn_float.cpp b/src/processors/processing_processor_mnn_float.cpp index c7c53c0..c02aefa 100644 --- a/src/processors/processing_processor_mnn_float.cpp +++ b/src/processors/processing_processor_mnn_float.cpp @@ -1,11 +1,14 @@ #include "processors/processing_processor_mnn_float.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -166,9 +169,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &floatData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -233,8 +238,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Float pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Float pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchLength ), 0.0f ); for ( int i = 0; i < patchLength; ++i ) @@ -283,10 +305,29 @@ namespace sgns::sgprocessing } } - auto hash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + for ( size_t idx = 0; idx < stitchedOutput.size(); ++idx ) { const int spatialIdx = static_cast( idx % length ); @@ -297,12 +338,45 @@ namespace sgns::sgprocessing } } + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the stitched-combined site. + // stitchedOutput is locally-owned, so quantizing it in place is safe. + std::vector preQuantizeSnapshot; + if ( execCtx.rawOutputCapture ) + { + const auto *preBytes = reinterpret_cast( stitchedOutput.data() ); + preQuantizeSnapshot.assign( preBytes, preBytes + stitchedOutput.size() * sizeof( float ) ); + } + sgprocmanagerquant::QuantizeFloatBuffer( stitchedOutput.data(), stitchedOutput.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( stitchedOutput.data() ); + execCtx.rawOutputCapture( + std::vector( quantizedBytes, quantizedBytes + stitchedOutput.size() * sizeof( float ) ), + preQuantizeSnapshot ); + } + std::string stitchedStr( reinterpret_cast( stitchedOutput.data() ), stitchedOutput.size() * sizeof( float ) ); subTaskResultHash = sgprocmanagersha::sha256( stitchedStr.c_str(), stitchedStr.size() ); m_progress = 100.0f; + m_progress = 100.0f; + + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -319,6 +393,10 @@ namespace sgns::sgprocessing } m_logger->info( "Float processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -326,25 +404,40 @@ namespace sgns::sgprocessing std::vector &modelFile, int length ) { - auto interpreter = std::unique_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); if ( !interpreter ) { m_logger->error( "Failed to create MNN interpreter from buffer" ); return nullptr; } + //MNN::BackendConfig backendConfig; + //backendConfig.precision = MNN::BackendConfig::Precision_High; + // Tested 2026-08-13 (Phase 13 gap-closure follow-up): forcing Precision_High produced + // a bit-for-bit IDENTICAL chunk-10 divergence vs. default Precision_Normal (maxAbsDelta, + // maxRelDelta, maxUlpDistance all unchanged) -- rules out FP16 backend opportunism as the + // source of this fixture's cross-hardware divergence. See STATE.md Blockers/Concerns. + MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_image.cpp b/src/processors/processing_processor_mnn_image.cpp index 5b71d70..aa2842a 100644 --- a/src/processors/processing_processor_mnn_image.cpp +++ b/src/processors/processing_processor_mnn_image.cpp @@ -1,10 +1,12 @@ #include "processors/processing_processor_mnn_image.hpp" #include "datasplitter/ImageSplitter.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include #include // For SHA256_DIGEST_LENGTH #include "util/sha256.hpp" +#include "util/quantization.hpp" #include "util/InputTypes.hpp" //#define STB_IMAGE_IMPLEMENTATION @@ -20,9 +22,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &imageData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFile_bytes; modelFile_bytes.assign(modelFile.begin(), modelFile.end()); @@ -61,12 +65,28 @@ namespace sgns::sgprocessing auto totalChunks = proc.get_dimensions().value().get_chunk_count().value(); m_progress = 0.0f; // Reset progress at start + + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Image pass cancelled" } }; + } for ( int chunkIdx = 0; chunkIdx < totalChunks; ++chunkIdx ) { m_logger->info( "Chunk IDX {} Total {}", chunkIdx, totalChunks ); + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Image pass cancelled" } }; + } std::vector shahash( SHA256_DIGEST_LENGTH ); // Chunk result hash should be calculated @@ -83,7 +103,20 @@ namespace sgns::sgprocessing const float *data = procresults->host(); size_t dataSize = procresults->elementSize() * sizeof( float ); - shahash = sgprocmanagersha::sha256( data, dataSize ); + + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + shahash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); std::string hashString( shahash.begin(), shahash.end() ); chunkhashes.push_back( shahash ); @@ -95,8 +128,22 @@ namespace sgns::sgprocessing std::this_thread::sleep_for(std::chrono::milliseconds(100)); } + + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + + m_progress = 100.0f; + ProcessingResult result; result.hash = subTaskResultHash; + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; //} //return subTaskResultHash; @@ -109,10 +156,7 @@ namespace sgns::sgprocessing const int origheight, const std::string filename) { - // ponytail: MNN's Vulkan backend is not safe to initialize concurrently. Keep the - // process-wide lock until MNN exposes a shareable runtime/session API. - static std::mutex mnn_vulkan_mutex; - std::lock_guard lock( mnn_vulkan_mutex ); + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); std::vector ret_vect(imgdata); diff --git a/src/processors/processing_processor_mnn_int.cpp b/src/processors/processing_processor_mnn_int.cpp index 5352ffd..c147281 100644 --- a/src/processors/processing_processor_mnn_int.cpp +++ b/src/processors/processing_processor_mnn_int.cpp @@ -1,11 +1,14 @@ #include "processors/processing_processor_mnn_int.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -118,9 +121,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &intData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -199,8 +204,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Int pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Int pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchLength ), 0.0f ); for ( int i = 0; i < patchLength; ++i ) @@ -249,10 +271,29 @@ namespace sgns::sgprocessing } } - auto hash = sgprocmanagersha::sha256( data , dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + for ( size_t idx = 0; idx < stitchedOutput.size(); ++idx ) { const int spatialIdx = static_cast( idx % length ); @@ -263,12 +304,45 @@ namespace sgns::sgprocessing } } + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the stitched-combined site. + // stitchedOutput is locally-owned, so quantizing it in place is safe. + std::vector preQuantizeSnapshot; + if ( execCtx.rawOutputCapture ) + { + const auto *preBytes = reinterpret_cast( stitchedOutput.data() ); + preQuantizeSnapshot.assign( preBytes, preBytes + stitchedOutput.size() * sizeof( float ) ); + } + sgprocmanagerquant::QuantizeFloatBuffer( stitchedOutput.data(), stitchedOutput.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( stitchedOutput.data() ); + execCtx.rawOutputCapture( + std::vector( quantizedBytes, quantizedBytes + stitchedOutput.size() * sizeof( float ) ), + preQuantizeSnapshot ); + } + std::string stitchedStr( reinterpret_cast( stitchedOutput.data() ), stitchedOutput.size() * sizeof( float ) ); subTaskResultHash = sgprocmanagersha::sha256( stitchedStr.c_str(), stitchedStr.size() ); m_progress = 100.0f; + m_progress = 100.0f; + + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -285,6 +359,10 @@ namespace sgns::sgprocessing } m_logger->info( "Int processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -292,7 +370,7 @@ namespace sgns::sgprocessing std::vector &modelFile, int length ) { - auto interpreter = std::unique_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); if ( !interpreter ) { m_logger->error( "Failed to create MNN interpreter from buffer" ); @@ -300,17 +378,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_mat2.cpp b/src/processors/processing_processor_mnn_mat2.cpp index 3164675..e492145 100644 --- a/src/processors/processing_processor_mnn_mat2.cpp +++ b/src/processors/processing_processor_mnn_mat2.cpp @@ -1,10 +1,13 @@ #include "processors/processing_processor_mnn_mat2.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -184,9 +187,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &mat2Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -255,8 +260,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Mat2 pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Mat2 pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchMatrices ) * 4, 0.0f ); @@ -311,10 +333,29 @@ namespace sgns::sgprocessing } } - auto hash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + for ( size_t idx = 0; idx < stitchedOutput.size(); ++idx ) { const int spatialIdx = static_cast( idx % matrixCount ); @@ -325,12 +366,45 @@ namespace sgns::sgprocessing } } + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the stitched-combined site. + // stitchedOutput is locally-owned, so quantizing it in place is safe. + std::vector preQuantizeSnapshot; + if ( execCtx.rawOutputCapture ) + { + const auto *preBytes = reinterpret_cast( stitchedOutput.data() ); + preQuantizeSnapshot.assign( preBytes, preBytes + stitchedOutput.size() * sizeof( float ) ); + } + sgprocmanagerquant::QuantizeFloatBuffer( stitchedOutput.data(), stitchedOutput.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( stitchedOutput.data() ); + execCtx.rawOutputCapture( + std::vector( quantizedBytes, quantizedBytes + stitchedOutput.size() * sizeof( float ) ), + preQuantizeSnapshot ); + } + std::string stitchedStr( reinterpret_cast( stitchedOutput.data() ), stitchedOutput.size() * sizeof( float ) ); subTaskResultHash = sgprocmanagersha::sha256( stitchedStr.c_str(), stitchedStr.size() ); m_progress = 100.0f; + m_progress = 100.0f; + + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -347,6 +421,10 @@ namespace sgns::sgprocessing } m_logger->info( "Mat2 processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -354,7 +432,7 @@ namespace sgns::sgprocessing std::vector &modelFile, int length ) { - auto interpreter = std::unique_ptr( + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); if ( !interpreter ) { @@ -363,17 +441,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_mat3.cpp b/src/processors/processing_processor_mnn_mat3.cpp index f8fa1e6..8c60c60 100644 --- a/src/processors/processing_processor_mnn_mat3.cpp +++ b/src/processors/processing_processor_mnn_mat3.cpp @@ -1,10 +1,13 @@ #include "processors/processing_processor_mnn_mat3.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -184,9 +187,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &mat3Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -255,8 +260,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Mat3 pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Mat3 pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchMatrices ) * 9, 0.0f ); @@ -311,10 +333,29 @@ namespace sgns::sgprocessing } } - auto hash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + for ( size_t idx = 0; idx < stitchedOutput.size(); ++idx ) { const int spatialIdx = static_cast( idx % matrixCount ); @@ -325,12 +366,45 @@ namespace sgns::sgprocessing } } + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the stitched-combined site. + // stitchedOutput is locally-owned, so quantizing it in place is safe. + std::vector preQuantizeSnapshot; + if ( execCtx.rawOutputCapture ) + { + const auto *preBytes = reinterpret_cast( stitchedOutput.data() ); + preQuantizeSnapshot.assign( preBytes, preBytes + stitchedOutput.size() * sizeof( float ) ); + } + sgprocmanagerquant::QuantizeFloatBuffer( stitchedOutput.data(), stitchedOutput.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( stitchedOutput.data() ); + execCtx.rawOutputCapture( + std::vector( quantizedBytes, quantizedBytes + stitchedOutput.size() * sizeof( float ) ), + preQuantizeSnapshot ); + } + std::string stitchedStr( reinterpret_cast( stitchedOutput.data() ), stitchedOutput.size() * sizeof( float ) ); subTaskResultHash = sgprocmanagersha::sha256( stitchedStr.c_str(), stitchedStr.size() ); m_progress = 100.0f; + m_progress = 100.0f; + + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -347,6 +421,10 @@ namespace sgns::sgprocessing } m_logger->info( "Mat3 processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -354,7 +432,7 @@ namespace sgns::sgprocessing std::vector &modelFile, int length ) { - auto interpreter = std::unique_ptr( + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); if ( !interpreter ) { @@ -363,17 +441,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_mat4.cpp b/src/processors/processing_processor_mnn_mat4.cpp index e36a8dd..5eb35c1 100644 --- a/src/processors/processing_processor_mnn_mat4.cpp +++ b/src/processors/processing_processor_mnn_mat4.cpp @@ -1,10 +1,13 @@ #include "processors/processing_processor_mnn_mat4.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -184,9 +187,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &mat4Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -255,8 +260,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Mat4 pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Mat4 pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchMatrices ) * 16, 0.0f ); @@ -311,10 +333,29 @@ namespace sgns::sgprocessing } } - auto hash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + for ( size_t idx = 0; idx < stitchedOutput.size(); ++idx ) { const int spatialIdx = static_cast( idx % matrixCount ); @@ -325,12 +366,45 @@ namespace sgns::sgprocessing } } + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the stitched-combined site. + // stitchedOutput is locally-owned, so quantizing it in place is safe. + std::vector preQuantizeSnapshot; + if ( execCtx.rawOutputCapture ) + { + const auto *preBytes = reinterpret_cast( stitchedOutput.data() ); + preQuantizeSnapshot.assign( preBytes, preBytes + stitchedOutput.size() * sizeof( float ) ); + } + sgprocmanagerquant::QuantizeFloatBuffer( stitchedOutput.data(), stitchedOutput.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( stitchedOutput.data() ); + execCtx.rawOutputCapture( + std::vector( quantizedBytes, quantizedBytes + stitchedOutput.size() * sizeof( float ) ), + preQuantizeSnapshot ); + } + std::string stitchedStr( reinterpret_cast( stitchedOutput.data() ), stitchedOutput.size() * sizeof( float ) ); subTaskResultHash = sgprocmanagersha::sha256( stitchedStr.c_str(), stitchedStr.size() ); m_progress = 100.0f; + m_progress = 100.0f; + + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -347,6 +421,10 @@ namespace sgns::sgprocessing } m_logger->info( "Mat4 processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -354,7 +432,7 @@ namespace sgns::sgprocessing std::vector &modelFile, int length ) { - auto interpreter = std::unique_ptr( + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); if ( !interpreter ) { @@ -363,17 +441,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_ml.cpp b/src/processors/processing_processor_mnn_ml.cpp index 15ec16c..c573169 100644 --- a/src/processors/processing_processor_mnn_ml.cpp +++ b/src/processors/processing_processor_mnn_ml.cpp @@ -10,9 +10,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &imageData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { (void)parameters; + (void)execCtx; std::vector modelFile_bytes; modelFile_bytes.assign(modelFile.begin(), modelFile.end()); diff --git a/src/processors/processing_processor_mnn_string.cpp b/src/processors/processing_processor_mnn_string.cpp index b56c98a..9cf5eee 100644 --- a/src/processors/processing_processor_mnn_string.cpp +++ b/src/processors/processing_processor_mnn_string.cpp @@ -1,11 +1,14 @@ #include "processors/processing_processor_mnn_string.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include +#include #include #include #include #include #include // For SHA256_DIGEST_LENGTH #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -44,26 +47,47 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &textData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; std::vector modelFile_bytes; modelFile_bytes.assign(modelFile.begin(), modelFile.end()); std::vector subTaskResultHash(SHA256_DIGEST_LENGTH); - + // Convert text data to string std::string inputText( textData.begin(), textData.end() ); m_logger->info( "Processing text input: {}", inputText ); - + // For string inputs, we process as a single "chunk" m_progress = 0.0f; - + std::vector shahash( SHA256_DIGEST_LENGTH ); - - // Default max length (could be extracted from parameters) + + // Default max length, overridden below by the job's schema-declared "maxLength" + // parameter when present. Different models compiled into different jobs require + // different FIXED input sequence lengths (e.g. 128 for the legacy multi-input BERT + // model, 16 for the tiny single-input embedding model) -- a single hardcoded literal + // cannot serve both, since resizing a fixed-shape model's input to any length other + // than the one baked in at export time breaks its downstream fully-connected layer. + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); int maxLength = 128; + if ( parameters ) + { + for ( const auto ¶m : *parameters ) + { + if ( param.get_name() == "maxLength" && param.get_type() == sgns::ParameterType::INT ) + { + const auto &def = param.get_parameter_default(); + if ( def.is_number_integer() && def.get() > 0 ) + { + maxLength = def.get(); + } + break; + } + } + } std::vector tokenIds; bool parsedTokenIds = TryParseTokenIds( inputText, tokenIds ); @@ -85,8 +109,23 @@ namespace sgns::sgprocessing } } - auto procresults = Process( tokenIds, modelFile_bytes, maxLength ); - + // resizeLen is the model's fixed/declared sequence length (from the schema's + // maxLength parameter, clamped to at least 1) -- NOT the raw token count -- so a + // fixed-shape model's input tensor is always resized to the exact length its + // compiled graph expects, regardless of how many tokens were actually parsed + // (shorter inputs are zero-padded by the existing fill loop in Process()). + const int resizeLen = std::max( 1, maxLength ); + auto procresults = Process( tokenIds, modelFile_bytes, resizeLen ); + + if ( !procresults || procresults->elementSize() == 0 ) + { + m_logger->error( "MNN string processing produced no output (see prior errors)" ); + ProcessingResult errResult; + errResult.error = ProcessingError{ ProcessingErrorStage::UNSPECIFIED, + "MNN string processing produced no output" }; + return errResult; + } + const float *data = procresults->host(); size_t dataSize = procresults->elementSize() * sizeof( float ); { @@ -103,10 +142,22 @@ namespace sgns::sgprocessing } m_logger->info( "{}", sample.str() ); } - shahash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + shahash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); std::string hashString( shahash.begin(), shahash.end() ); chunkhashes.push_back( shahash ); - + std::string combinedHash = std::string(subTaskResultHash.begin(), subTaskResultHash.end()) + hashString; subTaskResultHash = sgprocmanagersha::sha256( combinedHash.c_str(), combinedHash.length() ); @@ -151,8 +202,12 @@ namespace sgns::sgprocessing MNN::ScheduleConfig config; config.type = MNN_FORWARD_VULKAN; // Use Vulkan backend as requested config.numThread = 4; - - auto session = interpreter->createSession(config); + + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession(config); + } if (!session) { m_logger->error( "Failed to create MNN session" ); return std::make_unique(); @@ -183,7 +238,7 @@ namespace sgns::sgprocessing // BERT models expect: input_ids, attention_mask, token_type_ids (all same shape) for (const auto& inputPair : inputTensors) { auto tensor = inputPair.second; - if (tensor->elementSize() <= 4) { + if (tensor->elementSize() != maxLength) { m_logger->info( "Resizing '{}' to [1, {}]", inputPair.first, maxLength ); interpreter->resizeTensor( tensor, { 1, maxLength } ); } @@ -233,8 +288,13 @@ namespace sgns::sgprocessing // Run inference m_logger->info( "Running MNN inference" ); - interpreter->runSession(session); - + MNN::ErrorCode runResult = interpreter->runSession( session ); + if ( runResult != MNN::NO_ERROR ) + { + m_logger->error( "MNN runSession failed with ErrorCode {}", static_cast( runResult ) ); + return std::make_unique(); + } + // Get output tensor auto outputTensor = interpreter->getSessionOutput(session, nullptr); if (!outputTensor) { diff --git a/src/processors/processing_processor_mnn_tensor.cpp b/src/processors/processing_processor_mnn_tensor.cpp index ec80fad..eda169e 100644 --- a/src/processors/processing_processor_mnn_tensor.cpp +++ b/src/processors/processing_processor_mnn_tensor.cpp @@ -1,10 +1,13 @@ #include "processors/processing_processor_mnn_tensor.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -184,9 +187,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &tensorData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { - (void)parameters; + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -284,8 +289,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Tensor pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Tensor pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchLength ), 0.0f ); for ( int i = 0; i < patchLength; ++i ) @@ -334,10 +356,29 @@ namespace sgns::sgprocessing } } - auto hash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + for ( size_t idx = 0; idx < stitchedOutput.size(); ++idx ) { const int spatialIdx = static_cast( idx % length ); @@ -348,12 +389,45 @@ namespace sgns::sgprocessing } } + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the stitched-combined site. + // stitchedOutput is locally-owned, so quantizing it in place is safe. + std::vector preQuantizeSnapshot; + if ( execCtx.rawOutputCapture ) + { + const auto *preBytes = reinterpret_cast( stitchedOutput.data() ); + preQuantizeSnapshot.assign( preBytes, preBytes + stitchedOutput.size() * sizeof( float ) ); + } + sgprocmanagerquant::QuantizeFloatBuffer( stitchedOutput.data(), stitchedOutput.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( stitchedOutput.data() ); + execCtx.rawOutputCapture( + std::vector( quantizedBytes, quantizedBytes + stitchedOutput.size() * sizeof( float ) ), + preQuantizeSnapshot ); + } + std::string stitchedStr( reinterpret_cast( stitchedOutput.data() ), stitchedOutput.size() * sizeof( float ) ); subTaskResultHash = sgprocmanagersha::sha256( stitchedStr.c_str(), stitchedStr.size() ); m_progress = 100.0f; + m_progress = 100.0f; + + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -370,6 +444,10 @@ namespace sgns::sgprocessing } m_logger->info( "Tensor processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -377,7 +455,7 @@ namespace sgns::sgprocessing std::vector &modelFile, int length ) { - auto interpreter = std::unique_ptr( + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( modelFile.data(), modelFile.size() ) ); if ( !interpreter ) { @@ -386,17 +464,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_texture1d.cpp b/src/processors/processing_processor_mnn_texture1d.cpp index 834d3c4..89178d6 100644 --- a/src/processors/processing_processor_mnn_texture1d.cpp +++ b/src/processors/processing_processor_mnn_texture1d.cpp @@ -1,13 +1,16 @@ #include "processors/processing_processor_mnn_texture1d.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include #include +#include #include #include #include #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -255,8 +258,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &signalData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -332,8 +338,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Texture1D pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Texture1D pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchLength ), 0.0f ); for ( int i = 0; i < patchLength; ++i ) @@ -382,7 +405,19 @@ namespace sgns::sgprocessing } } - std::vector shahash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + std::vector shahash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); std::string hashString( shahash.begin(), shahash.end() ); chunkhashes.push_back( shahash ); @@ -409,8 +444,29 @@ namespace sgns::sgprocessing } } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + m_progress = 100.0f; + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -427,6 +483,9 @@ namespace sgns::sgprocessing m_logger->info( "Texture1D processing complete" ); + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -444,10 +503,14 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); diff --git a/src/processors/processing_processor_mnn_texturecube.cpp b/src/processors/processing_processor_mnn_texturecube.cpp index 2ffed93..2b2c9a1 100644 --- a/src/processors/processing_processor_mnn_texturecube.cpp +++ b/src/processors/processing_processor_mnn_texturecube.cpp @@ -1,14 +1,17 @@ #include "processors/processing_processor_mnn_texturecube.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include #include +#include #include #include #include "datasplitter/ImageSplitter.hpp" #include "util/InputTypes.hpp" #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -255,8 +258,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &cubeData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -350,8 +356,25 @@ namespace sgns::sgprocessing std::vector outputFloats; size_t totalChunks = 0; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "TextureCube pass cancelled" } }; + } + for ( int faceIndex = 0; faceIndex < 6; ++faceIndex ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "TextureCube pass cancelled" } }; + } + const auto &face = faces[faceIndex]; if ( hasChunkFields && isImageFormat ) @@ -377,6 +400,12 @@ namespace sgns::sgprocessing for ( int chunkIdx = 0; chunkIdx < chunkCount; ++chunkIdx ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "TextureCube pass cancelled" } }; + } + const auto chunkData = chunkSplitter.GetPart( chunkIdx ); const int chunkWidth = chunkSplitter.GetPartWidthActual( chunkIdx ); const int chunkHeight = chunkSplitter.GetPartHeightActual( chunkIdx ); @@ -390,11 +419,15 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); @@ -441,7 +474,19 @@ namespace sgns::sgprocessing const float *data = outputUserTensor->host(); const size_t dataSize = outputUserTensor->elementSize() * sizeof( float ); - auto hash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at this branch's chunk-hash site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); std::string combinedHash = std::string( subTaskResultHash.begin(), subTaskResultHash.end() ) + std::string( hash.begin(), hash.end() ); @@ -475,7 +520,19 @@ namespace sgns::sgprocessing const float *data = outputTensor->host(); const size_t dataSize = outputTensor->elementSize() * sizeof( float ); - auto hash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at this branch's chunk-hash site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + auto hash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); chunkhashes.emplace_back( hash.begin(), hash.end() ); std::string combinedHash = std::string( subTaskResultHash.begin(), subTaskResultHash.end() ) + std::string( hash.begin(), hash.end() ); @@ -486,8 +543,29 @@ namespace sgns::sgprocessing } } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + m_progress = 100.0f; + // Output budget check (EXEC-03) + if ( !outputFloats.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = outputFloats.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -504,6 +582,10 @@ namespace sgns::sgprocessing } m_logger->info( "TextureCube processing complete ({} chunks)", totalChunks ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -523,11 +605,15 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); diff --git a/src/processors/processing_processor_mnn_vec2.cpp b/src/processors/processing_processor_mnn_vec2.cpp index 6b4b8e5..28dbaa2 100644 --- a/src/processors/processing_processor_mnn_vec2.cpp +++ b/src/processors/processing_processor_mnn_vec2.cpp @@ -1,8 +1,10 @@ #include "processors/processing_processor_mnn_vec2.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" @@ -184,9 +186,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &vec2Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { (void)parameters; + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -255,8 +259,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Vec2 pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Vec2 pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchVectors ) * 2, 0.0f ); @@ -327,6 +348,22 @@ namespace sgns::sgprocessing m_progress = 100.0f; + m_progress = 100.0f; + + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -343,6 +380,10 @@ namespace sgns::sgprocessing } m_logger->info( "Vec2 processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -350,7 +391,7 @@ namespace sgns::sgprocessing std::vector &model, int length ) { - auto interpreter = std::unique_ptr( + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( model.data(), model.size() ) ); if ( !interpreter ) { @@ -359,17 +400,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_vec3.cpp b/src/processors/processing_processor_mnn_vec3.cpp index ede4016..3de5e93 100644 --- a/src/processors/processing_processor_mnn_vec3.cpp +++ b/src/processors/processing_processor_mnn_vec3.cpp @@ -1,8 +1,10 @@ #include "processors/processing_processor_mnn_vec3.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" @@ -184,9 +186,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &vec3Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { (void)parameters; + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -255,8 +259,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Vec3 pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Vec3 pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchVectors ) * 3, 0.0f ); @@ -325,8 +346,29 @@ namespace sgns::sgprocessing chunkhashes.push_back( subTaskResultHash ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + m_progress = 100.0f; + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -343,6 +385,10 @@ namespace sgns::sgprocessing } m_logger->info( "Vec3 processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -350,7 +396,7 @@ namespace sgns::sgprocessing std::vector &model, int length ) { - auto interpreter = std::unique_ptr( + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( model.data(), model.size() ) ); if ( !interpreter ) { @@ -359,17 +405,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_vec4.cpp b/src/processors/processing_processor_mnn_vec4.cpp index 9d8fa11..a6eddef 100644 --- a/src/processors/processing_processor_mnn_vec4.cpp +++ b/src/processors/processing_processor_mnn_vec4.cpp @@ -1,8 +1,10 @@ #include "processors/processing_processor_mnn_vec4.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include #include #include +#include #include #include "util/sha256.hpp" @@ -184,9 +186,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &vec4Data, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { (void)parameters; + const std::string passId = proc.get_name(); std::vector modelFileBytes; modelFileBytes.assign( modelFile.begin(), modelFile.end() ); @@ -255,8 +259,25 @@ namespace sgns::sgprocessing std::vector stitchedOutput; std::vector stitchedWeights; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Vec4 pass cancelled" } }; + } + for ( int start : starts ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Vec4 pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchVectors ) * 4, 0.0f ); @@ -325,8 +346,29 @@ namespace sgns::sgprocessing chunkhashes.push_back( subTaskResultHash ); } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + m_progress = 100.0f; + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -343,6 +385,10 @@ namespace sgns::sgprocessing } m_logger->info( "Vec4 processing complete" ); + + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -350,7 +396,7 @@ namespace sgns::sgprocessing std::vector &model, int length ) { - auto interpreter = std::unique_ptr( + auto interpreter = std::shared_ptr( MNN::Interpreter::createFromBuffer( model.data(), model.size() ) ); if ( !interpreter ) { @@ -359,17 +405,25 @@ namespace sgns::sgprocessing } MNN::ScheduleConfig config; - config.type = MNN_FORWARD_CPU; + config.type = MNN_FORWARD_VULKAN; config.numThread = 4; config.backendConfig = nullptr; - auto session = interpreter->createSession( config ); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession( config ); + } if ( !session ) { m_logger->error( "Failed to create MNN session" ); return nullptr; } + PushTeardown( [interpreter, session]() { + interpreter->releaseSession( session ); + } ); + auto inputTensor = interpreter->getSessionInput( session, nullptr ); if ( !inputTensor ) { diff --git a/src/processors/processing_processor_mnn_volume.cpp b/src/processors/processing_processor_mnn_volume.cpp index 2c51f45..acac84c 100644 --- a/src/processors/processing_processor_mnn_volume.cpp +++ b/src/processors/processing_processor_mnn_volume.cpp @@ -1,5 +1,7 @@ #include "processors/processing_processor_mnn_volume.hpp" +#include "processingbase/vulkan_init_guard.hpp" #include +#include #include #include #include @@ -10,6 +12,7 @@ #include #include // For SHA256_DIGEST_LENGTH #include "util/sha256.hpp" +#include "util/quantization.hpp" namespace sgns::sgprocessing { @@ -207,8 +210,11 @@ namespace sgns::sgprocessing const sgns::IoDeclaration &proc, std::vector &volumeData, std::vector &modelFile, - const std::vector *parameters ) + const std::vector *parameters, + const ExecutionContext &execCtx ) { + const float scale = sgprocmanagerquant::ResolveQuantScale( parameters ); + const std::string passId = proc.get_name(); std::vector modelFile_bytes; modelFile_bytes.assign(modelFile.begin(), modelFile.end()); @@ -321,6 +327,17 @@ namespace sgns::sgprocessing m_progress = 0.0f; + // LOAD_MODEL stage — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::LOAD_MODEL, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Volume pass cancelled" } }; + } + std::vector shahash( SHA256_DIGEST_LENGTH ); const auto startsX = ComputeWindowStarts( width, patchWidth, strideX ); @@ -340,6 +357,12 @@ namespace sgns::sgprocessing { for ( const int x : startsX ) { + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, ProcessingError{ ProcessingErrorStage::CANCELLED, "Volume pass cancelled" } }; + } + std::vector patch; patch.resize( static_cast( patchWidth ) * patchHeight * patchDepth, 0.0f ); @@ -492,7 +515,19 @@ namespace sgns::sgprocessing } } - shahash = sgprocmanagersha::sha256( data, dataSize ); + // Phase 10 CAPT-02: quantize-then-capture-then-hash at the per-chunk site. + // Never mutate MNN-owned `data` (const float*) in place -- copy first. + std::vector localCopy( data, data + ( dataSize / sizeof( float ) ) ); + sgprocmanagerquant::QuantizeFloatBuffer( localCopy.data(), localCopy.size(), scale ); + if ( execCtx.rawOutputCapture ) + { + const auto *quantizedBytes = reinterpret_cast( localCopy.data() ); + const auto *preQuantizeBytes = reinterpret_cast( data ); + execCtx.rawOutputCapture( std::vector( quantizedBytes, quantizedBytes + dataSize ), + std::vector( preQuantizeBytes, preQuantizeBytes + dataSize ) ); + } + + shahash = sgprocmanagersha::sha256( localCopy.data(), dataSize ); std::string hashString( shahash.begin(), shahash.end() ); chunkhashes.push_back( shahash ); @@ -504,6 +539,13 @@ namespace sgns::sgprocessing } } + // RUN + READ_OUTPUT stages — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::RUN, 75.0f ) ); + execCtx.progressCallback( ProgressEvent::ForMNN( passId, MNNStage::READ_OUTPUT, 100.0f ) ); + } + m_progress = 100.0f; if ( !stitchedOutput.empty() ) @@ -544,6 +586,20 @@ namespace sgns::sgprocessing m_logger->info( "Volume processing complete" ); + // Output budget check (EXEC-03) + if ( !stitchedOutput.empty() && execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = stitchedOutput.size() * sizeof( float ); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return ProcessingResult{ {}, nullptr, {}, + ProcessingError{ ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + + std::to_string( execCtx.maxOutputArtifactBytes ) } }; + } + } + ProcessingResult result; result.hash = subTaskResultHash; @@ -558,6 +614,9 @@ namespace sgns::sgprocessing result.output_buffers->second.push_back( std::move( outputBytes ) ); } + // Tear down all MNN sessions accumulated during processing + RunTeardown(); + return result; } @@ -583,7 +642,11 @@ namespace sgns::sgprocessing m_logger->info( "Using MNN Vulkan backend" ); config.numThread = 4; - auto session = interpreter->createSession(config); + MNN::Session *session = nullptr; + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + session = interpreter->createSession(config); + } if (!session) { m_logger->error( "Failed to create MNN session" ); return std::make_unique(); diff --git a/src/processors/processing_processor_render.cpp b/src/processors/processing_processor_render.cpp new file mode 100644 index 0000000..8cec5df --- /dev/null +++ b/src/processors/processing_processor_render.cpp @@ -0,0 +1,2227 @@ +#include "processors/processing_processor_render.hpp" +#include "processingbase/vulkan_init_guard.hpp" +#include "util/sha256.hpp" +#include "util/quantization.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ + + bool RenderProcessor::IsAcceptable( VkPhysicalDeviceType type ) + { + return type == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU + || type == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; + } + + namespace + { + /// Human-readable VkPhysicalDeviceType name for diagnostic logging -- + /// vk-bootstrap/Vulkan only give the caller the raw enum. + const char *VkPhysicalDeviceTypeName( VkPhysicalDeviceType type ) + { + switch ( type ) + { + case VK_PHYSICAL_DEVICE_TYPE_OTHER: return "OTHER"; + case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "INTEGRATED_GPU"; + case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "DISCRETE_GPU"; + case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "VIRTUAL_GPU"; + case VK_PHYSICAL_DEVICE_TYPE_CPU: return "CPU"; + default: return "UNKNOWN"; + } + } + } // namespace + + VkDeviceSize RenderProcessor::LargestDeviceLocalHeap( VkPhysicalDevice device ) + { + VkPhysicalDeviceMemoryProperties memProps; + vkGetPhysicalDeviceMemoryProperties( device, &memProps ); + VkDeviceSize largest = 0; + for ( uint32_t i = 0; i < memProps.memoryHeapCount; ++i ) + { + if ( memProps.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ) + largest = (std::max)( largest, memProps.memoryHeaps[i].size ); + } + return largest; + } + + bool RenderProcessor::InitializeContext() + { + if ( m_contextInitialized ) + return true; + + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + + if ( m_contextInitialized ) + return true; + + // On macOS MoltenVK is statically linked (libMoltenVK.a), so there is no + // libvulkan.dylib for vk-bootstrap's default dlopen path to find. + // Pass the statically-available vkGetInstanceProcAddr directly. +#if defined(__APPLE__) + vkb::InstanceBuilder instance_builder( vkGetInstanceProcAddr ); +#else + vkb::InstanceBuilder instance_builder; +#endif + auto inst_ret = instance_builder + .set_app_name( "SGProcessingManager RenderProcessor" ) + .set_app_version( 1, 0, 0 ) +#ifdef ENABLE_VULKAN_VALIDATION + .request_validation_layers() // best-effort (D-20, D-21) +#else + .request_validation_layers( false ) +#endif + .build(); + if ( !inst_ret ) + { + m_logger->error( "RenderProcessor: failed to create Vulkan instance: {}", + inst_ret.error().message() ); + return false; + } + auto vkb_instance = inst_ret.value(); + + vkb::PhysicalDeviceSelector selector( vkb_instance ); + // This is a headless/offscreen renderer -- no VkSurfaceKHR/swapchain ever exists + // (CTX-01/D-23). vk-bootstrap's PhysicalDeviceSelector defaults require_present to + // true, which rejects every device with vkb::PhysicalDeviceError::no_surface_provided + // when no surface was ever set. Disable that requirement explicitly. + selector.require_present( false ); + auto devices_ret = selector.select_devices(); + if ( !devices_ret ) + { + m_logger->error( "RenderProcessor: failed to enumerate physical devices: {}", + devices_ret.error().message() ); + vkb::destroy_instance( vkb_instance ); + return false; + } + + auto devices = devices_ret.value(); + + // Diagnostic (D-32 follow-up): log every enumerated device's name/type/vendor + // BEFORE the acceptability filter runs, so environments like WSL (whose Vulkan + // device reports an unexpected type) are debuggable from a plain run, not just + // via a debugger. + for ( const auto &d : devices ) + { + m_logger->info( "RenderProcessor: enumerated device \"{}\" type={} vendorID=0x{:04x} " + "deviceID=0x{:04x} apiVersion={}.{}.{}", + d.properties.deviceName, + VkPhysicalDeviceTypeName( d.properties.deviceType ), + d.properties.vendorID, + d.properties.deviceID, + VK_API_VERSION_MAJOR( d.properties.apiVersion ), + VK_API_VERSION_MINOR( d.properties.apiVersion ), + VK_API_VERSION_PATCH( d.properties.apiVersion ) ); + } + + devices.erase( + std::remove_if( devices.begin(), devices.end(), + []( const vkb::PhysicalDevice &d ) { + return !IsAcceptable( d.properties.deviceType ); + } ), + devices.end() ); + + if ( devices.empty() ) + { + m_logger->error( "RenderProcessor: no acceptable physical device found " + "(none with device type DISCRETE_GPU or INTEGRATED_GPU)" ); + vkb::destroy_instance( vkb_instance ); + return false; + } + + std::sort( devices.begin(), devices.end(), + []( const vkb::PhysicalDevice &a, const vkb::PhysicalDevice &b ) { + int rank_a = ( a.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ) ? 2 : 1; + int rank_b = ( b.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ) ? 2 : 1; + if ( rank_a != rank_b ) + return rank_a > rank_b; + return LargestDeviceLocalHeap( a.physical_device ) + > LargestDeviceLocalHeap( b.physical_device ); + } ); + + vkb::DeviceBuilder device_builder( devices[0] ); + auto dev_ret = device_builder.build(); + if ( !dev_ret ) + { + m_logger->error( "RenderProcessor: failed to create Vulkan device: {}", + dev_ret.error().message() ); + vkb::destroy_instance( vkb_instance ); + return false; + } + + auto vkb_device = dev_ret.value(); + auto queue_ret = vkb_device.get_queue( vkb::QueueType::graphics ); + if ( !queue_ret ) + { + m_logger->error( "RenderProcessor: failed to get graphics queue: {}", + queue_ret.error().message() ); + vkb::destroy_device( vkb_device ); + vkb::destroy_instance( vkb_instance ); + return false; + } + + // Reused (never re-queried) by RecordAndSubmit()'s VkCommandPool creation -- + // the same graphics queue family InitializeContext() already selected m_queue + // from, not a fresh PhysicalDeviceSelector-style re-selection. + auto queue_family_ret = vkb_device.get_queue_index( vkb::QueueType::graphics ); + if ( !queue_family_ret ) + { + m_logger->error( "RenderProcessor: failed to get graphics queue family index: {}", + queue_family_ret.error().message() ); + vkb::destroy_device( vkb_device ); + vkb::destroy_instance( vkb_instance ); + return false; + } + + m_instance = vkb_instance.instance; + m_physicalDevice = vkb_device.physical_device; + m_device = vkb_device.device; + m_queue = queue_ret.value(); + m_queueFamilyIndex = queue_family_ret.value(); + m_contextInitialized = true; + + return true; + } + + ProcessingResult RenderProcessor::MakeError( sgns::sgprocessing::ProcessingErrorStage stage, + const std::string &message ) + { + ProcessingResult result; + result.hash = std::vector( 32, 0 ); + ProcessingError error; + error.stage = stage; + error.message = message; + result.error = error; + return result; + } + + namespace + { + /// Bounds-checked little-endian primitive readers over a raw byte + /// buffer. Every read advances `offset`; callers must check the + /// return value before trusting `out`. Never reads past `size`. + bool ReadU32( const char *data, size_t size, size_t &offset, uint32_t &out ) + { + if ( offset + sizeof( uint32_t ) > size ) + { + return false; + } + std::memcpy( &out, data + offset, sizeof( uint32_t ) ); + offset += sizeof( uint32_t ); + return true; + } + + bool ReadU8( const char *data, size_t size, size_t &offset, uint8_t &out ) + { + if ( offset + sizeof( uint8_t ) > size ) + { + return false; + } + out = static_cast( data[offset] ); + offset += sizeof( uint8_t ); + return true; + } + + bool ReadF32( const char *data, size_t size, size_t &offset, float &out ) + { + if ( offset + sizeof( float ) > size ) + { + return false; + } + std::memcpy( &out, data + offset, sizeof( float ) ); + offset += sizeof( float ); + return true; + } + + bool ReadBytes( const char *data, size_t size, size_t &offset, size_t count, const char *&outPtr ) + { + if ( offset + count > size ) + { + return false; + } + outPtr = data + offset; + offset += count; + return true; + } + + bool ReadString( const char *data, size_t size, size_t &offset, std::string &out ) + { + uint32_t len = 0; + if ( !ReadU32( data, size, offset, len ) ) + { + return false; + } + if ( len == 0 ) + { + out.clear(); + return true; + } + const char *bytes = nullptr; + if ( !ReadBytes( data, size, offset, len, bytes ) ) + { + return false; + } + out.assign( bytes, len ); + return true; + } + } + + bool RenderProcessor::ParseCompiledStages( const std::vector &modelFile, + std::vector &outStages, + ProcessingResult &errorOut ) + { + outStages.clear(); + const char *data = modelFile.data(); + const size_t size = modelFile.size(); + size_t offset = 0; + + uint32_t stageCount = 0; + if ( !ReadU32( data, size, offset, stageCount ) ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ParseCompiledStages: truncated buffer reading stage_count" ); + return false; + } + + outStages.reserve( stageCount ); + for ( uint32_t i = 0; i < stageCount; ++i ) + { + ParsedStage stage; + + uint32_t stageTag = 0; + if ( !ReadU32( data, size, offset, stageTag ) ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ParseCompiledStages: truncated buffer reading stage_tag" ); + return false; + } + stage.stage = static_cast( stageTag ); + + uint32_t entryPointLen = 0; + if ( !ReadU32( data, size, offset, entryPointLen ) ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ParseCompiledStages: truncated buffer reading entry_point_len" ); + return false; + } + if ( entryPointLen > 0 ) + { + const char *bytes = nullptr; + if ( !ReadBytes( data, size, offset, entryPointLen, bytes ) ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ParseCompiledStages: truncated buffer reading entry_point bytes" ); + return false; + } + stage.entry_point.assign( bytes, entryPointLen ); + } + + uint32_t wordCount = 0; + if ( !ReadU32( data, size, offset, wordCount ) ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ParseCompiledStages: truncated buffer reading word_count" ); + return false; + } + if ( wordCount > 0 ) + { + size_t byteCount = static_cast( wordCount ) * sizeof( uint32_t ); + const char *bytes = nullptr; + if ( !ReadBytes( data, size, offset, byteCount, bytes ) ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ParseCompiledStages: truncated buffer reading spirv_words" ); + return false; + } + stage.spirv.resize( wordCount ); + std::memcpy( stage.spirv.data(), bytes, byteCount ); + } + + outStages.push_back( std::move( stage ) ); + } + + return true; + } + + bool RenderProcessor::ParseRenderPassConfig( + const std::vector &imageData, + sgns::RenderTarget &outTarget, + boost::optional &outPipelineState, + std::vector &outVertexLayout, + boost::optional> &outUniforms, + std::vector &outVertexBytes, + bool &outHasIndex, + sgns::IndexType &outIndexType, + std::vector &outIndexBytes, + uint32_t &outDataTransformCount, + ProcessingResult &errorOut ) + // Function-try-block: several generated setters below (set_width/set_height/ + // set_clear_depth/set_offset, etc.) enforce schema-level constraints and throw + // on violation. A malformed/truncated wire-format buffer must never crash the + // process -- convert any such exception into a structured RESOURCE_RESOLUTION + // error instead, per this task's "no crash/UB on out-of-bounds/malformed data" + // requirement. + try + { + outPipelineState = boost::none; + outVertexLayout.clear(); + outUniforms = boost::none; + outVertexBytes.clear(); + outHasIndex = false; + outIndexBytes.clear(); + outDataTransformCount = 0; + + const char *data = imageData.data(); + const size_t size = imageData.size(); + size_t offset = 0; + + auto fail = [&]( const std::string &message ) -> bool + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, message ); + return false; + }; + + uint32_t width = 0, height = 0, colorFormatTag = 0, depthFormatTag = 0; + if ( !ReadU32( data, size, offset, width ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading width" ); + } + if ( !ReadU32( data, size, offset, height ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading height" ); + } + if ( !ReadU32( data, size, offset, colorFormatTag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading color_format_tag" ); + } + if ( !ReadU32( data, size, offset, depthFormatTag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading depth_format_tag" ); + } + + outTarget.set_width( static_cast( width ) ); + outTarget.set_height( static_cast( height ) ); + outTarget.set_color_format( static_cast( colorFormatTag ) ); + outTarget.set_depth_format( static_cast( depthFormatTag ) ); + + std::vector clearColor( 4, 0.0 ); + for ( size_t i = 0; i < 4; ++i ) + { + float v = 0.0f; + if ( !ReadF32( data, size, offset, v ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading clear_color" ); + } + clearColor[i] = static_cast( v ); + } + outTarget.set_clear_color( clearColor ); + + float clearDepth = 0.0f; + if ( !ReadF32( data, size, offset, clearDepth ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading clear_depth" ); + } + outTarget.set_clear_depth( static_cast( clearDepth ) ); + + uint8_t hasPipelineState = 0; + if ( !ReadU8( data, size, offset, hasPipelineState ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading has_pipeline_state" ); + } + if ( hasPipelineState ) + { + sgns::PipelineState ps; + + uint8_t hasTopology = 0; + if ( !ReadU8( data, size, offset, hasTopology ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading has_topology" ); + } + if ( hasTopology ) + { + uint32_t tag = 0; + if ( !ReadU32( data, size, offset, tag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading topology_tag" ); + } + ps.set_topology( static_cast( tag ) ); + } + + uint8_t hasCullMode = 0; + if ( !ReadU8( data, size, offset, hasCullMode ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading has_cull_mode" ); + } + if ( hasCullMode ) + { + uint32_t tag = 0; + if ( !ReadU32( data, size, offset, tag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading cull_mode_tag" ); + } + ps.set_cull_mode( static_cast( tag ) ); + } + + uint8_t hasFrontFace = 0; + if ( !ReadU8( data, size, offset, hasFrontFace ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading has_front_face" ); + } + if ( hasFrontFace ) + { + uint32_t tag = 0; + if ( !ReadU32( data, size, offset, tag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading front_face_tag" ); + } + ps.set_front_face( static_cast( tag ) ); + } + + uint8_t hasDepthTest = 0; + if ( !ReadU8( data, size, offset, hasDepthTest ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading has_depth_test" ); + } + if ( hasDepthTest ) + { + uint32_t tag = 0; + if ( !ReadU32( data, size, offset, tag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading depth_test_tag" ); + } + ps.set_depth_test( static_cast( tag ) ); + } + + outPipelineState = ps; + } + + uint32_t vertexLayoutCount = 0; + if ( !ReadU32( data, size, offset, vertexLayoutCount ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading vertex_layout_count" ); + } + outVertexLayout.reserve( vertexLayoutCount ); + for ( uint32_t i = 0; i < vertexLayoutCount; ++i ) + { + std::string name; + if ( !ReadString( data, size, offset, name ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading vertex_layout name" ); + } + uint32_t formatTag = 0; + if ( !ReadU32( data, size, offset, formatTag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading vertex_layout format_tag" ); + } + uint32_t entryOffset = 0; + if ( !ReadU32( data, size, offset, entryOffset ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading vertex_layout offset" ); + } + + sgns::VertexLayoutEntry entry; + entry.set_name( name ); + entry.set_format( static_cast( formatTag ) ); + entry.set_offset( static_cast( entryOffset ) ); + outVertexLayout.push_back( std::move( entry ) ); + } + + uint8_t hasUniforms = 0; + if ( !ReadU8( data, size, offset, hasUniforms ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading has_uniforms" ); + } + if ( hasUniforms ) + { + uint32_t uniformCount = 0; + if ( !ReadU32( data, size, offset, uniformCount ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading uniform_count" ); + } + + std::map uniformMap; + for ( uint32_t i = 0; i < uniformCount; ++i ) + { + std::string name; + if ( !ReadString( data, size, offset, name ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading uniform name" ); + } + + sgns::RenderShaderUniform uniform; + + uint8_t hasSource = 0; + if ( !ReadU8( data, size, offset, hasSource ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading uniform has_source" ); + } + if ( hasSource ) + { + std::string source; + if ( !ReadString( data, size, offset, source ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading uniform source" ); + } + uniform.set_source( source ); + } + + uint8_t hasType = 0; + if ( !ReadU8( data, size, offset, hasType ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading uniform has_type" ); + } + if ( hasType ) + { + uint32_t typeTag = 0; + if ( !ReadU32( data, size, offset, typeTag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading uniform type_tag" ); + } + uniform.set_type( static_cast( typeTag ) ); + } + + std::string valueJson; + if ( !ReadString( data, size, offset, valueJson ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading uniform value" ); + } + if ( !valueJson.empty() ) + { + try + { + uniform.set_value( nlohmann::json::parse( valueJson ) ); + } + catch ( const std::exception &e ) + { + return fail( std::string( "ParseRenderPassConfig: malformed uniform value JSON: " ) + + e.what() ); + } + } + + uniformMap[name] = std::move( uniform ); + } + + outUniforms = std::move( uniformMap ); + } + + uint32_t vertexLen = 0; + if ( !ReadU32( data, size, offset, vertexLen ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading vertex_len" ); + } + if ( vertexLen > 0 ) + { + const char *bytes = nullptr; + if ( !ReadBytes( data, size, offset, vertexLen, bytes ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading vertex bytes" ); + } + outVertexBytes.assign( bytes, bytes + vertexLen ); + } + + uint8_t hasIndex = 0; + if ( !ReadU8( data, size, offset, hasIndex ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading has_index" ); + } + if ( hasIndex ) + { + uint32_t indexTypeTag = 0; + if ( !ReadU32( data, size, offset, indexTypeTag ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading index_type_tag" ); + } + uint32_t indexLen = 0; + if ( !ReadU32( data, size, offset, indexLen ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading index_len" ); + } + if ( indexLen > 0 ) + { + const char *bytes = nullptr; + if ( !ReadBytes( data, size, offset, indexLen, bytes ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading index bytes" ); + } + outIndexBytes.assign( bytes, bytes + indexLen ); + } + outHasIndex = true; + outIndexType = static_cast( indexTypeTag ); + } + else + { + outHasIndex = false; + } + + uint32_t dataTransformCount = 0; + if ( !ReadU32( data, size, offset, dataTransformCount ) ) + { + return fail( "ParseRenderPassConfig: truncated buffer reading data_transform_count" ); + } + outDataTransformCount = dataTransformCount; + + return true; + } + catch ( const std::exception &e ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + std::string( "ParseRenderPassConfig: exception while parsing: " ) + e.what() ); + return false; + } + + namespace + { + /// Appends `value` to `bytes`, then pads `bytes` up to the next + /// 16-byte-aligned boundary (this plan's std430-avoidance strategy -- + /// see 03-03-PLAN.md's objective). Every uniform's packed region gets + /// its own 16-byte-aligned slot regardless of its natural size. + void AppendPadded16( std::vector &bytes, const uint8_t *data, size_t size ) + { + bytes.insert( bytes.end(), data, data + size ); + size_t remainder = bytes.size() % 16; + if ( remainder != 0 ) + { + bytes.resize( bytes.size() + ( 16 - remainder ), 0 ); + } + } + + /// Converts a resolved nlohmann::json uniform value into raw bytes + /// according to its declared DataType. Returns false (never + /// crashes/UB) for a DataType this phase does not support as a + /// uniform (STRING/TENSOR/TEXTURE*/BUFFER). + bool PackUniformValue( sgns::DataType dataType, const nlohmann::json &value, std::vector &out ) + { + auto appendFloat = [&out]( double v ) + { + float f = static_cast( v ); + const uint8_t *bytes = reinterpret_cast( &f ); + out.insert( out.end(), bytes, bytes + sizeof( float ) ); + }; + + // Reads up to `count` numeric components from a JSON array (missing/ + // absent entries default to 0.0) -- never throws on a short/malformed + // array; a value that isn't an array at all yields an all-zero vector. + auto readVec = []( const nlohmann::json &v, size_t componentCount ) -> std::vector + { + std::vector result( componentCount, 0.0 ); + if ( v.is_array() ) + { + for ( size_t i = 0; i < componentCount && i < v.size(); ++i ) + { + if ( v[i].is_number() ) + { + result[i] = v[i].get(); + } + } + } + return result; + }; + + try + { + switch ( dataType ) + { + case sgns::DataType::FLOAT: + { + appendFloat( value.is_number() ? value.get() : 0.0 ); + return true; + } + case sgns::DataType::INT: + { + int32_t i = value.is_number() ? static_cast( value.get() ) : 0; + const uint8_t *bytes = reinterpret_cast( &i ); + out.insert( out.end(), bytes, bytes + sizeof( int32_t ) ); + return true; + } + case sgns::DataType::BOOL: + { + int32_t b = ( value.is_boolean() && value.get() ) ? 1 : 0; + const uint8_t *bytes = reinterpret_cast( &b ); + out.insert( out.end(), bytes, bytes + sizeof( int32_t ) ); + return true; + } + case sgns::DataType::VEC2: + { + for ( double d : readVec( value, 2 ) ) + { + appendFloat( d ); + } + return true; + } + case sgns::DataType::VEC3: + { + for ( double d : readVec( value, 3 ) ) + { + appendFloat( d ); + } + return true; + } + case sgns::DataType::VEC4: + { + for ( double d : readVec( value, 4 ) ) + { + appendFloat( d ); + } + return true; + } + case sgns::DataType::MAT2: + { + for ( double d : readVec( value, 4 ) ) + { + appendFloat( d ); + } + return true; + } + case sgns::DataType::MAT3: + { + for ( double d : readVec( value, 9 ) ) + { + appendFloat( d ); + } + return true; + } + case sgns::DataType::MAT4: + { + for ( double d : readVec( value, 16 ) ) + { + appendFloat( d ); + } + return true; + } + case sgns::DataType::STRING: + case sgns::DataType::TENSOR: + case sgns::DataType::TEXTURE1_D: + case sgns::DataType::TEXTURE2_D: + case sgns::DataType::TEXTURE3_D: + case sgns::DataType::TEXTURE_CUBE: + case sgns::DataType::BUFFER: + default: + return false; + } + } + catch ( const std::exception & ) + { + return false; + } + } + } + + bool RenderProcessor::ResolveUniforms( + const boost::optional> &uniforms, + const std::vector *parameters, + ResolvedUniforms &outResolved, + ProcessingResult &errorOut ) + { + outResolved.packedBytes.clear(); + outResolved.pushConstant = true; + + if ( !uniforms ) + { + return true; + } + + // std::map's natural key-sorted iteration order -- deterministic, + // satisfies DETV-01, matches SerializeRenderPassConfig()'s own + // iteration order (03-02-SUMMARY.md). + for ( const auto &entry : uniforms.value() ) + { + const std::string &name = entry.first; + const sgns::RenderShaderUniform &uniform = entry.second; + + nlohmann::json resolvedValue; + + if ( uniform.get_source() ) + { + const std::string &source = uniform.get_source().value(); + static const std::string kParameterPrefix = "parameter:"; + if ( source.rfind( kParameterPrefix, 0 ) != 0 ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ResolveUniforms: unsupported uniform source prefix for '" + name + "'" ); + return false; + } + std::string paramName = source.substr( kParameterPrefix.size() ); + + const sgns::Parameter *found = nullptr; + if ( parameters ) + { + for ( const auto ¶m : *parameters ) + { + if ( param.get_name() == paramName ) + { + found = ¶m; + break; + } + } + } + if ( !found ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ResolveUniforms: unresolvable parameter '" + paramName + + "' for uniform '" + name + "'" ); + return false; + } + resolvedValue = found->get_parameter_default(); + } + else + { + resolvedValue = uniform.get_value(); + } + + if ( !uniform.get_type() ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ResolveUniforms: uniform '" + name + "' has no declared DataType" ); + return false; + } + + std::vector packed; + if ( !PackUniformValue( uniform.get_type().value(), resolvedValue, packed ) ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "ResolveUniforms: unsupported DataType for uniform '" + name + "'" ); + return false; + } + + AppendPadded16( outResolved.packedBytes, packed.data(), packed.size() ); + } + + outResolved.pushConstant = ( outResolved.packedBytes.size() <= 128 ); + + return true; + } + + bool RenderProcessor::CheckFormatSupport( VkFormat format, + VkFormatFeatureFlagBits requiredFeature, + ProcessingResult &errorOut ) + { + VkFormatProperties props{}; + vkGetPhysicalDeviceFormatProperties( m_physicalDevice, format, &props ); + + if ( !( props.optimalTilingFeatures & requiredFeature ) ) + { + errorOut = MakeError( ProcessingErrorStage::FORMAT_UNSUPPORTED, + "CheckFormatSupport: VkFormat " + std::to_string( static_cast( format ) ) + + " does not support required feature " + + std::to_string( static_cast( requiredFeature ) ) + + " for optimal tiling" ); + return false; + } + + return true; + } + + namespace + { + /// Linear scan over VkPhysicalDeviceMemoryProperties::memoryTypes for + /// an index whose bit is set in `typeBits` and whose propertyFlags + /// contain all of `properties` -- mirrors LargestDeviceLocalHeap's + /// existing enumeration style. + bool FindMemoryTypeIndex( const VkPhysicalDeviceMemoryProperties &memProps, + uint32_t typeBits, + VkMemoryPropertyFlags properties, + uint32_t &outIndex ) + { + for ( uint32_t i = 0; i < memProps.memoryTypeCount; ++i ) + { + if ( ( typeBits & ( 1u << i ) ) && + ( memProps.memoryTypes[i].propertyFlags & properties ) == properties ) + { + outIndex = i; + return true; + } + } + return false; + } + } + + bool RenderProcessor::CreateBufferDedicated( VkDeviceSize size, + VkBufferUsageFlags usage, + VkMemoryPropertyFlags properties, + VkBuffer &outBuffer, + VkDeviceMemory &outMemory, + ProcessingResult &errorOut ) + { + VkBufferCreateInfo bufferInfo{}; + bufferInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + bufferInfo.size = size; + bufferInfo.usage = usage; + bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VkBuffer buffer = VK_NULL_HANDLE; + VkResult result = vkCreateBuffer( m_device, &bufferInfo, nullptr, &buffer ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::BUFFER_ALLOCATION, + "vkCreateBuffer failed: VkResult=" + std::to_string( result ) ); + return false; + } + + VkMemoryRequirements memRequirements{}; + vkGetBufferMemoryRequirements( m_device, buffer, &memRequirements ); + + VkPhysicalDeviceMemoryProperties memProps{}; + vkGetPhysicalDeviceMemoryProperties( m_physicalDevice, &memProps ); + + uint32_t memTypeIndex = 0; + if ( !FindMemoryTypeIndex( memProps, memRequirements.memoryTypeBits, properties, memTypeIndex ) ) + { + vkDestroyBuffer( m_device, buffer, nullptr ); + errorOut = MakeError( ProcessingErrorStage::BUFFER_ALLOCATION, + "CreateBufferDedicated: no suitable memory type found" ); + return false; + } + + VkMemoryAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocInfo.allocationSize = memRequirements.size; + allocInfo.memoryTypeIndex = memTypeIndex; + + VkDeviceMemory memory = VK_NULL_HANDLE; + result = vkAllocateMemory( m_device, &allocInfo, nullptr, &memory ); + if ( result != VK_SUCCESS ) + { + vkDestroyBuffer( m_device, buffer, nullptr ); + errorOut = MakeError( ProcessingErrorStage::BUFFER_ALLOCATION, + "CreateBufferDedicated: dedicated memory allocation failed: VkResult=" + + std::to_string( result ) ); + return false; + } + + result = vkBindBufferMemory( m_device, buffer, memory, 0 ); + if ( result != VK_SUCCESS ) + { + vkFreeMemory( m_device, memory, nullptr ); + vkDestroyBuffer( m_device, buffer, nullptr ); + errorOut = MakeError( ProcessingErrorStage::BUFFER_ALLOCATION, + "vkBindBufferMemory failed: VkResult=" + std::to_string( result ) ); + return false; + } + + outBuffer = buffer; + outMemory = memory; + + VkDevice device = m_device; + PushTeardown( [device, buffer, memory]() { + vkDestroyBuffer( device, buffer, nullptr ); + vkFreeMemory( device, memory, nullptr ); + } ); + + return true; + } + + bool RenderProcessor::CreateImageDedicated( const VkImageCreateInfo &imageInfo, + VkMemoryPropertyFlags properties, + VkImage &outImage, + VkDeviceMemory &outMemory, + ProcessingResult &errorOut ) + { + VkImage image = VK_NULL_HANDLE; + VkResult result = vkCreateImage( m_device, &imageInfo, nullptr, &image ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "vkCreateImage failed: VkResult=" + std::to_string( result ) ); + return false; + } + + VkMemoryRequirements memRequirements{}; + vkGetImageMemoryRequirements( m_device, image, &memRequirements ); + + VkPhysicalDeviceMemoryProperties memProps{}; + vkGetPhysicalDeviceMemoryProperties( m_physicalDevice, &memProps ); + + uint32_t memTypeIndex = 0; + if ( !FindMemoryTypeIndex( memProps, memRequirements.memoryTypeBits, properties, memTypeIndex ) ) + { + vkDestroyImage( m_device, image, nullptr ); + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "CreateImageDedicated: no suitable memory type found" ); + return false; + } + + VkMemoryAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocInfo.allocationSize = memRequirements.size; + allocInfo.memoryTypeIndex = memTypeIndex; + + VkDeviceMemory memory = VK_NULL_HANDLE; + result = vkAllocateMemory( m_device, &allocInfo, nullptr, &memory ); + if ( result != VK_SUCCESS ) + { + vkDestroyImage( m_device, image, nullptr ); + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "CreateImageDedicated: dedicated memory allocation failed: VkResult=" + + std::to_string( result ) ); + return false; + } + + result = vkBindImageMemory( m_device, image, memory, 0 ); + if ( result != VK_SUCCESS ) + { + vkFreeMemory( m_device, memory, nullptr ); + vkDestroyImage( m_device, image, nullptr ); + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "vkBindImageMemory failed: VkResult=" + std::to_string( result ) ); + return false; + } + + outImage = image; + outMemory = memory; + + VkDevice device = m_device; + PushTeardown( [device, image, memory]() { + vkDestroyImage( device, image, nullptr ); + vkFreeMemory( device, memory, nullptr ); + } ); + + return true; + } + + VkFormat RenderProcessor::ToVkFormat( sgns::ColorFormat fmt ) + { + switch ( fmt ) + { + case sgns::ColorFormat::RGBA8: + return VK_FORMAT_R8G8B8A8_UNORM; + case sgns::ColorFormat::RGB8: + return VK_FORMAT_R8G8B8_UNORM; + } + return VK_FORMAT_R8G8B8A8_UNORM; + } + + VkFormat RenderProcessor::ToVkFormat( sgns::DepthFormat fmt ) + { + switch ( fmt ) + { + case sgns::DepthFormat::D32_SFLOAT: + return VK_FORMAT_D32_SFLOAT; + case sgns::DepthFormat::D24_UNORM_S8_UINT: + return VK_FORMAT_D24_UNORM_S8_UINT; + } + return VK_FORMAT_D32_SFLOAT; + } + + uint32_t RenderProcessor::ColorFormatByteSize( sgns::ColorFormat fmt ) + { + switch ( fmt ) + { + case sgns::ColorFormat::RGBA8: + return 4; + case sgns::ColorFormat::RGB8: + return 3; + } + return 4; + } + + bool RenderProcessor::BuildRenderPass( const sgns::RenderTarget &target, ProcessingResult &errorOut ) + { + if ( target.get_width() < 1 || target.get_width() > static_cast( kMaxRenderDimension ) || + target.get_height() < 1 || target.get_height() > static_cast( kMaxRenderDimension ) ) + { + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "BuildRenderPass: render_target width/height out of bounds" ); + return false; + } + + VkFormat colorFormat = ToVkFormat( target.get_color_format() ); + VkFormat depthFormat = ToVkFormat( target.get_depth_format() ); + + if ( !CheckFormatSupport( colorFormat, VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT, errorOut ) ) + { + return false; + } + if ( !CheckFormatSupport( depthFormat, VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT, errorOut ) ) + { + return false; + } + + VkAttachmentDescription colorAttachment{}; + colorAttachment.format = colorFormat; + colorAttachment.samples = VK_SAMPLE_COUNT_1_BIT; + colorAttachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + colorAttachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + colorAttachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + colorAttachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + colorAttachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + colorAttachment.finalLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + + VkAttachmentDescription depthAttachment{}; + depthAttachment.format = depthFormat; + depthAttachment.samples = VK_SAMPLE_COUNT_1_BIT; + depthAttachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + depthAttachment.storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + depthAttachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + depthAttachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + depthAttachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + depthAttachment.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + + VkAttachmentDescription attachments[2] = { colorAttachment, depthAttachment }; + + VkAttachmentReference colorRef{}; + colorRef.attachment = 0; + colorRef.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + + VkAttachmentReference depthRef{}; + depthRef.attachment = 1; + depthRef.layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + + VkSubpassDescription subpass{}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &colorRef; + subpass.pDepthStencilAttachment = &depthRef; + + VkRenderPassCreateInfo renderPassInfo{}; + renderPassInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO; + renderPassInfo.attachmentCount = 2; + renderPassInfo.pAttachments = attachments; + renderPassInfo.subpassCount = 1; + renderPassInfo.pSubpasses = &subpass; + + VkRenderPass renderPass = VK_NULL_HANDLE; + VkResult result = vkCreateRenderPass( m_device, &renderPassInfo, nullptr, &renderPass ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::RENDER_PASS_CREATION, + "vkCreateRenderPass failed: VkResult=" + std::to_string( result ) ); + return false; + } + + m_renderPass = renderPass; + m_renderWidth = static_cast( target.get_width() ); + m_renderHeight = static_cast( target.get_height() ); + + VkDevice device = m_device; + PushTeardown( [device, renderPass]() { vkDestroyRenderPass( device, renderPass, nullptr ); } ); + + return true; + } + + bool RenderProcessor::BuildFramebuffer( const sgns::RenderTarget &target, ProcessingResult &errorOut ) + { + uint32_t width = static_cast( target.get_width() ); + uint32_t height = static_cast( target.get_height() ); + + VkFormat colorFormat = ToVkFormat( target.get_color_format() ); + VkFormat depthFormat = ToVkFormat( target.get_depth_format() ); + + VkImageCreateInfo colorImageInfo{}; + colorImageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + colorImageInfo.imageType = VK_IMAGE_TYPE_2D; + colorImageInfo.format = colorFormat; + colorImageInfo.extent = { width, height, 1 }; + colorImageInfo.mipLevels = 1; + colorImageInfo.arrayLayers = 1; + colorImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + colorImageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + colorImageInfo.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + colorImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + colorImageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + if ( !CreateImageDedicated( colorImageInfo, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, m_colorImage, m_colorMemory, + errorOut ) ) + { + return false; + } + + VkImageCreateInfo depthImageInfo{}; + depthImageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + depthImageInfo.imageType = VK_IMAGE_TYPE_2D; + depthImageInfo.format = depthFormat; + depthImageInfo.extent = { width, height, 1 }; + depthImageInfo.mipLevels = 1; + depthImageInfo.arrayLayers = 1; + depthImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + depthImageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + depthImageInfo.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; + depthImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + depthImageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + if ( !CreateImageDedicated( depthImageInfo, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, m_depthImage, m_depthMemory, + errorOut ) ) + { + return false; + } + + VkImageViewCreateInfo colorViewInfo{}; + colorViewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + colorViewInfo.image = m_colorImage; + colorViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + colorViewInfo.format = colorFormat; + colorViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + colorViewInfo.subresourceRange.baseMipLevel = 0; + colorViewInfo.subresourceRange.levelCount = 1; + colorViewInfo.subresourceRange.baseArrayLayer = 0; + colorViewInfo.subresourceRange.layerCount = 1; + + VkResult result = vkCreateImageView( m_device, &colorViewInfo, nullptr, &m_colorView ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "vkCreateImageView (color) failed: VkResult=" + std::to_string( result ) ); + return false; + } + { + VkDevice device = m_device; + VkImageView view = m_colorView; + PushTeardown( [device, view]() { vkDestroyImageView( device, view, nullptr ); } ); + } + + // D24_UNORM_S8_UINT has a stencil component the schema never exposes/uses; + // the image view's aspectMask must still include it when present, per + // Vulkan's depth-stencil-attachment image-view rules. + VkImageAspectFlags depthAspect = VK_IMAGE_ASPECT_DEPTH_BIT; + if ( target.get_depth_format() == sgns::DepthFormat::D24_UNORM_S8_UINT ) + { + depthAspect |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + + VkImageViewCreateInfo depthViewInfo{}; + depthViewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + depthViewInfo.image = m_depthImage; + depthViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + depthViewInfo.format = depthFormat; + depthViewInfo.subresourceRange.aspectMask = depthAspect; + depthViewInfo.subresourceRange.baseMipLevel = 0; + depthViewInfo.subresourceRange.levelCount = 1; + depthViewInfo.subresourceRange.baseArrayLayer = 0; + depthViewInfo.subresourceRange.layerCount = 1; + + result = vkCreateImageView( m_device, &depthViewInfo, nullptr, &m_depthView ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "vkCreateImageView (depth) failed: VkResult=" + std::to_string( result ) ); + return false; + } + { + VkDevice device = m_device; + VkImageView view = m_depthView; + PushTeardown( [device, view]() { vkDestroyImageView( device, view, nullptr ); } ); + } + + VkImageView attachments[2] = { m_colorView, m_depthView }; + + VkFramebufferCreateInfo framebufferInfo{}; + framebufferInfo.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO; + framebufferInfo.renderPass = m_renderPass; + framebufferInfo.attachmentCount = 2; + framebufferInfo.pAttachments = attachments; + framebufferInfo.width = width; + framebufferInfo.height = height; + framebufferInfo.layers = 1; + + VkFramebuffer framebuffer = VK_NULL_HANDLE; + result = vkCreateFramebuffer( m_device, &framebufferInfo, nullptr, &framebuffer ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::IMAGE_ALLOCATION, + "vkCreateFramebuffer failed: VkResult=" + std::to_string( result ) ); + return false; + } + + m_framebuffer = framebuffer; + + VkDevice device = m_device; + PushTeardown( [device, framebuffer]() { vkDestroyFramebuffer( device, framebuffer, nullptr ); } ); + + return true; + } + + VkFormat RenderProcessor::ToVkFormat( sgns::VertexLayoutFormat fmt ) + { + switch ( fmt ) + { + case sgns::VertexLayoutFormat::FLOAT32: + return VK_FORMAT_R32_SFLOAT; + case sgns::VertexLayoutFormat::FLOAT16: + return VK_FORMAT_R16_SFLOAT; + case sgns::VertexLayoutFormat::INT32: + return VK_FORMAT_R32_SINT; + } + return VK_FORMAT_R32_SFLOAT; + } + + VkPrimitiveTopology RenderProcessor::ToVkTopology( sgns::Topology t ) + { + switch ( t ) + { + case sgns::Topology::TRIANGLE_LIST: + return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + case sgns::Topology::LINE_LIST: + return VK_PRIMITIVE_TOPOLOGY_LINE_LIST; + case sgns::Topology::POINT_LIST: + return VK_PRIMITIVE_TOPOLOGY_POINT_LIST; + } + return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + } + + VkCullModeFlags RenderProcessor::ToVkCullMode( sgns::CullMode c ) + { + switch ( c ) + { + case sgns::CullMode::NONE: + return VK_CULL_MODE_NONE; + case sgns::CullMode::FRONT: + return VK_CULL_MODE_FRONT_BIT; + case sgns::CullMode::BACK: + return VK_CULL_MODE_BACK_BIT; + } + return VK_CULL_MODE_BACK_BIT; + } + + VkFrontFace RenderProcessor::ToVkFrontFace( sgns::FrontFace f ) + { + switch ( f ) + { + case sgns::FrontFace::CCW: + return VK_FRONT_FACE_COUNTER_CLOCKWISE; + case sgns::FrontFace::CW: + return VK_FRONT_FACE_CLOCKWISE; + } + return VK_FRONT_FACE_COUNTER_CLOCKWISE; + } + + VkBool32 RenderProcessor::ToVkBool( sgns::DepthTest d ) + { + return ( d == sgns::DepthTest::ENABLED ) ? VK_TRUE : VK_FALSE; + } + + uint32_t RenderProcessor::VertexFormatByteSize( sgns::VertexLayoutFormat f ) + { + switch ( f ) + { + case sgns::VertexLayoutFormat::FLOAT32: + return 4; + case sgns::VertexLayoutFormat::INT32: + return 4; + case sgns::VertexLayoutFormat::FLOAT16: + return 2; + } + return 4; + } + + bool RenderProcessor::BuildPipeline( const std::vector &stages, + const std::vector &vertexLayout, + const boost::optional &pipelineState, + const ResolvedUniforms &uniforms, + ProcessingResult &errorOut ) + { + std::vector shaderStages; + shaderStages.reserve( stages.size() ); + + for ( const auto &s : stages ) + { + VkShaderModuleCreateInfo moduleInfo{}; + moduleInfo.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + moduleInfo.codeSize = s.spirv.size() * sizeof( uint32_t ); + moduleInfo.pCode = s.spirv.data(); + + VkShaderModule module = VK_NULL_HANDLE; + VkResult result = vkCreateShaderModule( m_device, &moduleInfo, nullptr, &module ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::SHADER_MODULE_CREATION, + "vkCreateShaderModule failed: VkResult=" + std::to_string( result ) ); + return false; + } + + VkDevice device = m_device; + PushTeardown( [device, module]() { vkDestroyShaderModule( device, module, nullptr ); } ); + + VkPipelineShaderStageCreateInfo stageInfo{}; + stageInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stageInfo.stage = ( s.stage == sgns::Stage::VERTEX ) ? VK_SHADER_STAGE_VERTEX_BIT + : VK_SHADER_STAGE_FRAGMENT_BIT; + stageInfo.module = module; + stageInfo.pName = s.entry_point.c_str(); + shaderStages.push_back( stageInfo ); + } + + uint32_t stride = 0; + for ( const auto &entry : vertexLayout ) + { + stride += VertexFormatByteSize( entry.get_format() ); + } + + VkVertexInputBindingDescription bindingDesc{}; + bindingDesc.binding = 0; + bindingDesc.stride = stride; + bindingDesc.inputRate = VK_VERTEX_INPUT_RATE_VERTEX; + + std::vector attributeDescs; + attributeDescs.reserve( vertexLayout.size() ); + for ( size_t i = 0; i < vertexLayout.size(); ++i ) + { + VkVertexInputAttributeDescription attr{}; + attr.location = static_cast( i ); + attr.binding = 0; + attr.format = ToVkFormat( vertexLayout[i].get_format() ); + attr.offset = static_cast( vertexLayout[i].get_offset() ); + attributeDescs.push_back( attr ); + } + + VkPipelineVertexInputStateCreateInfo vertexInputInfo{}; + vertexInputInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO; + vertexInputInfo.vertexBindingDescriptionCount = vertexLayout.empty() ? 0 : 1; + vertexInputInfo.pVertexBindingDescriptions = vertexLayout.empty() ? nullptr : &bindingDesc; + vertexInputInfo.vertexAttributeDescriptionCount = static_cast( attributeDescs.size() ); + vertexInputInfo.pVertexAttributeDescriptions = attributeDescs.empty() ? nullptr : attributeDescs.data(); + + sgns::Topology topology = sgns::Topology::TRIANGLE_LIST; + sgns::CullMode cullMode = sgns::CullMode::BACK; + sgns::FrontFace frontFace = sgns::FrontFace::CCW; + sgns::DepthTest depthTest = sgns::DepthTest::ENABLED; + if ( pipelineState ) + { + if ( pipelineState->get_topology() ) + { + topology = pipelineState->get_topology().value(); + } + if ( pipelineState->get_cull_mode() ) + { + cullMode = pipelineState->get_cull_mode().value(); + } + if ( pipelineState->get_front_face() ) + { + frontFace = pipelineState->get_front_face().value(); + } + if ( pipelineState->get_depth_test() ) + { + depthTest = pipelineState->get_depth_test().value(); + } + } + + VkPipelineInputAssemblyStateCreateInfo inputAssembly{}; + inputAssembly.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO; + inputAssembly.topology = ToVkTopology( topology ); + inputAssembly.primitiveRestartEnable = VK_FALSE; + + VkPipelineRasterizationStateCreateInfo rasterizer{}; + rasterizer.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; + rasterizer.polygonMode = VK_POLYGON_MODE_FILL; + rasterizer.cullMode = ToVkCullMode( cullMode ); + rasterizer.frontFace = ToVkFrontFace( frontFace ); + rasterizer.lineWidth = 1.0f; + + VkPipelineDepthStencilStateCreateInfo depthStencil{}; + depthStencil.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO; + depthStencil.depthTestEnable = ToVkBool( depthTest ); + depthStencil.depthWriteEnable = depthStencil.depthTestEnable; // [ASSUMED] tied to depthTestEnable -- no + // separate schema field exists (RESEARCH.md A1) + depthStencil.depthCompareOp = VK_COMPARE_OP_LESS; // fixed per D-14, never schema-configurable + + VkPipelineMultisampleStateCreateInfo multisample{}; + multisample.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; + multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; // ALWAYS -- DETV-02, never configurable + + VkPipelineColorBlendAttachmentState colorBlendAttachment{}; + colorBlendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT; + colorBlendAttachment.blendEnable = VK_FALSE; + + VkPipelineColorBlendStateCreateInfo colorBlending{}; + colorBlending.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO; + colorBlending.attachmentCount = 1; + colorBlending.pAttachments = &colorBlendAttachment; + + // Fixed (never a runtime-settable pipeline attribute, per D-22) viewport/ + // scissor sized to BuildRenderPass()'s already-validated render target + // dimensions. + VkViewport viewport{}; + viewport.x = 0.0f; + viewport.y = 0.0f; + viewport.width = static_cast( m_renderWidth ); + viewport.height = static_cast( m_renderHeight ); + viewport.minDepth = 0.0f; + viewport.maxDepth = 1.0f; + + VkRect2D scissor{}; + scissor.offset = { 0, 0 }; + scissor.extent = { m_renderWidth, m_renderHeight }; + + VkPipelineViewportStateCreateInfo viewportState{}; + viewportState.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; + viewportState.viewportCount = 1; + viewportState.pViewports = &viewport; + viewportState.scissorCount = 1; + viewportState.pScissors = &scissor; + + // D-29/D-30: fixed 128-byte push-constant threshold, all-or-nothing. + bool usePushConstant = uniforms.pushConstant && !uniforms.packedBytes.empty(); + bool useDescriptorSet = !uniforms.pushConstant && !uniforms.packedBytes.empty(); + + VkPushConstantRange pushConstantRange{}; + if ( usePushConstant ) + { + pushConstantRange.stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT; + pushConstantRange.offset = 0; + pushConstantRange.size = static_cast( uniforms.packedBytes.size() ); + } + + VkResult result = VK_SUCCESS; + + if ( useDescriptorSet ) + { + VkDescriptorSetLayoutBinding binding{}; + binding.binding = 0; + binding.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; + binding.descriptorCount = 1; + binding.stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT; + + VkDescriptorSetLayoutCreateInfo layoutInfo{}; + layoutInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; + layoutInfo.bindingCount = 1; + layoutInfo.pBindings = &binding; + + result = vkCreateDescriptorSetLayout( m_device, &layoutInfo, nullptr, &m_descriptorSetLayout ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::PIPELINE_CREATION, + "vkCreateDescriptorSetLayout failed: VkResult=" + std::to_string( result ) ); + return false; + } + { + VkDevice device = m_device; + VkDescriptorSetLayout layout = m_descriptorSetLayout; + PushTeardown( [device, layout]() { vkDestroyDescriptorSetLayout( device, layout, nullptr ); } ); + } + + VkDescriptorPoolSize poolSize{}; + poolSize.type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; + poolSize.descriptorCount = 1; + + VkDescriptorPoolCreateInfo poolInfo{}; + poolInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + poolInfo.poolSizeCount = 1; + poolInfo.pPoolSizes = &poolSize; + poolInfo.maxSets = 1; // matches D-22's per-job-only lifetime + + result = vkCreateDescriptorPool( m_device, &poolInfo, nullptr, &m_descriptorPool ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::PIPELINE_CREATION, + "vkCreateDescriptorPool failed: VkResult=" + std::to_string( result ) ); + return false; + } + { + VkDevice device = m_device; + VkDescriptorPool pool = m_descriptorPool; + PushTeardown( [device, pool]() { vkDestroyDescriptorPool( device, pool, nullptr ); } ); + } + + VkDescriptorSetAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + allocInfo.descriptorPool = m_descriptorPool; + allocInfo.descriptorSetCount = 1; + allocInfo.pSetLayouts = &m_descriptorSetLayout; + + result = vkAllocateDescriptorSets( m_device, &allocInfo, &m_descriptorSet ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::PIPELINE_CREATION, + "vkAllocateDescriptorSets failed: VkResult=" + std::to_string( result ) ); + return false; + } + // m_descriptorSet is freed automatically when m_descriptorPool is + // destroyed -- no separate PushTeardown needed for the set itself. + } + + VkPipelineLayoutCreateInfo pipelineLayoutInfo{}; + pipelineLayoutInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + if ( usePushConstant ) + { + pipelineLayoutInfo.pushConstantRangeCount = 1; + pipelineLayoutInfo.pPushConstantRanges = &pushConstantRange; + } + if ( useDescriptorSet ) + { + pipelineLayoutInfo.setLayoutCount = 1; + pipelineLayoutInfo.pSetLayouts = &m_descriptorSetLayout; + } + // If uniforms.packedBytes is empty (no uniforms declared at all), neither + // branch above ran -- pipelineLayoutInfo keeps zero push-constant ranges + // and zero descriptor sets, exactly as required. + + result = vkCreatePipelineLayout( m_device, &pipelineLayoutInfo, nullptr, &m_pipelineLayout ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::PIPELINE_CREATION, + "vkCreatePipelineLayout failed: VkResult=" + std::to_string( result ) ); + return false; + } + { + VkDevice device = m_device; + VkPipelineLayout layout = m_pipelineLayout; + PushTeardown( [device, layout]() { vkDestroyPipelineLayout( device, layout, nullptr ); } ); + } + + VkGraphicsPipelineCreateInfo pipelineInfo{}; + pipelineInfo.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + pipelineInfo.stageCount = static_cast( shaderStages.size() ); + pipelineInfo.pStages = shaderStages.data(); + pipelineInfo.pVertexInputState = &vertexInputInfo; + pipelineInfo.pInputAssemblyState = &inputAssembly; + pipelineInfo.pViewportState = &viewportState; + pipelineInfo.pRasterizationState = &rasterizer; + pipelineInfo.pMultisampleState = &multisample; + pipelineInfo.pDepthStencilState = &depthStencil; + pipelineInfo.pColorBlendState = &colorBlending; + pipelineInfo.layout = m_pipelineLayout; + pipelineInfo.renderPass = m_renderPass; + pipelineInfo.subpass = 0; + + result = vkCreateGraphicsPipelines( m_device, VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &m_pipeline ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::PIPELINE_CREATION, + "vkCreateGraphicsPipelines failed: VkResult=" + std::to_string( result ) ); + return false; + } + { + VkDevice device = m_device; + VkPipeline pipeline = m_pipeline; + PushTeardown( [device, pipeline]() { vkDestroyPipeline( device, pipeline, nullptr ); } ); + } + + return true; + } + + bool RenderProcessor::UploadBuffers( const std::vector &vertexBytes, + bool hasIndex, + sgns::IndexType indexType, + const std::vector &indexBytes, + uint32_t stride, + const ResolvedUniforms &uniforms, + ProcessingResult &errorOut ) + { + // Validated BEFORE any buffer is created / any vkCmdBindVertexBuffers or + // vkCmdDrawIndexed is ever recorded -- closes T-03-03-02 (out-of-bounds GPU + // buffer read from a byte length that doesn't match the pipeline's implied + // stride/index count). + if ( stride == 0 || vertexBytes.size() % stride != 0 ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "UploadBuffers: vertex buffer byte length (" + + std::to_string( vertexBytes.size() ) + + ") is not an exact multiple of the pipeline's computed stride (" + + std::to_string( stride ) + ")" ); + return false; + } + m_vertexCount = static_cast( vertexBytes.size() / stride ); + + m_hasIndexBuffer = hasIndex; + m_indexType = indexType; + m_indexCount = 0; + + if ( hasIndex ) + { + size_t indexElemSize = ( indexType == sgns::IndexType::UINT16 ) ? sizeof( uint16_t ) : sizeof( uint32_t ); + if ( indexBytes.size() % indexElemSize != 0 ) + { + errorOut = MakeError( ProcessingErrorStage::RESOURCE_RESOLUTION, + "UploadBuffers: index buffer byte length (" + + std::to_string( indexBytes.size() ) + + ") is not an exact multiple of the index type's byte size (" + + std::to_string( indexElemSize ) + ")" ); + return false; + } + m_indexCount = static_cast( indexBytes.size() / indexElemSize ); + } + + // Vertex buffer -- HOST_VISIBLE|HOST_COHERENT direct write (D-20/D-21), no + // staging+device-local path. + if ( !CreateBufferDedicated( vertexBytes.size(), VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + m_vertexBuffer, m_vertexMemory, errorOut ) ) + { + return false; + } + { + void *mapped = nullptr; + VkResult result = vkMapMemory( m_device, m_vertexMemory, 0, vertexBytes.size(), 0, &mapped ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::BUFFER_ALLOCATION, + "UploadBuffers: vkMapMemory (vertex) failed: VkResult=" + + std::to_string( result ) ); + return false; + } + std::memcpy( mapped, vertexBytes.data(), vertexBytes.size() ); + vkUnmapMemory( m_device, m_vertexMemory ); // HOST_COHERENT -- no flush needed (D-20) + } + + if ( hasIndex ) + { + if ( !CreateBufferDedicated( indexBytes.size(), VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + m_indexBuffer, m_indexMemory, errorOut ) ) + { + return false; + } + void *mapped = nullptr; + VkResult result = vkMapMemory( m_device, m_indexMemory, 0, indexBytes.size(), 0, &mapped ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::BUFFER_ALLOCATION, + "UploadBuffers: vkMapMemory (index) failed: VkResult=" + + std::to_string( result ) ); + return false; + } + std::memcpy( mapped, indexBytes.data(), indexBytes.size() ); + vkUnmapMemory( m_device, m_indexMemory ); + } + + m_usePushConstant = uniforms.pushConstant && !uniforms.packedBytes.empty(); + m_pushConstantBytes = m_usePushConstant ? uniforms.packedBytes : std::vector(); + + // Descriptor-set path only -- the push-constant path needs no VkBuffer at + // all (bytes copied directly from m_pushConstantBytes at record time). + if ( !uniforms.packedBytes.empty() && !uniforms.pushConstant ) + { + if ( !CreateBufferDedicated( uniforms.packedBytes.size(), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + m_uniformBuffer, m_uniformMemory, errorOut ) ) + { + return false; + } + void *mapped = nullptr; + VkResult result = vkMapMemory( m_device, m_uniformMemory, 0, uniforms.packedBytes.size(), 0, &mapped ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::BUFFER_ALLOCATION, + "UploadBuffers: vkMapMemory (uniform) failed: VkResult=" + + std::to_string( result ) ); + return false; + } + std::memcpy( mapped, uniforms.packedBytes.data(), uniforms.packedBytes.size() ); + vkUnmapMemory( m_device, m_uniformMemory ); + + if ( m_descriptorSet != VK_NULL_HANDLE ) + { + VkDescriptorBufferInfo bufferInfo{}; + bufferInfo.buffer = m_uniformBuffer; + bufferInfo.offset = 0; + bufferInfo.range = uniforms.packedBytes.size(); + + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = m_descriptorSet; + write.dstBinding = 0; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; + write.pBufferInfo = &bufferInfo; + + vkUpdateDescriptorSets( m_device, 1, &write, 0, nullptr ); + } + } + + return true; + } + + bool RenderProcessor::RecordAndSubmit( const sgns::RenderTarget &target, ProcessingResult &errorOut ) + { + VkCommandPoolCreateInfo poolInfo{}; + poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + poolInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT; + poolInfo.queueFamilyIndex = m_queueFamilyIndex; + + VkResult result = vkCreateCommandPool( m_device, &poolInfo, nullptr, &m_commandPool ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::DRAW_SUBMISSION, + "RecordAndSubmit: vkCreateCommandPool failed: VkResult=" + + std::to_string( result ) ); + return false; + } + { + VkDevice device = m_device; + VkCommandPool pool = m_commandPool; + // Pool destruction frees m_commandBuffer too -- no separate teardown entry. + PushTeardown( [device, pool]() { vkDestroyCommandPool( device, pool, nullptr ); } ); + } + + VkCommandBufferAllocateInfo cbAllocInfo{}; + cbAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cbAllocInfo.commandPool = m_commandPool; + cbAllocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cbAllocInfo.commandBufferCount = 1; + + result = vkAllocateCommandBuffers( m_device, &cbAllocInfo, &m_commandBuffer ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::DRAW_SUBMISSION, + "RecordAndSubmit: vkAllocateCommandBuffers failed: VkResult=" + + std::to_string( result ) ); + return false; + } + + VkCommandBufferBeginInfo beginInfo{}; + beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + + result = vkBeginCommandBuffer( m_commandBuffer, &beginInfo ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::DRAW_SUBMISSION, + "RecordAndSubmit: vkBeginCommandBuffer failed: VkResult=" + + std::to_string( result ) ); + return false; + } + + VkClearValue clearValues[2]{}; + const auto &clearColor = target.get_clear_color(); + for ( size_t i = 0; i < 4 && i < clearColor.size(); ++i ) + { + clearValues[0].color.float32[i] = static_cast( clearColor[i] ); + } + clearValues[1].depthStencil.depth = static_cast( target.get_clear_depth() ); + clearValues[1].depthStencil.stencil = 0; + + VkRenderPassBeginInfo rpBeginInfo{}; + rpBeginInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO; + rpBeginInfo.renderPass = m_renderPass; + rpBeginInfo.framebuffer = m_framebuffer; + rpBeginInfo.renderArea.offset = { 0, 0 }; + rpBeginInfo.renderArea.extent = { m_renderWidth, m_renderHeight }; + rpBeginInfo.clearValueCount = 2; + rpBeginInfo.pClearValues = clearValues; + + vkCmdBeginRenderPass( m_commandBuffer, &rpBeginInfo, VK_SUBPASS_CONTENTS_INLINE ); + + vkCmdBindPipeline( m_commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipeline ); + + VkDeviceSize vbOffset = 0; + vkCmdBindVertexBuffers( m_commandBuffer, 0, 1, &m_vertexBuffer, &vbOffset ); + + if ( m_hasIndexBuffer ) + { + vkCmdBindIndexBuffer( m_commandBuffer, m_indexBuffer, 0, + ( m_indexType == sgns::IndexType::UINT16 ) ? VK_INDEX_TYPE_UINT16 + : VK_INDEX_TYPE_UINT32 ); + } + + if ( m_usePushConstant && !m_pushConstantBytes.empty() ) + { + vkCmdPushConstants( m_commandBuffer, m_pipelineLayout, + VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, + static_cast( m_pushConstantBytes.size() ), m_pushConstantBytes.data() ); + } + else if ( m_descriptorSet != VK_NULL_HANDLE ) + { + vkCmdBindDescriptorSets( m_commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipelineLayout, 0, 1, + &m_descriptorSet, 0, nullptr ); + } + + if ( m_hasIndexBuffer ) + { + vkCmdDrawIndexed( m_commandBuffer, m_indexCount, 1, 0, 0, 0 ); + } + else + { + vkCmdDraw( m_commandBuffer, m_vertexCount, 1, 0, 0 ); + } + + vkCmdEndRenderPass( m_commandBuffer ); + + // Readback copy recorded INSIDE this same command buffer, immediately after + // vkCmdEndRenderPass and before vkEndCommandBuffer -- no second command + // buffer/submission (Pitfall 4). The render pass's color attachment + // finalLayout is already VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL (plan 03-04's + // BuildRenderPass), so no extra image-layout-transition barrier is needed + // here. + VkDeviceSize stagingSize = static_cast( target.get_width() ) * + static_cast( target.get_height() ) * + ColorFormatByteSize( target.get_color_format() ); + + if ( !CreateBufferDedicated( stagingSize, VK_BUFFER_USAGE_TRANSFER_DST_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + m_stagingBuffer, m_stagingMemory, errorOut ) ) + { + return false; + } + + VkBufferImageCopy region{}; + region.bufferOffset = 0; + region.bufferRowLength = 0; + region.bufferImageHeight = 0; + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.imageSubresource.mipLevel = 0; + region.imageSubresource.baseArrayLayer = 0; + region.imageSubresource.layerCount = 1; + region.imageOffset = { 0, 0, 0 }; + region.imageExtent = { m_renderWidth, m_renderHeight, 1 }; + + vkCmdCopyImageToBuffer( m_commandBuffer, m_colorImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, m_stagingBuffer, + 1, ®ion ); + + result = vkEndCommandBuffer( m_commandBuffer ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::DRAW_SUBMISSION, + "RecordAndSubmit: vkEndCommandBuffer failed: VkResult=" + + std::to_string( result ) ); + return false; + } + + VkSubmitInfo submitInfo{}; + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &m_commandBuffer; + + result = vkQueueSubmit( m_queue, 1, &submitInfo, VK_NULL_HANDLE ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::DRAW_SUBMISSION, + "RecordAndSubmit: vkQueueSubmit failed: VkResult=" + std::to_string( result ) ); + return false; + } + + // D-23: synchronous wait, RenderProcessor's own independent VkDevice -- this + // cannot stall a host application's separate VkDevice/queue. + result = vkDeviceWaitIdle( m_device ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::DRAW_SUBMISSION, + "RecordAndSubmit: vkDeviceWaitIdle failed: VkResult=" + std::to_string( result ) ); + return false; + } + + return true; + } + + bool RenderProcessor::Readback( const sgns::RenderTarget &target, std::vector &outBytes, + ProcessingResult &errorOut ) + { + VkDeviceSize size = static_cast( target.get_width() ) * + static_cast( target.get_height() ) * + ColorFormatByteSize( target.get_color_format() ); + + void *mapped = nullptr; + VkResult result = vkMapMemory( m_device, m_stagingMemory, 0, size, 0, &mapped ); + if ( result != VK_SUCCESS ) + { + errorOut = MakeError( ProcessingErrorStage::READBACK, + "Readback: vkMapMemory failed: VkResult=" + std::to_string( result ) ); + return false; + } + + outBytes.resize( static_cast( size ) ); + std::memcpy( outBytes.data(), mapped, static_cast( size ) ); + vkUnmapMemory( m_device, m_stagingMemory ); // HOST_COHERENT -- no invalidate needed (D-20) + + return true; + } + + ProcessingResult RenderProcessor::StartProcessing( + std::vector> &chunkhashes, + const sgns::IoDeclaration &proc, + std::vector &imageData, + std::vector &modelFile, + const std::vector *parameters, + const ExecutionContext &execCtx ) + { + (void)proc; + (void)chunkhashes; + + // Extract pass_id for progress events + const std::string passId = proc.get_name(); + + if ( !InitializeContext() ) + { + RunTeardown(); + return MakeError( ProcessingErrorStage::CONTEXT_INIT_FAILED, "InitializeContext failed" ); + } + + ProcessingResult errorOut; + + // (1) Invert plan 03-01's compiled-stage wire format. + std::vector stages; + if ( !ParseCompiledStages( modelFile, stages, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + // COMPILE stage complete — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForRender( passId, RenderStage::COMPILE, 25.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return MakeError( ProcessingErrorStage::CANCELLED, "Render pass cancelled" ); + } + + // (2) ParseRenderPassConfig() is the ONLY source of RenderTarget/ + // PipelineState/VertexLayoutEntry/uniforms/vertex-index bytes/ + // dataTransformCount -- StartProcessing()'s own parameters never carry a + // Pass/RenderShaderConfig object. + sgns::RenderTarget renderTarget; + boost::optional pipelineState; + std::vector vertexLayout; + boost::optional> uniformsMap; + std::vector vertexBytes; + bool hasIndex = false; + sgns::IndexType indexType = sgns::IndexType::UINT32; + std::vector indexBytes; + uint32_t dataTransformCount = 0; + + if ( !ParseRenderPassConfig( imageData, renderTarget, pipelineState, vertexLayout, uniformsMap, vertexBytes, + hasIndex, indexType, indexBytes, dataTransformCount, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + // (3) Resolve literal/parameter:-sourced uniform values into packed bytes. + ResolvedUniforms resolvedUniforms; + if ( !ResolveUniforms( uniformsMap, parameters, resolvedUniforms, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + const int maskBits = sgns::sgprocmanagerquant::ResolveByteQuantMode( parameters ); + + // (4)-(6): build the offscreen render pass/framebuffer/pipeline (plan 03-04). + if ( !BuildRenderPass( renderTarget, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + if ( !BuildFramebuffer( renderTarget, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + if ( !BuildPipeline( stages, vertexLayout, pipelineState, resolvedUniforms, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + // BUILD_PIPELINE stage complete — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForRender( passId, RenderStage::BUILD_PIPELINE, 50.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return MakeError( ProcessingErrorStage::CANCELLED, "Render pass cancelled" ); + } + + // (7) Upload vertex/index/uniform buffers -- stride computed identically to + // BuildPipeline()'s own vertex-input stride (sum of VertexFormatByteSize() + // over vertexLayout), computed once and passed to both. + uint32_t stride = 0; + for ( const auto &entry : vertexLayout ) + { + stride += VertexFormatByteSize( entry.get_format() ); + } + + if ( !UploadBuffers( vertexBytes, hasIndex, indexType, indexBytes, stride, resolvedUniforms, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + // (8) RENDER-07: no data_transform executor exists anywhere in this codebase + if ( dataTransformCount > 0 ) + { + RunTeardown(); + return MakeError( ProcessingErrorStage::DATA_TRANSFORM_UNSUPPORTED, + "data_transform declared (" + std::to_string( dataTransformCount ) + + " entries) but no executor exists in this phase" ); + } + + // (9)-(10): record+submit the single command buffer (including the readback + // copy recorded inline) and map the staging buffer's bytes out. + if ( !RecordAndSubmit( renderTarget, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + // DRAW stage complete — fire progress and check cancel + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForRender( passId, RenderStage::DRAW, 75.0f ) ); + } + if ( execCtx.cancelToken.IsCancelled() ) + { + RunTeardown(); + return MakeError( ProcessingErrorStage::CANCELLED, "Render pass cancelled" ); + } + + std::vector readbackBytes; + if ( !Readback( renderTarget, readbackBytes, errorOut ) ) + { + RunTeardown(); + return errorOut; + } + + // READBACK stage complete — fire progress + if ( execCtx.progressCallback ) + { + execCtx.progressCallback( ProgressEvent::ForRender( passId, RenderStage::READBACK, 100.0f ) ); + } + + // Output budget check (EXEC-03, D-03/D-08) + if ( execCtx.maxOutputArtifactBytes > 0 ) + { + size_t outputSize = readbackBytes.size(); + if ( outputSize > execCtx.maxOutputArtifactBytes ) + { + RunTeardown(); + return MakeError( ProcessingErrorStage::BUDGET_EXCEEDED, + "Output artifact size " + std::to_string( outputSize ) + " exceeds budget " + std::to_string( execCtx.maxOutputArtifactBytes ) ); + } + } + + // (11) Success: tear down every per-job Vulkan object (D-22/D-23) before + // populating the final ProcessingResult from the raw readback bytes. + RunTeardown(); + + // Phase 10 CAPT-02: quantize (no-op stub) then offer the pre-/post-quantize + // bytes to the opt-in capture callback before the single combined-hash call. + // readbackBytes is locally-owned (not foreign MNN tensor memory), so it is + // safe to mutate in place -- no copy-before-mutate constraint applies here. + std::vector preQuantizeSnapshot; + if ( execCtx.rawOutputCapture ) + { + preQuantizeSnapshot = readbackBytes; + } + sgns::sgprocmanagerquant::QuantizeByteBuffer( readbackBytes.data(), readbackBytes.size(), maskBits ); + if ( execCtx.rawOutputCapture ) + { + execCtx.rawOutputCapture( readbackBytes, preQuantizeSnapshot ); + } + + ProcessingResult result; + result.hash = sgns::sgprocmanagersha::sha256( readbackBytes.data(), readbackBytes.size() ); + result.output_buffers = + std::make_shared, std::vector>>>( + std::vector{ std::string{} }, + std::vector>{ std::vector( readbackBytes.begin(), readbackBytes.end() ) } ); + result.error = std::nullopt; + m_progress = 100.0f; + + return result; + } + +} diff --git a/src/processors/vulkan_gpu_probe.cpp b/src/processors/vulkan_gpu_probe.cpp new file mode 100644 index 0000000..4c0bdf4 --- /dev/null +++ b/src/processors/vulkan_gpu_probe.cpp @@ -0,0 +1,67 @@ +#include "processors/vulkan_gpu_probe.hpp" +#include "processors/processing_processor_render.hpp" +#include "processingbase/vulkan_init_guard.hpp" +#include +#include +#include + +namespace sgns::sgprocessing +{ + bool HasUsableVulkanDevice() + { + try + { + std::lock_guard lock( sgns::sgprocessing::VulkanInitMutex() ); + + // On macOS MoltenVK is statically linked (libMoltenVK.a), so there is + // no libvulkan.dylib for vk-bootstrap's default dlopen path to find. + // Pass the statically-available vkGetInstanceProcAddr directly to + // bypass dynamic loading entirely. +#if defined(__APPLE__) + vkb::InstanceBuilder instance_builder( vkGetInstanceProcAddr ); +#else + vkb::InstanceBuilder instance_builder; +#endif + auto inst_ret = instance_builder.set_app_name( "SGProcessingManager GPU Probe" ) + .set_app_version( 1, 0, 0 ) + .request_validation_layers( false ) + .build(); + if ( !inst_ret ) + { + // No instance was created -- nothing to destroy. + return false; + } + auto vkb_instance = inst_ret.value(); + + vkb::PhysicalDeviceSelector selector( vkb_instance ); + // Headless/offscreen probe -- no VkSurfaceKHR ever exists, same rationale + // as RenderProcessor::InitializeContext()'s own require_present(false). + selector.require_present( false ); + auto devices_ret = selector.select_devices(); + if ( !devices_ret ) + { + vkb::destroy_instance( vkb_instance ); + return false; + } + + auto devices = devices_ret.value(); + + devices.erase( std::remove_if( devices.begin(), + devices.end(), + []( const vkb::PhysicalDevice &d ) + { return !RenderProcessor::IsAcceptable( d.properties.deviceType ); } ), + devices.end() ); + + bool usable = !devices.empty(); + + vkb::destroy_instance( vkb_instance ); + + return usable; + } + catch ( ... ) + { + // Never throw -- a probe failure of any kind means "no usable device". + return false; + } + } +} diff --git a/src/shaders/CMakeLists.txt b/src/shaders/CMakeLists.txt new file mode 100644 index 0000000..7172aa3 --- /dev/null +++ b/src/shaders/CMakeLists.txt @@ -0,0 +1,18 @@ +add_library(SGShaderCompiler STATIC + shader_compiler.cpp + ../../include/shaders/shader_compiler.hpp +) + +target_include_directories(SGShaderCompiler PUBLIC + $ + $ + $ +) + +target_link_libraries(SGShaderCompiler PUBLIC + sgprocmanagerlogger + sgprocmanagertypes + shaderc::shaderc +) + +sgnus_install(SGShaderCompiler) diff --git a/src/shaders/shader_compiler.cpp b/src/shaders/shader_compiler.cpp new file mode 100644 index 0000000..7f217be --- /dev/null +++ b/src/shaders/shader_compiler.cpp @@ -0,0 +1,94 @@ +#include "shaders/shader_compiler.hpp" + +#include +#include + +#include + +OUTCOME_CPP_DEFINE_CATEGORY_3( sgns::sgprocessing, ShaderCompiler::Error, e ) +{ + switch ( e ) + { + case sgns::sgprocessing::ShaderCompiler::Error::COMPILE_FAILED: + return "Shader source failed to compile (GLSL -> SPIR-V)"; + case sgns::sgprocessing::ShaderCompiler::Error::VALIDATION_FAILED: + return "SPIR-V failed SPIRV-Tools validation"; + } + return "Unknown error"; +} + +namespace sgns::sgprocessing +{ + outcome::result ShaderCompiler::CompileAndValidate( const std::vector &source_bytes, + sgns::Stage stage, + sgns::ShaderSourceType type, + const std::string &entry_point ) + { + auto message_consumer = [ this ]( spv_message_level_t, const char *, const spv_position_t &, + const char *message ) + { + m_logger->error( "SPIRV-Tools validation message: {}", message ? message : "" ); + }; + + if ( type == sgns::ShaderSourceType::GLSL ) + { + // ---- GLSL -> SPIR-V compile path ---- + shaderc_shader_kind kind = ( stage == sgns::Stage::VERTEX ) ? shaderc_glsl_vertex_shader + : shaderc_glsl_fragment_shader; + + shaderc::Compiler compiler; + shaderc::CompileOptions options; + options.SetTargetEnvironment( shaderc_target_env_vulkan, shaderc_env_version_vulkan_1_3 ); + options.SetOptimizationLevel( shaderc_optimization_level_zero ); + + shaderc::SpvCompilationResult result = compiler.CompileGlslToSpv( + source_bytes.data(), source_bytes.size(), kind, "shader", entry_point.c_str(), options ); + + if ( result.GetCompilationStatus() != shaderc_compilation_status_success ) + { + m_logger->error( "ShaderCompiler: GLSL compile failed: {}", result.GetErrorMessage() ); + return outcome::failure( Error::COMPILE_FAILED ); + } + + std::vector spirv_words( result.cbegin(), result.cend() ); + + // Mandatory validation gate -- shaderc's CompileGlslToSpv() success does NOT mean + // SPIRV-Tools has validated the module. Never skip this call. + spvtools::SpirvTools tools( SPV_ENV_VULKAN_1_3 ); + tools.SetMessageConsumer( message_consumer ); + if ( !tools.Validate( spirv_words.data(), spirv_words.size() ) ) + { + m_logger->error( "ShaderCompiler: compiled SPIR-V failed SPIRV-Tools validation" ); + return outcome::failure( Error::VALIDATION_FAILED ); + } + + return CompiledShaderStage{ std::move( spirv_words ), stage }; + } + else // sgns::ShaderSourceType::SPIRV -- compilation skipped entirely + { + if ( source_bytes.size() % 4 != 0 ) + { + m_logger->error( "ShaderCompiler: direct SPIR-V submission has a size ({}) that is " + "not a multiple of 4 bytes", + source_bytes.size() ); + return outcome::failure( Error::VALIDATION_FAILED ); + } + + std::vector spirv_words( source_bytes.size() / 4 ); + std::memcpy( spirv_words.data(), source_bytes.data(), source_bytes.size() ); + + // Mandatory validation gate -- a directly-submitted payload gets zero exemption from + // this check, whether it is adversarial bytes or a mutated copy of previously-valid + // SPIR-V. This is the exact gate Pitfall 2 warns must never be bypassed. + spvtools::SpirvTools tools( SPV_ENV_VULKAN_1_3 ); + tools.SetMessageConsumer( message_consumer ); + if ( !tools.Validate( spirv_words.data(), spirv_words.size() ) ) + { + m_logger->error( "ShaderCompiler: directly-submitted SPIR-V failed SPIRV-Tools validation" ); + return outcome::failure( Error::VALIDATION_FAILED ); + } + + return CompiledShaderStage{ std::move( spirv_words ), stage }; + } + } +} diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 3d91776..566dac1 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -25,6 +25,36 @@ target_link_libraries(sgprocmanagersha ) sgnus_install(sgprocmanagersha) +add_library(sgprocmanagerquant + quantization.cpp + ../../include/util/quantization.hpp +) +target_include_directories(sgprocmanagerquant PUBLIC + $ + $ + $ +) +target_link_libraries(sgprocmanagerquant + PUBLIC + nlohmann_json::nlohmann_json +) +sgnus_install(sgprocmanagerquant) + +add_library(sgprocmanagerdiff + diff_utils.cpp + ../../include/util/diff_utils.hpp +) +target_include_directories(sgprocmanagerdiff PUBLIC + $ + $ + $ +) +target_link_libraries(sgprocmanagerdiff + PUBLIC + nlohmann_json::nlohmann_json +) +sgnus_install(sgprocmanagerdiff) + add_library(sgprocmanagertypes InputTypes.cpp ../../include/util/InputTypes.hpp diff --git a/src/util/diff_utils.cpp b/src/util/diff_utils.cpp new file mode 100644 index 0000000..3203006 --- /dev/null +++ b/src/util/diff_utils.cpp @@ -0,0 +1,256 @@ +#include "util/diff_utils.hpp" + +#include +#include +#include + +#include + +namespace sgns::sgprocmanagerdiff +{ + namespace + { + // Mirrors quantization.cpp's power-of-two check exactly (Phase 14 + // D-05/Pitfall 2): never use a transcendental logarithm/exponent + // function here -- a transcendental-function-based check's last-bit + // behavior is platform-dependent, which would reintroduce exactly the + // cross-hardware nondeterminism this milestone exists to eliminate. + // The integer bit-trick below is deterministic on every platform. + // Deliberately NOT shared with quantization.cpp -- see this plan's + // rationale: an isolated duplicate, not a refactor of + // existing Phase 14 code. + bool IsPositivePowerOfTwo( double value ) + { + if ( !( value > 0.0 ) ) + { + return false; + } + if ( std::floor( value ) != value ) + { + return false; + } + const auto asInt = static_cast( value ); + return asInt != 0u && ( asInt & ( asInt - 1u ) ) == 0u; + } + + // Isolated parameter lookup -- deliberately duplicates + // quantization.cpp's quantScale resolver find-by-name-and-type loop, + // but returns boost::none on any invalid/missing case instead of a + // fallback constant, since this plan's D-03/D-04 branch needs to + // distinguish "validly declared" from "fell back" (a distinction the + // existing quantization.cpp resolver's return type cannot express). + // Must NOT call into or modify quantization.cpp's own resolver. + boost::optional TryGetDeclaredQuantScale( const std::vector *parameters ) + { + if ( parameters ) + { + for ( const auto ¶m : *parameters ) + { + if ( param.get_name() == "quantScale" && param.get_type() == sgns::ParameterType::FLOAT ) + { + const auto &def = param.get_parameter_default(); + if ( def.is_number() ) + { + const double declared = def.get(); + if ( IsPositivePowerOfTwo( declared ) ) + { + return static_cast( declared ); + } + } + break; + } + } + } + return boost::none; + } + + // Isolated parameter lookup -- deliberately duplicates + // quantization.cpp's byteQuantMode resolver find-by-name-and-type + // loop, but returns boost::none on any invalid/missing case instead + // of a fallback constant. Must NOT call into or modify + // quantization.cpp's own resolver. + boost::optional TryGetDeclaredByteQuantMode( const std::vector *parameters ) + { + if ( parameters ) + { + for ( const auto ¶m : *parameters ) + { + if ( param.get_name() == "byteQuantMode" && param.get_type() == sgns::ParameterType::INT ) + { + const auto &def = param.get_parameter_default(); + if ( def.is_number_integer() ) + { + const int declared = def.get(); + if ( declared >= 0 && declared <= 8 ) + { + return declared; + } + } + break; + } + } + } + return boost::none; + } + } // namespace + + int64_t OrderedFloatBits( float f ) + { + int32_t bits; + std::memcpy( &bits, &f, sizeof( bits ) ); + int64_t wide = static_cast( bits ); + if ( bits < 0 ) + { + wide = static_cast( 0x80000000LL ) - wide; + } + return wide; + } + + int64_t UlpDistanceFloat( float a, float b ) + { + return std::llabs( OrderedFloatBits( a ) - OrderedFloatBits( b ) ); + } + + ElementDiffStats ComputeFloat32Diff( const std::vector &a, const std::vector &b ) + { + ElementDiffStats stats; + if ( a.size() != b.size() ) + { + stats.sizeMismatch = true; + return stats; + } + + stats.elementCount = a.size() / sizeof( float ); + size_t exceedingCount = 0; + + for ( size_t idx = 0; idx < stats.elementCount; ++idx ) + { + float valA; + float valB; + std::memcpy( &valA, a.data() + idx * sizeof( float ), sizeof( float ) ); + std::memcpy( &valB, b.data() + idx * sizeof( float ), sizeof( float ) ); + + float absDelta = std::fabs( valA - valB ); + float denom = std::max( { std::fabs( valA ), std::fabs( valB ), kRelativeDeltaEpsilonFloor } ); + float relDelta = absDelta / denom; + int64_t ulp = UlpDistanceFloat( valA, valB ); + + if ( relDelta > kDefaultFloatRelativeThreshold ) + { + ++exceedingCount; + } + + stats.maxAbsDelta = std::max( stats.maxAbsDelta, static_cast( absDelta ) ); + stats.maxRelDelta = std::max( stats.maxRelDelta, static_cast( relDelta ) ); + stats.maxUlpDistance = std::max( stats.maxUlpDistance, ulp ); + } + + stats.percentExceedingThreshold = + stats.elementCount == 0 ? 0.0 : 100.0 * static_cast( exceedingCount ) / static_cast( stats.elementCount ); + + return stats; + } + + ElementDiffStats ComputeUint8Diff( const std::vector &a, const std::vector &b ) + { + ElementDiffStats stats; + if ( a.size() != b.size() ) + { + stats.sizeMismatch = true; + return stats; + } + + stats.elementCount = a.size(); + size_t exceedingCount = 0; + + for ( size_t idx = 0; idx < stats.elementCount; ++idx ) + { + int valA = static_cast( a[idx] ); + int valB = static_cast( b[idx] ); + + int absDelta = std::abs( valA - valB ); + double denom = static_cast( std::max( { valA, valB, 1 } ) ); + double relDelta = static_cast( absDelta ) / denom; + int64_t ulp = absDelta; + + if ( absDelta > kDefaultByteAbsoluteThreshold ) + { + ++exceedingCount; + } + + stats.maxAbsDelta = std::max( stats.maxAbsDelta, static_cast( absDelta ) ); + stats.maxRelDelta = std::max( stats.maxRelDelta, relDelta ); + stats.maxUlpDistance = std::max( stats.maxUlpDistance, ulp ); + } + + stats.percentExceedingThreshold = + stats.elementCount == 0 ? 0.0 : 100.0 * static_cast( exceedingCount ) / static_cast( stats.elementCount ); + + return stats; + } + + ChunkElementType ResolveChunkElementTypeHint( const std::vector *parameters ) + { + const auto declaredMaskBits = TryGetDeclaredByteQuantMode( parameters ); + if ( declaredMaskBits && *declaredMaskBits > 0 ) + { + return ChunkElementType::UINT8; + } + return ChunkElementType::FLOAT32; + } + + bool IsFloatChunkWithinTolerance( const std::vector &a, + const std::vector &b, + const std::vector *parameters, + ElementDiffStats &statsOut ) + { + statsOut = ComputeFloat32Diff( a, b ); + if ( statsOut.sizeMismatch ) + { + return false; + } + + const auto declaredScale = TryGetDeclaredQuantScale( parameters ); + if ( declaredScale ) + { + // D-03: grid-step-derived bound -- two grid steps of margin, + // mirroring the project's own "one full step of margin above the + // confirmed boundary" philosophy (quantization.cpp's S=2^15 + // derivation history). + return statsOut.maxAbsDelta <= 2.0 / static_cast( *declaredScale ); + } + + // D-04: capture_diff's existing relative-threshold check, already + // computed inside ComputeFloat32Diff against + // kDefaultFloatRelativeThreshold. Zero-elements-may-exceed policy, + // not a percentage-based bar. + return statsOut.percentExceedingThreshold == 0.0; + } + + bool IsByteChunkWithinTolerance( const std::vector &a, + const std::vector &b, + const std::vector *parameters, + ElementDiffStats &statsOut ) + { + statsOut = ComputeUint8Diff( a, b ); + if ( statsOut.sizeMismatch ) + { + return false; + } + + const auto declaredMaskBits = TryGetDeclaredByteQuantMode( parameters ); + if ( declaredMaskBits ) + { + // D-03: mask-width-derived bound -- two values masking to the + // same quantized value can differ by up to (1<( ( 1 << *declaredMaskBits ) - 1 ); + return statsOut.maxAbsDelta <= bound; + } + + // D-04: capture_diff's existing absolute-threshold check, already + // computed inside ComputeUint8Diff against + // kDefaultByteAbsoluteThreshold. Zero-elements-may-exceed policy, + // not a percentage-based bar. + return statsOut.percentExceedingThreshold == 0.0; + } +} // namespace sgns::sgprocmanagerdiff diff --git a/src/util/quantization.cpp b/src/util/quantization.cpp new file mode 100644 index 0000000..1bbb91c --- /dev/null +++ b/src/util/quantization.cpp @@ -0,0 +1,198 @@ + + +#include "util/quantization.hpp" + +#include +#include + +namespace sgns::sgprocmanagerquant +{ + namespace + { + // Phase 14 D-05/Pitfall 2: never use std::log2/std::pow here -- a + // transcendental-function-based check's last-bit behavior is + // platform-dependent, which would reintroduce exactly the + // cross-hardware nondeterminism this milestone exists to eliminate. + // The integer bit-trick below is deterministic on every platform. + bool IsPositivePowerOfTwo( double value ) + { + if ( !( value > 0.0 ) ) + { + return false; + } + if ( std::floor( value ) != value ) + { + return false; + } + const auto asInt = static_cast( value ); + return asInt != 0u && ( asInt & ( asInt - 1u ) ) == 0u; + } + } // namespace + + float ResolveQuantScale( const std::vector *parameters ) + { + constexpr float kFallbackScale = 32768.0f; // 2^15, exact v2.1 constant (D-04) + + if ( parameters ) + { + for ( const auto ¶m : *parameters ) + { + if ( param.get_name() == "quantScale" && param.get_type() == sgns::ParameterType::FLOAT ) + { + const auto &def = param.get_parameter_default(); + if ( def.is_number() ) + { + const double declared = def.get(); + if ( IsPositivePowerOfTwo( declared ) ) + { + return static_cast( declared ); + } + } + break; + } + } + } + + return kFallbackScale; + } + + int ResolveByteQuantMode( const std::vector *parameters ) + { + constexpr int kFallbackMaskBits = 0; // Identity no-op, exact v2.1 behavior (D-08) + + if ( parameters ) + { + for ( const auto ¶m : *parameters ) + { + if ( param.get_name() == "byteQuantMode" && param.get_type() == sgns::ParameterType::INT ) + { + const auto &def = param.get_parameter_default(); + if ( def.is_number_integer() ) + { + const int declared = def.get(); + if ( declared >= 0 && declared <= 8 ) + { + return declared; + } + } + break; + } + } + } + + return kFallbackMaskBits; + } + + void QuantizeFloatBuffer( float *data, size_t count, float scale ) + { + // Phase 13 Plan 13-04 gap-closure widening (supersedes Phase 12 D-05's + // 2^20 value): the original S=2^20 grid step (9.5367431640625e-07) + // gave only a ~9.14x margin over Phase 11's measured cross-machine + // maxAbsDelta (1.043081283569336e-07); Phase 13's own fresh + // re-validation (13-SCOPE-BOUNDARY.md) measured a post-quantization + // maxAbsDelta of exactly 9.5367431640625e-07 (one full old-grid step) + // with 12 of 15 MNN chunk hashes still diverging cross-hardware -- + // direct evidence the ~9x margin was insufficient. + // + // A local binary search over power-of-two S values (13-04-PLAN.md + // Task 1, revised approach) against processing_conformance_security_ + // test's Secv01CounterTest.MnnCorruptedModelStillDiverges found: + // S=2^20 (9.5367431640625e-07 grid step) -- SECV-01 passes (baseline) + // S=2^17 (7.62939453125e-06 grid step) -- SECV-01 passes + // S=2^16 (1.52587890625e-05 grid step) -- SECV-01 passes + // S=2^15 (3.0517578125e-05 grid step) -- SECV-01 passes + // S=2^14 (6.103515625e-05 grid step) -- SECV-01 FAILS (the + // deliberately corrupted MNN model's artifactId collides + // bit-for-bit with the correct model's, memcmp equal, 0 vs 0 -- + // confirmed deterministic, not flaky, by re-running twice) + // S=2^15 is chosen: the widest power-of-two grid step confirmed safe, + // one full power-of-two step of margin above the confirmed S=2^14 + // failure boundary (not the exact edge), giving 32x the old S=2^20 + // grid step (~292x Phase 11's original maxAbsDelta) while still + // leaving SECV-01's corrupted-model divergence fully intact. + // + // Phase 14 (QUANT-CFG-01/02): this constant is no longer hardcoded + // here -- callers resolve it via ResolveQuantScale() (D-04/D-05 + // fallback to this exact 32768.0f value) and pass it as `scale`. + + for ( size_t i = 0; i < count; ++i ) + { + float x = data[i]; + + // Extract the bit pattern via memcpy (never a reinterpret_cast + // type-pun), mirroring HalfToFloat's existing bit-punning style + // (processing_processor_mnn_float.cpp). + uint32_t bits = 0; + std::memcpy( &bits, &x, sizeof( bits ) ); + + const uint32_t exponentBits = bits & 0x7F800000u; + const uint32_t mantissaBits = bits & 0x007FFFFFu; + + // Branch order is itself the D-07 requirement: every special-value + // check below is evaluated before the rounding arithmetic in the + // final else arm ever runs. + + // 1. Denormal (both signs, D-07): biased exponent field is zero but + // mantissa is nonzero. Flush to canonical +0.0 (D-08). + if ( exponentBits == 0u && mantissaBits != 0u ) + { + data[i] = 0.0f; + } + // 2. NaN: canonicalize to the hardcoded quiet-NaN bit pattern + // 0x7FC00000 (D-09), regardless of payload/sign/signaling bit. + else if ( std::isnan( x ) ) + { + constexpr uint32_t kCanonicalNaN = 0x7FC00000u; + std::memcpy( &data[i], &kCanonicalNaN, sizeof( kCanonicalNaN ) ); + } + // 3. Infinity: two distinct fixed bit patterns (D-06), never + // collapsed to one value. + else if ( std::isinf( x ) ) + { + constexpr uint32_t kPositiveInfinity = 0x7F800000u; + constexpr uint32_t kNegativeInfinity = 0xFF800000u; + if ( std::signbit( x ) ) + { + std::memcpy( &data[i], &kNegativeInfinity, sizeof( kNegativeInfinity ) ); + } + else + { + std::memcpy( &data[i], &kPositiveInfinity, sizeof( kPositiveInfinity ) ); + } + } + // 4. Signed zero (D-08): +0.0 and -0.0 both compare equal to 0.0f + // under IEEE equality; collapse to the single canonical zero. + else if ( x == 0.0f ) + { + data[i] = 0.0f; + } + // 5. Ordinary finite value: fixed-point scale-round-cast (D-03). + else + { + data[i] = std::round( x * scale ) / scale; + } + } + } + + void QuantizeByteBuffer( uint8_t *data, size_t count, int maskBits ) + { + // D-01/QUANT-04: byte-identity no-op when nothing (valid) is + // schema-declared -- see header doc comment for the Phase 11 + // empirical justification (contentHashMatch: true, all deltas 0.0). + // Phase 14 D-07: maskBits<=0 (absent/N=0) is exactly this v2.1 + // identity behavior, unchanged. + if ( maskBits <= 0 ) + { + return; + } + + // D-06: clear the low `maskBits` bits of every byte. maskBits is + // resolver-validated to [0, 8] (ResolveByteQuantMode), so the shift + // below never exceeds the width of an unsigned int. + const uint8_t mask = static_cast( ~( ( 1u << maskBits ) - 1u ) ); + for ( size_t i = 0; i < count; ++i ) + { + data[i] &= mask; + } + } +} // namespace sgns::sgprocmanagerquant diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..c0aa6f4 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,6 @@ +include(GoogleTest) +add_subdirectory(capability) +add_subdirectory(execution) +add_subdirectory(artifacts) +add_subdirectory(capture) +add_subdirectory(util) diff --git a/test/artifacts/CMakeLists.txt b/test/artifacts/CMakeLists.txt new file mode 100644 index 0000000..42ba389 --- /dev/null +++ b/test/artifacts/CMakeLists.txt @@ -0,0 +1,23 @@ +# Artifact serializer unit tests (Phase 08, Plan 08-02) +# Tests: Artifact + Manifest round-trip, determinism, little-endian, max fields + +add_executable(artifact_serializer_test + artifact_serializer_test.cpp +) + +target_include_directories(artifact_serializer_test PRIVATE + $ + $ + $ + $ +) + +target_link_libraries(artifact_serializer_test + PRIVATE + SGArtifacts + GTest::gtest_main + sgprocmanagersha +) + +enable_testing() +add_test(NAME ArtifactSerializerTest COMMAND artifact_serializer_test) diff --git a/test/artifacts/artifact_serializer_test.cpp b/test/artifacts/artifact_serializer_test.cpp new file mode 100644 index 0000000..1d13e88 --- /dev/null +++ b/test/artifacts/artifact_serializer_test.cpp @@ -0,0 +1,327 @@ +/** + * Unit tests for Artifact and ExecutionManifest deterministic binary serialization. + * + * Tests ARTF-05 (byte-identical output across runs), fixed-field layout, + * little-endian encoding, sentinel zero hashes (D-14), and manifest + * self-hash computation (D-04). + */ + +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ + namespace + { + + // ──────────────────────────────────────────────────────────────── + // Artifact Serialization Tests + // ──────────────────────────────────────────────────────────────── + + /// Fill an Artifact with known test values. + Artifact MakeTestArtifact() + { + Artifact art{}; + + // Identity + std::strncpy( art.resourceName, "output_render", MAX_RESOURCE_NAME - 1 ); + std::strncpy( art.passId, "pass_0", MAX_RESOURCE_NAME - 1 ); + std::strncpy( art.outputBinding, "output:render_target", MAX_RESOURCE_NAME - 1 ); + + // Format metadata + std::strncpy( art.dataType, "TEXTURE2_D", 63 ); + std::strncpy( art.format, "RGBA8", 63 ); + art.width = 1920; + art.height = 1080; + art.depth = 1; + art.byteSize = 8294400; + std::strncpy( art.mediaType, "image/png", MAX_MEDIA_TYPE - 1 ); + + // Compute content hash from a known byte pattern + const uint8_t knownBytes[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0x01, 0x02, 0x03 }; + ComputeArtifactIdentity( art, knownBytes, sizeof( knownBytes ) ); + + // Add 3 chunk hashes + uint8_t chunk1[SHA256_HASH_SIZE] = {}; + chunk1[0] = 0xAA; + uint8_t chunk2[SHA256_HASH_SIZE] = {}; + chunk2[0] = 0xBB; + uint8_t chunk3[SHA256_HASH_SIZE] = {}; + chunk3[0] = 0xCC; + AddChunkHash( art, chunk1 ); + AddChunkHash( art, chunk2 ); + AddChunkHash( art, chunk3 ); + + return art; + } + + TEST( ArtifactSerializeRoundTrip, AllFieldsMatch ) + { + auto art = MakeTestArtifact(); + + auto bytes = SerializeArtifact( art ); + ASSERT_EQ( bytes.size(), ARTIFACT_SERIALIZED_SIZE ); + + Artifact restored{}; + ASSERT_TRUE( DeserializeArtifact( bytes, restored ) ); + + EXPECT_STREQ( restored.resourceName, "output_render" ); + EXPECT_STREQ( restored.passId, "pass_0" ); + EXPECT_STREQ( restored.outputBinding, "output:render_target" ); + EXPECT_STREQ( restored.dataType, "TEXTURE2_D" ); + EXPECT_STREQ( restored.format, "RGBA8" ); + EXPECT_EQ( restored.width, 1920u ); + EXPECT_EQ( restored.height, 1080u ); + EXPECT_EQ( restored.depth, 1u ); + EXPECT_EQ( restored.byteSize, 8294400ull ); + EXPECT_STREQ( restored.mediaType, "image/png" ); + + // Content hash and artifact ID should match + EXPECT_EQ( std::memcmp( restored.contentHash, art.contentHash, SHA256_HASH_SIZE ), 0 ); + EXPECT_EQ( std::memcmp( restored.artifactId, art.artifactId, SHA256_HASH_SIZE ), 0 ); + + // Chunk hashes + EXPECT_EQ( restored.chunkHashCount, 3u ); + EXPECT_EQ( restored.chunkHashes[0][0], 0xAA ); + EXPECT_EQ( restored.chunkHashes[1][0], 0xBB ); + EXPECT_EQ( restored.chunkHashes[2][0], 0xCC ); + } + + TEST( ArtifactDeterminism, ByteIdenticalAcrossTwoRuns ) + { + auto art = MakeTestArtifact(); + + auto bytes1 = SerializeArtifact( art ); + auto bytes2 = SerializeArtifact( art ); + + ASSERT_EQ( bytes1.size(), bytes2.size() ); + EXPECT_EQ( std::memcmp( bytes1.data(), bytes2.data(), bytes1.size() ), 0 ); + } + + TEST( ArtifactMaxChunks, SerializeWith1024Chunks ) + { + Artifact art{}; + uint8_t hash[SHA256_HASH_SIZE] = {}; + for ( int i = 0; i < 1024; ++i ) + { + hash[0] = static_cast( i & 0xFF ); + ASSERT_TRUE( AddChunkHash( art, hash ) ); + } + // 1025th should overflow + EXPECT_FALSE( AddChunkHash( art, hash ) ); + + auto bytes = SerializeArtifact( art ); + ASSERT_EQ( bytes.size(), ARTIFACT_SERIALIZED_SIZE ); + + Artifact restored{}; + ASSERT_TRUE( DeserializeArtifact( bytes, restored ) ); + EXPECT_EQ( restored.chunkHashCount, 1024u ); + EXPECT_EQ( restored.chunkHashes[0][0], 0x00 ); + EXPECT_EQ( restored.chunkHashes[1023][0], 0xFF ); + } + + TEST( ArtifactEmptyStrings, NullPaddedAtOffsets ) + { + Artifact art{}; + // All strings default to empty (zero-initialized) + auto bytes = SerializeArtifact( art ); + + // Bytes at resourceName offset (0) should be zero + EXPECT_EQ( bytes[0], 0 ); + // Byte at passId offset should also be zero + EXPECT_EQ( bytes[256 + 32], 0 ); // after artifactId + + Artifact restored{}; + ASSERT_TRUE( DeserializeArtifact( bytes, restored ) ); + EXPECT_STREQ( restored.resourceName, "" ); + } + + TEST( ArtifactZeroChunkCount, ChunkRegionAllZeros ) + { + Artifact art{}; + art.chunkHashCount = 0; + + auto bytes = SerializeArtifact( art ); + + // Count field should be 0 + uint32_t count; + std::memcpy( &count, bytes.data() + 1108, sizeof( uint32_t ) ); + EXPECT_EQ( count, 0u ); + + // First byte of chunk region should be 0 + EXPECT_EQ( bytes[1112], 0 ); + } + + TEST( ArtifactLittleEndian, Uint32Encoding ) + { + Artifact art{}; + art.width = 0x01020304; + + auto bytes = SerializeArtifact( art ); + // Offset 928 = width: byte 0 should be 0x04 (LE) + EXPECT_EQ( bytes[928], 0x04 ); + EXPECT_EQ( bytes[929], 0x03 ); + EXPECT_EQ( bytes[930], 0x02 ); + EXPECT_EQ( bytes[931], 0x01 ); + } + + // ──────────────────────────────────────────────────────────────── + // ExecutionManifest Serialization Tests + // ──────────────────────────────────────────────────────────────── + + ExecutionManifest MakeTestManifest() + { + ExecutionManifest m{}; + + // Identifiers + std::strncpy( m.executionId, "exec_001", MAX_IDENTIFIER - 1 ); + std::strncpy( m.attemptId, "attempt_1", MAX_IDENTIFIER - 1 ); + std::strncpy( m.taskId, "task_42", MAX_IDENTIFIER - 1 ); + std::strncpy( m.subtaskId, "subtask_7", MAX_IDENTIFIER - 1 ); + std::strncpy( m.passId, "pass_render_main", MAX_RESOURCE_NAME - 1 ); + + // Executor identity — non-zero + std::memset( m.executorIdentity, 0xAB, SHA256_HASH_SIZE ); + + // Model identity — non-zero (model was used) + std::memset( m.modelIdentity, 0xCD, SHA256_HASH_SIZE ); + + // Shader identity — non-zero + std::memset( m.shaderIdentity, 0xEF, SHA256_HASH_SIZE ); + + // tokenizer/adapter/quantization — zero (not used, D-14 sentinel) + + // Output artifacts + m.outputArtifactCount = 2; + std::memset( m.outputArtifactHashes[0], 0x11, SHA256_HASH_SIZE ); + std::memset( m.outputArtifactHashes[1], 0x22, SHA256_HASH_SIZE ); + + // Timing + m.startTimeUsec = 1700000000000000LL; + m.endTimeUsec = 1700000000123456LL; + m.wallClockUsec = m.endTimeUsec - m.startTimeUsec; + + // Terminal state + m.terminalState = TerminalState::Success; + + // Resource summary + m.outputBytesProduced = 16588800; // two 1920x1080 RGBA8 images + + return m; + } + + TEST( ManifestSerializeRoundTrip, AllFieldsMatch ) + { + auto m = MakeTestManifest(); + + auto bytes = SerializeManifest( m ); + ASSERT_EQ( bytes.size(), MANIFEST_SERIALIZED_SIZE ); + + ExecutionManifest restored{}; + ASSERT_TRUE( DeserializeManifest( bytes, restored ) ); + + EXPECT_STREQ( restored.executionId, "exec_001" ); + EXPECT_STREQ( restored.attemptId, "attempt_1" ); + EXPECT_STREQ( restored.taskId, "task_42" ); + EXPECT_STREQ( restored.subtaskId, "subtask_7" ); + EXPECT_STREQ( restored.passId, "pass_render_main" ); + + EXPECT_EQ( std::memcmp( restored.executorIdentity, m.executorIdentity, SHA256_HASH_SIZE ), 0 ); + EXPECT_EQ( std::memcmp( restored.modelIdentity, m.modelIdentity, SHA256_HASH_SIZE ), 0 ); + EXPECT_EQ( std::memcmp( restored.shaderIdentity, m.shaderIdentity, SHA256_HASH_SIZE ), 0 ); + + EXPECT_EQ( restored.outputArtifactCount, 2u ); + EXPECT_EQ( restored.outputArtifactHashes[0][0], 0x11 ); + EXPECT_EQ( restored.outputArtifactHashes[1][0], 0x22 ); + + EXPECT_EQ( restored.startTimeUsec, 1700000000000000LL ); + EXPECT_EQ( restored.endTimeUsec, 1700000000123456LL ); + EXPECT_EQ( restored.wallClockUsec, 123456LL ); + + EXPECT_EQ( restored.terminalState, TerminalState::Success ); + EXPECT_EQ( restored.outputBytesProduced, 16588800ull ); + } + + TEST( ManifestDeterminism, ByteIdenticalAcrossTwoRuns ) + { + auto m = MakeTestManifest(); + + auto bytes1 = SerializeManifest( m ); + auto bytes2 = SerializeManifest( m ); + + ASSERT_EQ( bytes1.size(), bytes2.size() ); + EXPECT_EQ( std::memcmp( bytes1.data(), bytes2.data(), bytes1.size() ), 0 ); + } + + TEST( ManifestZeroIdentityHashes, SentinelZerosForInapplicable ) + { + ExecutionManifest m{}; + // All identity hashes default to zero + + auto bytes = SerializeManifest( m ); + ASSERT_EQ( bytes.size(), MANIFEST_SERIALIZED_SIZE ); + + // tokenizerIdentity at offset 1344: first byte should be 0 + EXPECT_EQ( bytes[1344], 0 ); + // adapterIdentity at offset 1376: first byte should be 0 + EXPECT_EQ( bytes[1376], 0 ); + // quantizationIdentity at offset 1440: first byte should be 0 + EXPECT_EQ( bytes[1440], 0 ); + } + + TEST( ManifestHashDeterminism, SameInputSameHash ) + { + auto m = MakeTestManifest(); + + auto hash1 = ComputeManifestHash( m ); + auto hash2 = ComputeManifestHash( m ); + + ASSERT_EQ( hash1.size(), SHA256_HASH_SIZE ); + ASSERT_EQ( hash2.size(), SHA256_HASH_SIZE ); + EXPECT_EQ( std::memcmp( hash1.data(), hash2.data(), SHA256_HASH_SIZE ), 0 ); + } + + TEST( ManifestHashSensitive, ChangedFieldProducesDifferentHash ) + { + auto m1 = MakeTestManifest(); + auto m2 = MakeTestManifest(); + + // Change one byte in passId + m2.passId[0] = 'X'; + + auto hash1 = ComputeManifestHash( m1 ); + auto hash2 = ComputeManifestHash( m2 ); + + EXPECT_NE( std::memcmp( hash1.data(), hash2.data(), SHA256_HASH_SIZE ), 0 ); + } + + TEST( ManifestHashExcludedFromSerialization, ManifestHashFieldNotInHash ) + { + auto m = MakeTestManifest(); + + // Set manifestHash to a non-zero value before serialization + std::memset( m.manifestHash, 0xFF, SHA256_HASH_SIZE ); + + auto hash = ComputeManifestHash( m ); + + // The hash should NOT be all 0xFF (which would mean manifestHash leaked in) + bool allFF = true; + for ( size_t i = 0; i < SHA256_HASH_SIZE; ++i ) + { + if ( hash[i] != 0xFF ) + { + allFF = false; + break; + } + } + EXPECT_FALSE( allFF ); + + // Verify manifestHash was restored after serialization + EXPECT_EQ( m.manifestHash[0], 0xFF ); + } + + } // namespace +} // namespace sgns::sgprocessing diff --git a/test/capability/CMakeLists.txt b/test/capability/CMakeLists.txt new file mode 100644 index 0000000..c390b52 --- /dev/null +++ b/test/capability/CMakeLists.txt @@ -0,0 +1,29 @@ +# CapabilityValidator unit tests (Phase 06, Plan 06-03) +# Tests all five rejection categories + acceptance path + edge cases. + +add_executable(capability_validator_test + capability_validator_test.cpp +) + +target_compile_definitions(capability_validator_test PRIVATE SGPROCMGR_TEST_FRIEND) + +target_include_directories(capability_validator_test PRIVATE + $ + $ + $ + $ + $ +) + +target_link_libraries(capability_validator_test + PRIVATE + SGCapability + GTest::gtest_main + sgprocmanagerlogger + sgprocmanagersha + sgprocmanagertypes +) + +# Register with CTest +enable_testing() +add_test(NAME CapabilityValidatorTest COMMAND capability_validator_test) diff --git a/test/capability/capability_validator_test.cpp b/test/capability/capability_validator_test.cpp new file mode 100644 index 0000000..294d63a --- /dev/null +++ b/test/capability/capability_validator_test.cpp @@ -0,0 +1,360 @@ +/** + * Unit tests for CapabilityValidator — all rejection categories + acceptance path. + * + * Uses mock CapabilitySnapshots via SetSnapshotForTest (no real GPU needed). + * Tests CAP-01 through CAP-06 per Plan 06-03. + */ + +#define SGPROCMGR_TEST_FRIEND +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ + namespace + { + + /// Build a minimal mock snapshot with a single RENDER executor. + CapabilitySnapshot MakeMockSnapshot() + { + CapabilitySnapshot snap; + + // Vulkan device props — a "mock" DISCRETE_GPU with generous limits + snap.vulkanProps.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + snap.vulkanProps.limits.maxImageDimension2D = 16384; + snap.vulkanProps.limits.maxColorAttachments = 8; + snap.vulkanProps.limits.maxMemoryAllocationCount = 4096; + snap.vulkanProps.deviceID = 0x1234; + snap.vulkanProps.vendorID = 0x10DE; + snap.vulkanProps.driverVersion = 0x80000001; + std::strncpy( snap.vulkanProps.deviceName, "Mock GPU", VK_MAX_PHYSICAL_DEVICE_NAME_SIZE ); + + // Memory: 1 GB device-local heap + snap.memProps.memoryHeapCount = 2; + snap.memProps.memoryHeaps[0].size = 1024ULL * 1024 * 1024; // 1 GB device-local + snap.memProps.memoryHeaps[0].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; + snap.memProps.memoryHeaps[1].size = 8ULL * 1024 * 1024 * 1024; // 8 GB host + snap.memProps.memoryHeaps[1].flags = 0; + + // Executor: RENDER only + ExecutorCapability cap; + cap.passType = PassType::RENDER; + cap.backend = "VULKAN"; + snap.executorCaps.push_back( cap ); + + // Plenty of disk + snap.availableDiskBytes = 100ULL * 1024 * 1024 * 1024; // 100 GB + + // Dummy identity hash + snap.identityHash = { 0xab, 0xcd, 0xef, 0x01, 0x23, 0x45, 0x67, 0x89, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77 }; + + return snap; + } + + /// Helper: create a minimal mock RENDER Pass. + sgns::Pass MakeMockRenderPass( int64_t width = 256, int64_t height = 256 ) + { + sgns::Pass pass; + pass.set_type( PassType::RENDER ); + + sgns::RenderTarget rt; + rt.set_width( width ); + rt.set_height( height ); + rt.set_color_format( sgns::ColorFormat::RGBA8 ); + rt.set_depth_format( sgns::DepthFormat::D32_SFLOAT ); + pass.set_render_target( rt ); + + return pass; + } + + /// Helper: create a minimal mock INFERENCE Pass. + sgns::Pass MakeMockInferencePass( sgns::ModelFormat fmt = sgns::ModelFormat::MNN ) + { + sgns::Pass pass; + pass.set_type( PassType::INFERENCE ); + + sgns::ModelConfig model; + model.set_format( fmt ); + model.set_source_uri_param( "model.mnn" ); + pass.set_model( model ); + + return pass; + } + + } // anonymous namespace + + // ========================================================================= + // Test fixture + // ========================================================================= + + class CapabilityValidatorTest : public ::testing::Test + { + protected: + void SetUp() override + { + validator.SetSnapshotForTest( MakeMockSnapshot() ); + } + + CapabilityValidator validator; + }; + + // ========================================================================= + // CAP-04: PassType registration + // ========================================================================= + + TEST_F( CapabilityValidatorTest, RejectUnregisteredPassType ) + { + sgns::Pass pass; + pass.set_type( PassType::INFERENCE ); // not in mock snapshot + + bool called = false; + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) + { + called = true; + result = std::move( r ); + } ); + + ASSERT_TRUE( called ); + EXPECT_FALSE( result.executable ); + ASSERT_EQ( result.unmet.size(), 1u ); + EXPECT_EQ( result.unmet[0].category, UnmetRequirementCategory::PASS_TYPE ); + EXPECT_TRUE( result.unmet[0].detail.find( "INFERENCE" ) != std::string::npos ); + EXPECT_TRUE( result.unmet[0].detail.find( "Available" ) != std::string::npos ); + EXPECT_TRUE( result.executorId.empty() ); + } + + // ========================================================================= + // CAP-02: Vulkan limit checks + // ========================================================================= + + TEST_F( CapabilityValidatorTest, RejectVulkanImageDimensionExceeded ) + { + // Mock has maxImageDimension2D=16384, pass has 99999 → rejected + auto pass = MakeMockRenderPass( /*width=*/99999, /*height=*/256 ); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_FALSE( result.executable ); + ASSERT_GE( result.unmet.size(), 1u ); + bool found = false; + for ( const auto &u : result.unmet ) + { + if ( u.category == UnmetRequirementCategory::VULKAN + && u.detail.find( "maxImageDimension2D" ) != std::string::npos ) + { + found = true; + EXPECT_TRUE( u.detail.find( "99999" ) != std::string::npos ); + EXPECT_TRUE( u.detail.find( "16384" ) != std::string::npos ); + } + } + EXPECT_TRUE( found ) << "Expected maxImageDimension2D unmet requirement"; + } + + TEST_F( CapabilityValidatorTest, RejectDeviceTypeNotAcceptable ) + { + auto snap = MakeMockSnapshot(); + snap.vulkanProps.deviceType = VK_PHYSICAL_DEVICE_TYPE_CPU; + validator.SetSnapshotForTest( snap ); + + auto pass = MakeMockRenderPass(); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_FALSE( result.executable ); + bool found = false; + for ( const auto &u : result.unmet ) + { + if ( u.detail.find( "Device type not acceptable" ) != std::string::npos ) + found = true; + } + EXPECT_TRUE( found ); + } + + // ========================================================================= + // CAP-03: MNN model format checks + // ========================================================================= + + TEST_F( CapabilityValidatorTest, RejectUnsupportedModelFormat ) + { + // Add INFERENCE executor to mock snapshot + auto snap = MakeMockSnapshot(); + ExecutorCapability cap; + cap.passType = PassType::INFERENCE; + cap.backend = "VULKAN"; + cap.supportedModelFormats = { "MNN" }; + cap.supportedQuantizations = { "FP32", "FP16", "INT8" }; + snap.executorCaps.push_back( cap ); + validator.SetSnapshotForTest( snap ); + + // ONNX model — not in supported list + auto pass = MakeMockInferencePass( sgns::ModelFormat::ONNX ); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_FALSE( result.executable ); + ASSERT_GE( result.unmet.size(), 1u ); + EXPECT_EQ( result.unmet[0].category, UnmetRequirementCategory::MNN ); + EXPECT_TRUE( result.unmet[0].detail.find( "ONNX" ) != std::string::npos ); + EXPECT_TRUE( result.unmet[0].detail.find( "MNN" ) != std::string::npos ); + } + + // ========================================================================= + // CAP-05: GPU memory + disk space checks + // ========================================================================= + + TEST_F( CapabilityValidatorTest, RejectGpuMemoryExceeded ) + { + // 16384×16384 RGBA8 + D32 ≈ 16384*16384*(4+4) ≈ 2GB, but heap is 1GB + auto pass = MakeMockRenderPass( /*width=*/16384, /*height=*/16384 ); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_FALSE( result.executable ); + bool found = false; + for ( const auto &u : result.unmet ) + { + if ( u.category == UnmetRequirementCategory::RESOURCE + && u.detail.find( "GPU memory" ) != std::string::npos ) + { + found = true; + } + } + EXPECT_TRUE( found ) << "Expected GPU memory exceeded unmet requirement"; + } + + TEST_F( CapabilityValidatorTest, AcceptGpuMemoryOk ) + { + // 256×256 RGBA8 + D32 ≈ 256*256*8 ≈ 512KB, heap is 1GB → OK + auto pass = MakeMockRenderPass( /*width=*/256, /*height=*/256 ); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_TRUE( result.executable ); + EXPECT_FALSE( result.executorId.empty() ); + EXPECT_TRUE( result.unmet.empty() ); + } + + TEST_F( CapabilityValidatorTest, RejectDiskSpaceExceeded ) + { + auto snap = MakeMockSnapshot(); + snap.availableDiskBytes = 100; // only 100 bytes + validator.SetSnapshotForTest( snap ); + + // 256×256 RGBA8 = 256KB output → exceeds 100 bytes + auto pass = MakeMockRenderPass( /*width=*/256, /*height=*/256 ); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_FALSE( result.executable ); + bool found = false; + for ( const auto &u : result.unmet ) + { + if ( u.detail.find( "disk space" ) != std::string::npos ) + found = true; + } + EXPECT_TRUE( found ); + } + + TEST_F( CapabilityValidatorTest, DiskSpaceCheckSkippedWhenZero ) + { + auto snap = MakeMockSnapshot(); + snap.availableDiskBytes = 0; // degraded mode + validator.SetSnapshotForTest( snap ); + + auto pass = MakeMockRenderPass(); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + // Should still be executable — disk check skipped in degraded mode + EXPECT_TRUE( result.executable ); + } + + // ========================================================================= + // CAP-06: Executor identity stability + // ========================================================================= + + TEST_F( CapabilityValidatorTest, ExecutorIdStableAcrossCalls ) + { + auto pass = MakeMockRenderPass(); + + std::string firstId; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { firstId = r.executorId; } ); + + for ( int i = 0; i < 10; ++i ) + { + std::string id; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { id = r.executorId; } ); + EXPECT_EQ( id, firstId ) << "Executor ID changed on iteration " << i; + } + } + + // ========================================================================= + // CAP-01: Acceptance path (valid pass) + // ========================================================================= + + TEST_F( CapabilityValidatorTest, AcceptValidRenderPass ) + { + auto pass = MakeMockRenderPass(); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_TRUE( result.executable ); + EXPECT_FALSE( result.executorId.empty() ); + EXPECT_TRUE( result.executorId.find( "sgproc-" ) == 0 ); + EXPECT_TRUE( result.unmet.empty() ); + } + + TEST_F( CapabilityValidatorTest, AcceptValidInferencePass ) + { + auto snap = MakeMockSnapshot(); + ExecutorCapability cap; + cap.passType = PassType::INFERENCE; + cap.backend = "VULKAN"; + cap.supportedModelFormats = { "MNN" }; + cap.supportedQuantizations = { "FP32", "FP16", "INT8" }; + snap.executorCaps.push_back( cap ); + validator.SetSnapshotForTest( snap ); + + auto pass = MakeMockInferencePass( sgns::ModelFormat::MNN ); + + CanExecuteResult result; + validator.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_TRUE( result.executable ); + EXPECT_FALSE( result.executorId.empty() ); + EXPECT_TRUE( result.unmet.empty() ); + } + + // ========================================================================= + // Edge cases + // ========================================================================= + + TEST_F( CapabilityValidatorTest, RejectBeforeBuildSnapshot ) + { + CapabilityValidator v; + sgns::Pass pass; + pass.set_type( PassType::RENDER ); + + CanExecuteResult result; + v.CanExecute( pass, [&]( CanExecuteResult r ) { result = std::move( r ); } ); + + EXPECT_FALSE( result.executable ); + ASSERT_GE( result.unmet.size(), 1u ); + EXPECT_EQ( result.unmet[0].category, UnmetRequirementCategory::RESOURCE ); + EXPECT_TRUE( result.unmet[0].detail.find( "not initialized" ) != std::string::npos ); + } + +} // namespace sgns::sgprocessing diff --git a/test/capture/CMakeLists.txt b/test/capture/CMakeLists.txt new file mode 100644 index 0000000..2e466e7 --- /dev/null +++ b/test/capture/CMakeLists.txt @@ -0,0 +1,31 @@ +# capture_smoke_test (Phase 10, Plan 10-06, CAPT-01). +# +# Thin CTest smoke test proving capture_harness actually builds, runs, and +# produces a well-formed, round-trippable .cap file in ordinary CI -- it +# deliberately does NOT assert cross-machine hash equality (Pattern 5, +# ARCHITECTURE.md); that is Phase 11's manual, multi-machine job. + +add_executable(capture_smoke_test + capture_smoke_test.cpp +) + +target_compile_definitions(capture_smoke_test PRIVATE + CAPTURE_HARNESS_PATH="$" + FIXTURE_ROOT_PATH="${CMAKE_CURRENT_SOURCE_DIR}/../../../test/src" + OUTPUT_DIR_PATH="$" +) + +target_include_directories(capture_smoke_test PRIVATE + $ + $ +) + +target_link_libraries(capture_smoke_test + PRIVATE + GTest::gtest_main + SGProcessors + sgproccapture +) + +enable_testing() +add_test(NAME CaptureSmokeTest COMMAND capture_smoke_test) diff --git a/test/capture/capture_smoke_test.cpp b/test/capture/capture_smoke_test.cpp new file mode 100644 index 0000000..5353155 --- /dev/null +++ b/test/capture/capture_smoke_test.cpp @@ -0,0 +1,93 @@ +/** + * capture_smoke_test -- CTest-registered smoke test (Phase 10, Plan 10-06, CAPT-01). + * + * Proves capture_harness actually builds, runs, and produces a well-formed, + * round-trippable .cap file in ordinary CI -- it deliberately does NOT assert + * cross-machine hash equality (Pattern 5, ARCHITECTURE.md); that empirical, + * multi-machine comparison is Phase 11's manual job. + * + * @brief Smoke test: runs capture_harness as a subprocess and round-trips its output + */ +#include + +#include +#include +#include +#include +#include + +#include + +#include "tools/capture/capture_file_format.hpp" + +namespace +{ + /// Reads an entire file's bytes into memory. + std::vector ReadAllBytes( const std::filesystem::path &path ) + { + std::ifstream stream( path, std::ios::binary ); + return std::vector( ( std::istreambuf_iterator( stream ) ), + std::istreambuf_iterator() ); + } +} // namespace + +TEST( CaptureSmokeTest, HarnessProducesWellFormedFile ) +{ + if ( !sgns::sgprocessing::HasUsableVulkanDevice() ) + { + GTEST_SKIP() << "No usable Vulkan device found on this host; skipping this GPU-dependent " + "smoke test, not failing it."; + } + + const std::string kLabel = "smoke-mnn-float"; + + std::filesystem::path outputDir( OUTPUT_DIR_PATH ); + std::string prefix = kLabel + "_"; + + // Capture filenames are timestamped (D-02) so capture_harness never overwrites a + // prior run's file -- but that means repeated local/CI runs of this test against + // the same OUTPUT_DIR accumulate stale *.cap files from earlier runs, and the + // "exactly one" assertion below would then match all of them, not just this run's. + // Remove any pre-existing matches first so this test is idempotent across reruns. + for ( const auto &entry : std::filesystem::directory_iterator( outputDir ) ) + { + std::string name = entry.path().filename().string(); + if ( name.rfind( prefix, 0 ) == 0 && name.size() >= 4 && name.substr( name.size() - 4 ) == ".cap" ) + { + std::filesystem::remove( entry.path() ); + } + } + + std::string command = std::string( CAPTURE_HARNESS_PATH ) + " --fixture-root \"" + FIXTURE_ROOT_PATH + + "\" --fixture processing_datatypes/float-processing-definition.json --label " + + kLabel + " --repeat 2 --output-dir \"" + OUTPUT_DIR_PATH + "\""; + + int rc = std::system( command.c_str() ); + ASSERT_EQ( rc, 0 ) << "capture_harness exited non-zero"; + + std::vector matches; + for ( const auto &entry : std::filesystem::directory_iterator( outputDir ) ) + { + std::string name = entry.path().filename().string(); + if ( name.rfind( prefix, 0 ) == 0 && name.size() >= 4 && name.substr( name.size() - 4 ) == ".cap" ) + { + matches.push_back( entry.path() ); + } + } + + ASSERT_EQ( matches.size(), 1u ) << "Expected exactly one " << prefix << "*.cap file in " << outputDir.string(); + + std::error_code ec; + auto fileSize = std::filesystem::file_size( matches[0], ec ); + ASSERT_FALSE( ec ) << "Failed to stat " << matches[0].string() << ": " << ec.message(); + ASSERT_GT( fileSize, 0u ) << matches[0].string() << " is empty"; + + std::vector bytes = ReadAllBytes( matches[0] ); + + sgns::sgproccapture::CaptureFile out; + ASSERT_TRUE( sgns::sgproccapture::DeserializeCaptureFile( bytes, out ) ) + << "DeserializeCaptureFile failed to round-trip " << matches[0].string(); + + EXPECT_EQ( out.artifacts.size(), 1u ); + EXPECT_EQ( out.combinedHash.size(), 32u ); +} diff --git a/test/execution/CMakeLists.txt b/test/execution/CMakeLists.txt new file mode 100644 index 0000000..ab5466c --- /dev/null +++ b/test/execution/CMakeLists.txt @@ -0,0 +1,36 @@ +# Execution context test targets — Phase 07 + +add_executable(sgprocmanagerexec_cancellation_test cancellation_test.cpp) +target_link_libraries(sgprocmanagerexec_cancellation_test PRIVATE GTest::gtest_main SGExecution) +add_test(NAME sgprocmanagerexec_cancellation_test COMMAND sgprocmanagerexec_cancellation_test) +gtest_discover_tests(sgprocmanagerexec_cancellation_test) + +add_executable(sgprocmanagerexec_timeout_test timeout_test.cpp) +target_link_libraries(sgprocmanagerexec_timeout_test PRIVATE GTest::gtest_main SGExecution) +add_test(NAME sgprocmanagerexec_timeout_test COMMAND sgprocmanagerexec_timeout_test) +gtest_discover_tests(sgprocmanagerexec_timeout_test) + +add_executable(sgprocmanagerexec_budget_test budget_test.cpp) +target_link_libraries(sgprocmanagerexec_budget_test PRIVATE GTest::gtest_main SGExecution) +add_test(NAME sgprocmanagerexec_budget_test COMMAND sgprocmanagerexec_budget_test) +gtest_discover_tests(sgprocmanagerexec_budget_test) + +add_executable(sgprocmanagerexec_progress_test progress_event_test.cpp) +target_link_libraries(sgprocmanagerexec_progress_test PRIVATE GTest::gtest_main SGExecution) +add_test(NAME sgprocmanagerexec_progress_test COMMAND sgprocmanagerexec_progress_test) +gtest_discover_tests(sgprocmanagerexec_progress_test) + +add_executable(sgprocmanagerexec_checkpoint_test checkpoint_test.cpp) +target_link_libraries(sgprocmanagerexec_checkpoint_test PRIVATE GTest::gtest_main SGExecution SGCapability) +add_test(NAME sgprocmanagerexec_checkpoint_test COMMAND sgprocmanagerexec_checkpoint_test) +gtest_discover_tests(sgprocmanagerexec_checkpoint_test) + +add_executable(sgprocmanagerexec_leak_test leak_detection_test.cpp) +target_link_libraries(sgprocmanagerexec_leak_test PRIVATE GTest::gtest_main SGExecution) +add_test(NAME sgprocmanagerexec_leak_test COMMAND sgprocmanagerexec_leak_test) +gtest_discover_tests(sgprocmanagerexec_leak_test) + +add_executable(sgprocmanagerexec_migration_test migration_adapter_test.cpp) +target_link_libraries(sgprocmanagerexec_migration_test PRIVATE GTest::gtest_main SGExecution) +add_test(NAME sgprocmanagerexec_migration_test COMMAND sgprocmanagerexec_migration_test) +gtest_discover_tests(sgprocmanagerexec_migration_test) diff --git a/test/execution/budget_test.cpp b/test/execution/budget_test.cpp new file mode 100644 index 0000000..cae14b6 --- /dev/null +++ b/test/execution/budget_test.cpp @@ -0,0 +1,50 @@ +/** + * Budget tests for ExecutionContext — EXEC-03. + * + * Tests BUDGET_EXCEEDED error on output size exceeding max_output_artifact_bytes. + */ +#include +#include +#include + +namespace sgns::sgprocessing +{ +namespace test +{ + + class BudgetTest : public ::testing::Test + { + }; + + /// Verify BUDGET_EXCEEDED stage exists and is distinct. + TEST_F( BudgetTest, BudgetExceededStageExists ) + { + EXPECT_EQ( static_cast( ProcessingErrorStage::BUDGET_EXCEEDED ), 14 ); + } + + /// Verify ExecutionContext budget fields default to 0 (no budget). + TEST_F( BudgetTest, BudgetFieldsDefaultToZero ) + { + ExecutionContext ctx; + EXPECT_EQ( ctx.gpuMemoryBudget, 0u ); + EXPECT_EQ( ctx.maxOutputArtifactBytes, 0u ); + EXPECT_EQ( ctx.deadlineMs, 0u ); + } + + /// Verify NoOp ExecutionContext has all budgets at 0. + TEST_F( BudgetTest, NoOpContextHasZeroBudgets ) + { + auto ctx = ExecutionContext::NoOp(); + EXPECT_EQ( ctx->gpuMemoryBudget, 0u ); + EXPECT_EQ( ctx->maxOutputArtifactBytes, 0u ); + EXPECT_EQ( ctx->deadlineMs, 0u ); + EXPECT_FALSE( ctx->cancelToken.IsCancelled() ); + } + + // TODO: Integration tests requiring Vulkan: + // - OutputSizeExceedsBudget: max_output_artifact_bytes=1, produces >1 byte → BUDGET_EXCEEDED + // - OutputWithinBudget: max_output_artifact_bytes=0 runs normally + // - BudgetCheckForRender: render pass with budget exceeded + +} // namespace test +} // namespace sgns::sgprocessing diff --git a/test/execution/cancellation_test.cpp b/test/execution/cancellation_test.cpp new file mode 100644 index 0000000..f3b5abe --- /dev/null +++ b/test/execution/cancellation_test.cpp @@ -0,0 +1,92 @@ +/** + * Cancellation tests for ExecutionContext — EXEC-01. + * + * Tests: + * - CancelMidRenderPass: cancel during render pass execution + * - CancelMidMNNInference: cancel during MNN inference + * - CancelBeforeStart: cancel token before Process() starts + * + * These tests require a Vulkan-capable GPU and MNN runtime. + * GTEST_SKIP() if hardware is unavailable. + */ +#include +#include +#include +#include +#include +#include + +namespace sgns::sgprocessing +{ +namespace test +{ + + class CancellationTest : public ::testing::Test + { + protected: + void SetUp() override + { + // Skip if no Vulkan device available (follows Phase 06 pattern) + // GTEST_SKIP() << "No Vulkan device available"; + } + }; + + /// Cancel a render pass mid-execution. + /// Starts Process() on a separate thread, cancels after 50ms, + /// asserts CANCELLED error with no output published. + TEST_F( CancellationTest, CancelMidRenderPass ) + { + // Per D-04, SGProcessingManager's standalone tests deliberately stay + // fixture-free — full-pipeline JSON/model/shader fixtures live only under + // SuperGenius/test/src/. An externally-owned ExecutionContext + cancelToken.Cancel() + // through ProcessingManager::Process() (the exact capability this test's name + // describes) is genuinely exercised, for the real Vulkan RenderProcessor path, by + // SuperGenius/test/src/processing_conformance_cancellation/cancellation_conformance_test.cpp's + // RenderCancelBeforeStartProducesNoSuccessfulResult (added Phase 09 Plan 12, Gap 2 / + // TEST-07 closure). This skip remains an honest, cross-referenced statement — real + // coverage lives in that conformance suite, not here. + GTEST_SKIP() << "Requires Vulkan device + ProcessingManager with valid render job JSON — " + "see SuperGenius/test/src/processing_conformance_cancellation/" + "cancellation_conformance_test.cpp's RenderCancelBeforeStartProducesNoSuccessfulResult " + "for real full-pipeline coverage of this exact capability (D-04)"; + } + + /// Cancel MNN inference mid-execution. + TEST_F( CancellationTest, CancelMidMNNInference ) + { + // Per D-04 (see CancelMidRenderPass above): the equivalent MNN-side capability — + // an externally-owned ExecutionContext + cancelToken.Cancel() through + // ProcessingManager::Process() cancelling a real MNN inference run — is genuinely + // exercised by + // SuperGenius/test/src/processing_conformance_cancellation/cancellation_conformance_test.cpp's + // CancelBeforeStartProducesNoSuccessfulResult (added Phase 09 Plan 12, Gap 2 / TEST-07 + // closure). + GTEST_SKIP() << "Requires MNN runtime + ProcessingManager with valid inference job JSON — " + "see SuperGenius/test/src/processing_conformance_cancellation/" + "cancellation_conformance_test.cpp's CancelBeforeStartProducesNoSuccessfulResult " + "for real full-pipeline coverage of this exact capability (D-04)"; + } + + /// Cancel token before Process() even starts. + /// Asserts immediate CANCELLED return. + TEST_F( CancellationTest, CancelBeforeStart ) + { + // Test CancellationToken directly (no ProcessingManager needed) + CancellationToken token; + EXPECT_FALSE( token.IsCancelled() ); + + bool callbackInvoked = false; + token.SetCallback( [&callbackInvoked]() { callbackInvoked = true; } ); + + token.Cancel(); + EXPECT_TRUE( token.IsCancelled() ); + EXPECT_TRUE( callbackInvoked ); + + // Cancel() called again should not invoke callback a second time + callbackInvoked = false; + token.Cancel(); + EXPECT_FALSE( callbackInvoked ); + } + +} // namespace test +} // namespace sgns::sgprocessing diff --git a/test/execution/checkpoint_test.cpp b/test/execution/checkpoint_test.cpp new file mode 100644 index 0000000..596df65 --- /dev/null +++ b/test/execution/checkpoint_test.cpp @@ -0,0 +1,39 @@ +/** + * Checkpoint support tests — EXEC-05. + * + * Tests that supports_checkpointing is false for all registered executors. + */ +#include +#include +#include + +namespace sgns::sgprocessing +{ +namespace test +{ + + class CheckpointTest : public ::testing::Test + { + }; + + /// Verify PassTypeHash works correctly for map lookups. + TEST_F( CheckpointTest, PassTypeHashWorks ) + { + PassTypeHash hash; + EXPECT_EQ( hash( PassType::RENDER ), static_cast( PassType::RENDER ) ); + } + + /// Verify CapabilitySnapshot::checkpointSupport exists and is empty by default. + TEST_F( CheckpointTest, CheckpointSupportDefaultEmpty ) + { + CapabilitySnapshot snap; + EXPECT_TRUE( snap.checkpointSupport.empty() ); + } + + // TODO: Integration tests requiring CapabilityValidator: + // - CheckpointNotSupportedForRender: query supports_checkpointing for RENDER → false + // - CheckpointFlagInRegistry: ExecutorRegistryEntry.supports_checkpointing is false + // - AllExecutorsCheckpointFalse: all registered executors have false + +} // namespace test +} // namespace sgns::sgprocessing diff --git a/test/execution/leak_detection_test.cpp b/test/execution/leak_detection_test.cpp new file mode 100644 index 0000000..2a33958 --- /dev/null +++ b/test/execution/leak_detection_test.cpp @@ -0,0 +1,62 @@ +/** + * Repeat-run leak detection tests — EXEC-06 (D-16). + * + * Runs cancel/timeout/budget scenarios in N >= 10 iterations, + * tracking resource usage to assert no monotonic growth. + * + * Strategy: Process memory tracking (fallback approach). + * Tracks process RSS before/after iterations using platform APIs. + * Vulkan Validation Layers (when ENABLE_VULKAN_VALIDATION) provide + * object-level leak detection as the preferred approach. + */ +#include +#include + +namespace sgns::sgprocessing +{ +namespace test +{ + + class LeakDetectionTest : public ::testing::Test + { + protected: + void SetUp() override + { + // Skip if no Vulkan device available + } + }; + + /// Verify CancellationToken cleanup: no resources leaked after repeated use. + TEST_F( LeakDetectionTest, TokenNoLeakOverIterations ) + { + for ( int i = 0; i < 10; ++i ) + { + CancellationToken token; + bool called = false; + token.SetCallback( [&called]() { called = true; } ); + token.Cancel(); + EXPECT_TRUE( called ); + EXPECT_TRUE( token.IsCancelled() ); + } + } + + /// Verify ExecutionContext NoOp is consistent across iterations. + TEST_F( LeakDetectionTest, NoOpContextConsistency ) + { + for ( int i = 0; i < 10; ++i ) + { + auto ctx = ExecutionContext::NoOp(); + EXPECT_FALSE( ctx->cancelToken.IsCancelled() ); + EXPECT_EQ( ctx->deadlineMs, 0u ); + EXPECT_EQ( ctx->gpuMemoryBudget, 0u ); + EXPECT_EQ( ctx->maxOutputArtifactBytes, 0u ); + } + } + + // TODO: Integration tests requiring Vulkan: + // - CancelNoLeakOverIterations: N=10 cancel scenarios, track Vulkan/MNN objects + // - TimeoutNoLeakOverIterations: N=10 timeout scenarios + // - BudgetExceededNoLeakOverIterations: N=10 budget scenarios + +} // namespace test +} // namespace sgns::sgprocessing diff --git a/test/execution/migration_adapter_test.cpp b/test/execution/migration_adapter_test.cpp new file mode 100644 index 0000000..390b0ee --- /dev/null +++ b/test/execution/migration_adapter_test.cpp @@ -0,0 +1,41 @@ +/** + * Migration adapter tests — EXEC-07. + * + * Verifies that ExecutionContext::NoOp() produces identical behavior to + * the old non-ExecutionContext path, proving the adapter didn't change behavior. + */ +#include +#include +#include + +namespace sgns::sgprocessing +{ +namespace test +{ + + class MigrationAdapterTest : public ::testing::Test + { + }; + + /// Verify NoOp ExecutionContext doesn't cancel. + TEST_F( MigrationAdapterTest, NoOpDoesNotCancel ) + { + auto ctx = ExecutionContext::NoOp(); + EXPECT_FALSE( ctx->cancelToken.IsCancelled() ); + } + + /// Verify NoOp progress callback doesn't throw. + TEST_F( MigrationAdapterTest, NoOpProgressCallbackDoesNotThrow ) + { + auto ctx = ExecutionContext::NoOp(); + EXPECT_NO_THROW( ctx->progressCallback( + ProgressEvent::ForRender( "test", RenderStage::COMPILE, 0.0f ) ) ); + } + + // TODO: Integration tests requiring Vulkan/MNN: + // - MNNImageSameOutputThroughAdapter: identical output with NoOp vs real context + // - RenderProcessorSameOutputThroughAdapter: identical output + // - AllProcessorsCompileAndRun: all 15 processors instantiate and run with NoOp + +} // namespace test +} // namespace sgns::sgprocessing diff --git a/test/execution/progress_event_test.cpp b/test/execution/progress_event_test.cpp new file mode 100644 index 0000000..1dcaa5a --- /dev/null +++ b/test/execution/progress_event_test.cpp @@ -0,0 +1,71 @@ +/** + * Progress event tests — EXEC-04. + * + * Tests ProgressEvent struct, stage enums, and factory methods. + */ +#include +#include + +namespace sgns::sgprocessing +{ +namespace test +{ + + class ProgressEventTest : public ::testing::Test + { + }; + + /// Verify ProgressEvent default values. + TEST_F( ProgressEventTest, DefaultValues ) + { + ProgressEvent ev; + EXPECT_TRUE( ev.pass_id.empty() ); + EXPECT_EQ( ev.render_stage, RenderStage::COMPILE ); + EXPECT_EQ( ev.mnn_stage, MNNStage::LOAD_MODEL ); + EXPECT_FLOAT_EQ( ev.percent, 0.0f ); + } + + /// Verify ForRender factory populates render_stage. + TEST_F( ProgressEventTest, ForRenderFactory ) + { + auto ev = ProgressEvent::ForRender( "render_pass_1", RenderStage::BUILD_PIPELINE, 50.0f ); + EXPECT_EQ( ev.pass_id, "render_pass_1" ); + EXPECT_EQ( ev.render_stage, RenderStage::BUILD_PIPELINE ); + EXPECT_EQ( ev.mnn_stage, MNNStage::LOAD_MODEL ); // default + EXPECT_FLOAT_EQ( ev.percent, 50.0f ); + } + + /// Verify ForMNN factory populates mnn_stage. + TEST_F( ProgressEventTest, ForMNNFactory ) + { + auto ev = ProgressEvent::ForMNN( "mnn_pass_1", MNNStage::RUN, 75.0f ); + EXPECT_EQ( ev.pass_id, "mnn_pass_1" ); + EXPECT_EQ( ev.render_stage, RenderStage::COMPILE ); // default + EXPECT_EQ( ev.mnn_stage, MNNStage::RUN ); + EXPECT_FLOAT_EQ( ev.percent, 75.0f ); + } + + /// Verify stage enums have correct numeric values. + TEST_F( ProgressEventTest, RenderStageEnumValues ) + { + EXPECT_EQ( static_cast( RenderStage::COMPILE ), 0 ); + EXPECT_EQ( static_cast( RenderStage::BUILD_PIPELINE ), 1 ); + EXPECT_EQ( static_cast( RenderStage::DRAW ), 2 ); + EXPECT_EQ( static_cast( RenderStage::READBACK ), 3 ); + } + + /// Verify MNN stage enums have correct numeric values. + TEST_F( ProgressEventTest, MNNStageEnumValues ) + { + EXPECT_EQ( static_cast( MNNStage::LOAD_MODEL ), 0 ); + EXPECT_EQ( static_cast( MNNStage::CREATE_SESSION ), 1 ); + EXPECT_EQ( static_cast( MNNStage::RUN ), 2 ); + EXPECT_EQ( static_cast( MNNStage::READ_OUTPUT ), 3 ); + } + + // TODO: Integration tests requiring Vulkan: + // - RenderProcessorProgressEvents: capture 4 events with correct progression + // - MNNProcessorProgressEvents: capture 4 MNN stages + +} // namespace test +} // namespace sgns::sgprocessing diff --git a/test/execution/timeout_test.cpp b/test/execution/timeout_test.cpp new file mode 100644 index 0000000..3f30d9f --- /dev/null +++ b/test/execution/timeout_test.cpp @@ -0,0 +1,60 @@ +/** + * Timeout tests for ExecutionContext — EXEC-02. + * + * Tests deadline expiry produces TIMED_OUT distinct from CANCELLED. + */ +#include +#include +#include + +namespace sgns::sgprocessing +{ +namespace test +{ + + class TimeoutTest : public ::testing::Test + { + }; + + /// Verify TIMED_OUT error stage is distinct from CANCELLED and BUDGET_EXCEEDED. + TEST_F( TimeoutTest, ErrorStagesAreDistinct ) + { + EXPECT_NE( static_cast( ProcessingErrorStage::TIMED_OUT ), + static_cast( ProcessingErrorStage::CANCELLED ) ); + EXPECT_NE( static_cast( ProcessingErrorStage::TIMED_OUT ), + static_cast( ProcessingErrorStage::BUDGET_EXCEEDED ) ); + EXPECT_NE( static_cast( ProcessingErrorStage::CANCELLED ), + static_cast( ProcessingErrorStage::BUDGET_EXCEEDED ) ); + } + + /// Verify CancellationToken cancel callback is invoked at most once. + TEST_F( TimeoutTest, CancelCallbackInvokedOnce ) + { + CancellationToken token; + int callCount = 0; + token.SetCallback( [&callCount]() { ++callCount; } ); + + token.Cancel(); + EXPECT_EQ( callCount, 1 ); + + // Second Cancel() should not invoke again + token.Cancel(); + EXPECT_EQ( callCount, 1 ); + } + + /// Verify IsCancelled() returns true after Cancel(). + TEST_F( TimeoutTest, IsCancelledAfterCancel ) + { + CancellationToken token; + EXPECT_FALSE( token.IsCancelled() ); + token.Cancel(); + EXPECT_TRUE( token.IsCancelled() ); + } + + // TODO: Integration tests requiring Vulkan: + // - DeadlineExpiryReturnsTimedOut: job with per_pass_deadline_ms=100, sleep >100ms + // - NoDeadlineRunsNormally: job with deadline=0 completes normally + // - DeadlineDistinctFromCancel: TIMED_OUT != CANCELLED error codes + +} // namespace test +} // namespace sgns::sgprocessing diff --git a/test/util/CMakeLists.txt b/test/util/CMakeLists.txt new file mode 100644 index 0000000..ff33d24 --- /dev/null +++ b/test/util/CMakeLists.txt @@ -0,0 +1,33 @@ +# Quantization unit tests (Phase 12, Plan 12-01) +# Tests: QuantizeFloatBuffer IEEE-754 canonicalization + fixed-grid rounding, +# QuantizeByteBuffer identity behavior. + +add_executable(quantization_test + quantization_test.cpp +) + +target_link_libraries(quantization_test + PRIVATE + sgprocmanagerquant + GTest::gtest_main +) + +enable_testing() +add_test(NAME QuantizationTest COMMAND quantization_test) + +# diff_utils unit tests (Phase 15, Plan 15-01) +# Tests: extracted ComputeFloat32Diff/ComputeUint8Diff correctness, +# ResolveChunkElementTypeHint defaults, IsFloatChunkWithinTolerance/ +# IsByteChunkWithinTolerance D-03/D-04 pass/fail boundaries. + +add_executable(diff_utils_test + diff_utils_test.cpp +) + +target_link_libraries(diff_utils_test + PRIVATE + sgprocmanagerdiff + GTest::gtest_main +) + +add_test(NAME DiffUtilsTest COMMAND diff_utils_test) diff --git a/test/util/diff_utils_test.cpp b/test/util/diff_utils_test.cpp new file mode 100644 index 0000000..1c7535c --- /dev/null +++ b/test/util/diff_utils_test.cpp @@ -0,0 +1,254 @@ +// Phase 15, Plan 15-01: unit tests for diff_utils.hpp/.cpp -- the extracted +// capture_diff diff primitives (ComputeFloat32Diff/ComputeUint8Diff) plus the +// new D-03/D-04 tolerance-derivation functions +// (ResolveChunkElementTypeHint/IsFloatChunkWithinTolerance/ +// IsByteChunkWithinTolerance). +// +// Mirrors quantization_test.cpp's structure: pure in-memory unit tests, no +// file fixtures, a local Parameter-vector-building helper via +// set_name/set_type/set_parameter_default, and memcpy-based bit-pattern +// construction (never approximate float comparison for exact cases). + +#include + +#include +#include +#include +#include + +#include "util/diff_utils.hpp" + +namespace sgns::sgprocmanagerdiff +{ + namespace + { + uint32_t BitsOf( float value ) + { + uint32_t bits = 0; + std::memcpy( &bits, &value, sizeof( bits ) ); + return bits; + } + + float FloatFromBits( uint32_t bits ) + { + float value = 0.0f; + std::memcpy( &value, &bits, sizeof( value ) ); + return value; + } + + // Packs a vector of floats into a raw little/native-endian byte + // buffer, mirroring how ComputeFloat32Diff reads raw capture bytes. + std::vector BuildFloatBytes( const std::vector &values ) + { + std::vector bytes( values.size() * sizeof( float ) ); + for ( size_t i = 0; i < values.size(); ++i ) + { + std::memcpy( bytes.data() + i * sizeof( float ), &values[i], sizeof( float ) ); + } + return bytes; + } + + std::vector BuildByteBytes( const std::vector &values ) + { + return values; + } + + // Phase 14, Plan 14-01's parameters-building helper, mirrored exactly + // (quantization_test.cpp). + std::vector MakeParameters( const std::string &name, + sgns::ParameterType type, + const nlohmann::json &defaultValue ) + { + sgns::Parameter param; + param.set_name( name ); + param.set_type( type ); + param.set_parameter_default( defaultValue ); + return { param }; + } + } // namespace + + class DiffUtilsTest : public ::testing::Test + { + }; + + // --- ComputeFloat32Diff / ComputeUint8Diff extraction correctness --- + + TEST_F( DiffUtilsTest, ComputeFloat32DiffMatchesKnownDelta ) + { + auto bytesA = BuildFloatBytes( { 1.0f, 2.0f } ); + auto bytesB = BuildFloatBytes( { 1.0f, 2.5f } ); + + ElementDiffStats stats = ComputeFloat32Diff( bytesA, bytesB ); + + ASSERT_FALSE( stats.sizeMismatch ); + ASSERT_EQ( stats.elementCount, 2u ); + ASSERT_DOUBLE_EQ( stats.maxAbsDelta, 0.5 ); + } + + TEST_F( DiffUtilsTest, ComputeFloat32DiffDetectsSizeMismatch ) + { + auto bytesA = BuildFloatBytes( { 1.0f } ); + auto bytesB = BuildFloatBytes( { 1.0f, 2.0f } ); + + ElementDiffStats stats = ComputeFloat32Diff( bytesA, bytesB ); + + ASSERT_TRUE( stats.sizeMismatch ); + } + + TEST_F( DiffUtilsTest, ComputeUint8DiffMatchesKnownDelta ) + { + auto bytesA = BuildByteBytes( { 10, 20 } ); + auto bytesB = BuildByteBytes( { 10, 25 } ); + + ElementDiffStats stats = ComputeUint8Diff( bytesA, bytesB ); + + ASSERT_FALSE( stats.sizeMismatch ); + ASSERT_EQ( stats.elementCount, 2u ); + ASSERT_DOUBLE_EQ( stats.maxAbsDelta, 5.0 ); + } + + TEST_F( DiffUtilsTest, ComputeUint8DiffDetectsSizeMismatch ) + { + auto bytesA = BuildByteBytes( { 10 } ); + auto bytesB = BuildByteBytes( { 10, 20 } ); + + ElementDiffStats stats = ComputeUint8Diff( bytesA, bytesB ); + + ASSERT_TRUE( stats.sizeMismatch ); + } + + // --- ResolveChunkElementTypeHint --- + + TEST_F( DiffUtilsTest, ResolveChunkElementTypeHintDefaultsToFloatWhenNothingDeclared ) + { + ASSERT_EQ( ResolveChunkElementTypeHint( nullptr ), ChunkElementType::FLOAT32 ); + } + + TEST_F( DiffUtilsTest, ResolveChunkElementTypeHintDefaultsToFloatWhenOnlyQuantScaleDeclared ) + { + const auto parameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, 32768.0 ); + ASSERT_EQ( ResolveChunkElementTypeHint( ¶meters ), ChunkElementType::FLOAT32 ); + } + + TEST_F( DiffUtilsTest, ResolveChunkElementTypeHintReturnsUint8WhenByteQuantModeDeclared ) + { + const auto parameters = MakeParameters( "byteQuantMode", sgns::ParameterType::INT, 3 ); + ASSERT_EQ( ResolveChunkElementTypeHint( ¶meters ), ChunkElementType::UINT8 ); + } + + // --- IsFloatChunkWithinTolerance: D-03 grid-step bound (declared quantScale) --- + + TEST_F( DiffUtilsTest, IsFloatChunkWithinToleranceUsesGridStepBoundWhenDeclaredPasses ) + { + const auto parameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, 32768.0 ); + auto bytesA = BuildFloatBytes( { 0.0f } ); + auto bytesB = BuildFloatBytes( { 2.0f / 32768.0f } ); // exactly at the 2/S bound + + ElementDiffStats stats; + ASSERT_TRUE( IsFloatChunkWithinTolerance( bytesA, bytesB, ¶meters, stats ) ); + ASSERT_FALSE( stats.sizeMismatch ); + } + + TEST_F( DiffUtilsTest, IsFloatChunkWithinToleranceUsesGridStepBoundWhenDeclaredFails ) + { + const auto parameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, 32768.0 ); + auto bytesA = BuildFloatBytes( { 0.0f } ); + auto bytesB = BuildFloatBytes( { 3.0f / 32768.0f } ); // strictly more than the 2/S bound + + ElementDiffStats stats; + ASSERT_FALSE( IsFloatChunkWithinTolerance( bytesA, bytesB, ¶meters, stats ) ); + } + + // --- IsFloatChunkWithinTolerance: D-04 fallback (no valid quantScale declared) --- + + TEST_F( DiffUtilsTest, IsFloatChunkWithinToleranceFallsBackToRelativeThresholdWhenUndeclaredPasses ) + { + // relDelta = 0.00005 / 1.00005 ~= 5e-5, within kDefaultFloatRelativeThreshold (1e-4). + auto bytesA = BuildFloatBytes( { 1.0f } ); + auto bytesB = BuildFloatBytes( { 1.00005f } ); + + ElementDiffStats stats; + ASSERT_TRUE( IsFloatChunkWithinTolerance( bytesA, bytesB, nullptr, stats ) ); + } + + TEST_F( DiffUtilsTest, IsFloatChunkWithinToleranceFallsBackToRelativeThresholdWhenUndeclaredFails ) + { + // relDelta ~= 0.0003/1.0003 ~= 3e-4, exceeds kDefaultFloatRelativeThreshold (1e-4). + auto bytesA = BuildFloatBytes( { 1.0f } ); + auto bytesB = BuildFloatBytes( { 1.0003f } ); + + ElementDiffStats stats; + ASSERT_FALSE( IsFloatChunkWithinTolerance( bytesA, bytesB, nullptr, stats ) ); + } + + TEST_F( DiffUtilsTest, IsFloatChunkWithinToleranceDetectsSizeMismatch ) + { + auto bytesA = BuildFloatBytes( { 1.0f } ); + auto bytesB = BuildFloatBytes( { 1.0f, 2.0f } ); + + ElementDiffStats stats; + ASSERT_FALSE( IsFloatChunkWithinTolerance( bytesA, bytesB, nullptr, stats ) ); + ASSERT_TRUE( stats.sizeMismatch ); + } + + // --- IsByteChunkWithinTolerance: D-03 mask-width bound (declared byteQuantMode) --- + + TEST_F( DiffUtilsTest, IsByteChunkWithinToleranceUsesMaskBoundWhenDeclaredPasses ) + { + const auto parameters = MakeParameters( "byteQuantMode", sgns::ParameterType::INT, 3 ); + auto bytesA = BuildByteBytes( { 0 } ); + auto bytesB = BuildByteBytes( { 7 } ); // exactly (1<<3)-1 + + ElementDiffStats stats; + ASSERT_TRUE( IsByteChunkWithinTolerance( bytesA, bytesB, ¶meters, stats ) ); + } + + TEST_F( DiffUtilsTest, IsByteChunkWithinToleranceUsesMaskBoundWhenDeclaredFails ) + { + const auto parameters = MakeParameters( "byteQuantMode", sgns::ParameterType::INT, 3 ); + auto bytesA = BuildByteBytes( { 0 } ); + auto bytesB = BuildByteBytes( { 8 } ); // strictly more than (1<<3)-1 + + ElementDiffStats stats; + ASSERT_FALSE( IsByteChunkWithinTolerance( bytesA, bytesB, ¶meters, stats ) ); + } + + // --- IsByteChunkWithinTolerance: D-04 fallback (no valid byteQuantMode declared) --- + + TEST_F( DiffUtilsTest, IsByteChunkWithinToleranceFallsBackToAbsoluteThresholdWhenUndeclaredPasses ) + { + auto bytesA = BuildByteBytes( { 10 } ); + auto bytesB = BuildByteBytes( { 11 } ); // absDelta = 1 == kDefaultByteAbsoluteThreshold + + ElementDiffStats stats; + ASSERT_TRUE( IsByteChunkWithinTolerance( bytesA, bytesB, nullptr, stats ) ); + } + + TEST_F( DiffUtilsTest, IsByteChunkWithinToleranceFallsBackToAbsoluteThresholdWhenUndeclaredFails ) + { + auto bytesA = BuildByteBytes( { 10 } ); + auto bytesB = BuildByteBytes( { 12 } ); // absDelta = 2, exceeds kDefaultByteAbsoluteThreshold + + ElementDiffStats stats; + ASSERT_FALSE( IsByteChunkWithinTolerance( bytesA, bytesB, nullptr, stats ) ); + } + + TEST_F( DiffUtilsTest, IsByteChunkWithinToleranceDetectsSizeMismatch ) + { + auto bytesA = BuildByteBytes( { 10 } ); + auto bytesB = BuildByteBytes( { 10, 20 } ); + + ElementDiffStats stats; + ASSERT_FALSE( IsByteChunkWithinTolerance( bytesA, bytesB, nullptr, stats ) ); + ASSERT_TRUE( stats.sizeMismatch ); + } + + // Silence unused-function warnings for BitsOf/FloatFromBits (kept for + // parity with quantization_test.cpp's helper set, available for future + // exact-bit-pattern assertions in this suite). + TEST_F( DiffUtilsTest, BitHelpersRoundTrip ) + { + ASSERT_EQ( FloatFromBits( BitsOf( 1.5f ) ), 1.5f ); + } + +} // namespace sgns::sgprocmanagerdiff diff --git a/test/util/quantization_test.cpp b/test/util/quantization_test.cpp new file mode 100644 index 0000000..3028051 --- /dev/null +++ b/test/util/quantization_test.cpp @@ -0,0 +1,220 @@ +// Phase 12, Plan 12-01: unit tests for the real QuantizeFloatBuffer/ +// QuantizeByteBuffer implementations (D-03 through D-09). +// +// Pure in-memory unit tests -- no file fixtures, no ProcessorConformanceFixture +// base needed. Bit patterns are compared via memcpy-extracted uint32_t and +// ASSERT_EQ, never via approximate float comparison, since D-09/D-06/D-08 +// require exact canonical output. +// +// Phase 14, Plan 14-01: extended with ResolveQuantScale/ResolveByteQuantMode +// fallback/boundary coverage (D-04/D-05/D-07/D-08), and every pre-existing +// QuantizeFloatBuffer/QuantizeByteBuffer call updated to the new required +// 3-arg signature (scale/maskBits are no longer compile-time constants). + +#include + +#include +#include +#include +#include +#include + +#include "util/quantization.hpp" + +namespace sgns::sgprocmanagerquant +{ + namespace + { + uint32_t BitsOf( float value ) + { + uint32_t bits = 0; + std::memcpy( &bits, &value, sizeof( bits ) ); + return bits; + } + + float FloatFromBits( uint32_t bits ) + { + float value = 0.0f; + std::memcpy( &value, &bits, sizeof( value ) ); + return value; + } + } // namespace + + class QuantizationTest : public ::testing::Test + { + }; + + TEST_F( QuantizationTest, QuantizeFloatBufferCanonicalizesNaN ) + { + // NaN with nonzero payload -> exact canonical quiet-NaN bit pattern. + float data1[1] = { FloatFromBits( 0x7FC00123u ) }; + QuantizeFloatBuffer( data1, 1, 32768.0f ); + ASSERT_EQ( BitsOf( data1[0] ), 0x7FC00000u ); + + // Negative NaN -> sign discarded, same hardcoded canonical pattern (D-09). + float data2[1] = { FloatFromBits( 0xFFC00000u ) }; + QuantizeFloatBuffer( data2, 1, 32768.0f ); + ASSERT_EQ( BitsOf( data2[0] ), 0x7FC00000u ); + } + + TEST_F( QuantizationTest, QuantizeFloatBufferCanonicalizesPositiveInfinity ) + { + float data[1] = { FloatFromBits( 0x7F800000u ) }; + QuantizeFloatBuffer( data, 1, 32768.0f ); + ASSERT_EQ( BitsOf( data[0] ), 0x7F800000u ); + } + + TEST_F( QuantizationTest, QuantizeFloatBufferCanonicalizesNegativeInfinity ) + { + // -Inf stays distinct from +Inf (D-06), never collapsed. + float data[1] = { FloatFromBits( 0xFF800000u ) }; + QuantizeFloatBuffer( data, 1, 32768.0f ); + ASSERT_EQ( BitsOf( data[0] ), 0xFF800000u ); + } + + TEST_F( QuantizationTest, QuantizeFloatBufferCanonicalizesDenormals ) + { + // Smallest positive denormal, smallest negative denormal -> both flush + // to canonical +0.0. + float data[2] = { FloatFromBits( 0x00000001u ), FloatFromBits( 0x80000001u ) }; + QuantizeFloatBuffer( data, 2, 32768.0f ); + ASSERT_EQ( BitsOf( data[0] ), 0x00000000u ); + ASSERT_EQ( BitsOf( data[1] ), 0x00000000u ); + } + + TEST_F( QuantizationTest, QuantizeFloatBufferCanonicalizesSignedZero ) + { + // -0.0 and +0.0 both collapse to the single canonical zero bit pattern. + float data[2] = { FloatFromBits( 0x80000000u ), FloatFromBits( 0x00000000u ) }; + QuantizeFloatBuffer( data, 2, 32768.0f ); + ASSERT_EQ( BitsOf( data[0] ), 0x00000000u ); + ASSERT_EQ( BitsOf( data[1] ), 0x00000000u ); + } + + TEST_F( QuantizationTest, QuantizeFloatBufferRoundsToFixedGrid ) + { + // Ordinary finite value, not on the 2^-15 grid. + constexpr float kScale = 32768.0f; // 2^15, matches Phase 13 Plan 13-04 gap-closure widening + float data[1] = { 0.1f }; + QuantizeFloatBuffer( data, 1, kScale ); + + const float expected = std::round( 0.1f * kScale ) / kScale; + ASSERT_EQ( BitsOf( data[0] ), BitsOf( expected ) ); + + // Grid-alignment property, independent of the formula-echo check above: + // (output * scale) must itself be an exact integer. + const float scaled = data[0] * kScale; + ASSERT_EQ( scaled, std::round( scaled ) ); + } + + TEST_F( QuantizationTest, QuantizeByteBufferIsIdentity ) + { + uint8_t data[5] = { 0, 1, 127, 128, 255 }; + const uint8_t expected[5] = { 0, 1, 127, 128, 255 }; + QuantizeByteBuffer( data, 5, 0 ); + ASSERT_EQ( std::memcmp( data, expected, sizeof( data ) ), 0 ); + } + + TEST_F( QuantizationTest, QuantizeByteBufferClearsLowBits ) + { + // D-06: maskBits=3 clears exactly the low 3 bits of every byte. + uint8_t data[3] = { 0xFF, 0x07, 0xAA }; + const uint8_t expected[3] = { 0xF8, 0x00, 0xA8 }; + QuantizeByteBuffer( data, 3, 3 ); + ASSERT_EQ( std::memcmp( data, expected, sizeof( data ) ), 0 ); + } + + TEST_F( QuantizationTest, QuantizeByteBufferMasksAllBitsAtBoundaryEight ) + { + // D-07/D-08 disclosed boundary: maskBits=8 is valid (not a fallback + // trigger) and collapses every byte to 0x00. + uint8_t data[3] = { 0xFF, 0x01, 0x80 }; + const uint8_t expected[3] = { 0x00, 0x00, 0x00 }; + QuantizeByteBuffer( data, 3, 8 ); + ASSERT_EQ( std::memcmp( data, expected, sizeof( data ) ), 0 ); + } + + namespace + { + // Phase 14, Task 2: builds a one-element parameters vector for a + // Resolve* test case, using Parameter's public setters. + std::vector MakeParameters( const std::string &name, + sgns::ParameterType type, + const nlohmann::json &defaultValue ) + { + sgns::Parameter param; + param.set_name( name ); + param.set_type( type ); + param.set_parameter_default( defaultValue ); + return { param }; + } + } // namespace + + TEST_F( QuantizationTest, ResolveQuantScaleFallsBackOnNullParameters ) + { + ASSERT_EQ( BitsOf( ResolveQuantScale( nullptr ) ), BitsOf( 32768.0f ) ); + } + + TEST_F( QuantizationTest, ResolveQuantScaleFallsBackOnMissingEntry ) + { + const std::vector parameters; + ASSERT_EQ( BitsOf( ResolveQuantScale( ¶meters ) ), BitsOf( 32768.0f ) ); + } + + TEST_F( QuantizationTest, ResolveQuantScaleUsesValidPowerOfTwo ) + { + const auto parameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, 16384.0 ); + ASSERT_EQ( BitsOf( ResolveQuantScale( ¶meters ) ), BitsOf( 16384.0f ) ); + } + + TEST_F( QuantizationTest, ResolveQuantScaleFallsBackOnNonPowerOfTwo ) + { + const auto parameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, 100.0 ); + ASSERT_EQ( BitsOf( ResolveQuantScale( ¶meters ) ), BitsOf( 32768.0f ) ); + } + + TEST_F( QuantizationTest, ResolveQuantScaleFallsBackOnNonPositive ) + { + const auto zeroParameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, 0.0 ); + ASSERT_EQ( BitsOf( ResolveQuantScale( &zeroParameters ) ), BitsOf( 32768.0f ) ); + + const auto negativeParameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, -8.0 ); + ASSERT_EQ( BitsOf( ResolveQuantScale( &negativeParameters ) ), BitsOf( 32768.0f ) ); + } + + TEST_F( QuantizationTest, ResolveQuantScaleFallsBackOnNonNumeric ) + { + const auto parameters = MakeParameters( "quantScale", sgns::ParameterType::FLOAT, std::string( "16384" ) ); + ASSERT_EQ( BitsOf( ResolveQuantScale( ¶meters ) ), BitsOf( 32768.0f ) ); + } + + TEST_F( QuantizationTest, ResolveByteQuantModeFallsBackOnNullParameters ) + { + ASSERT_EQ( ResolveByteQuantMode( nullptr ), 0 ); + } + + TEST_F( QuantizationTest, ResolveByteQuantModeUsesValidValue ) + { + const auto parameters = MakeParameters( "byteQuantMode", sgns::ParameterType::INT, 3 ); + ASSERT_EQ( ResolveByteQuantMode( ¶meters ), 3 ); + } + + TEST_F( QuantizationTest, ResolveByteQuantModeAcceptsBoundaryEight ) + { + const auto parameters = MakeParameters( "byteQuantMode", sgns::ParameterType::INT, 8 ); + ASSERT_EQ( ResolveByteQuantMode( ¶meters ), 8 ); + } + + TEST_F( QuantizationTest, ResolveByteQuantModeFallsBackJustAboveBoundary ) + { + const auto parameters = MakeParameters( "byteQuantMode", sgns::ParameterType::INT, 9 ); + ASSERT_EQ( ResolveByteQuantMode( ¶meters ), 0 ); + } + + TEST_F( QuantizationTest, ResolveByteQuantModeFallsBackOnNegative ) + { + const auto parameters = MakeParameters( "byteQuantMode", sgns::ParameterType::INT, -1 ); + ASSERT_EQ( ResolveByteQuantMode( ¶meters ), 0 ); + } + +} // namespace sgns::sgprocmanagerquant diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000..0187894 --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(capture) diff --git a/tools/capture/CMakeLists.txt b/tools/capture/CMakeLists.txt new file mode 100644 index 0000000..d83b1c0 --- /dev/null +++ b/tools/capture/CMakeLists.txt @@ -0,0 +1,47 @@ +# Phase 10 (capture-harness-diff-tool-quantization-stub) -- standalone CLI tooling. +# Neither capture_harness nor capture_diff is CTest-gated (Pattern 5): a meaningful +# cross-machine pass/fail needs Phase 11's physical machines. + +add_library(sgproccapture STATIC + capture_file_format.cpp + capture_file_format.hpp +) + +target_include_directories(sgproccapture PUBLIC + $ + # capture_file_format.cpp includes its own header via the + # "tools/capture/capture_file_format.hpp" root-relative path (Plan 10-04); + # add the SGProcessingManager root so that path resolves. + $ +) + +target_link_libraries(sgproccapture + PUBLIC + sgprocmanagersha + SGArtifacts +) + +sgnus_install(sgproccapture) + +add_executable(capture_harness + capture_harness.cpp +) + +target_link_libraries(capture_harness + PRIVATE + ProcessingBase + sgproccapture + nlohmann_json::nlohmann_json + ipfs-bitswap-cpp +) + +add_executable(capture_diff + capture_diff.cpp +) + +target_link_libraries(capture_diff + PRIVATE + sgproccapture + sgprocmanagerdiff + nlohmann_json::nlohmann_json +) diff --git a/tools/capture/capture_diff.cpp b/tools/capture/capture_diff.cpp new file mode 100644 index 0000000..0f923e4 --- /dev/null +++ b/tools/capture/capture_diff.cpp @@ -0,0 +1,338 @@ +/** + * capture_diff -- standalone CLI tool (Phase 10, Plan 10-05, DIFF-01/02/03). + * + * Reads two .cap files (produced by capture_harness, possibly on different + * machines) and reports quantitative per-element divergence (absolute delta, + * relative delta, ULP distance, whole-buffer summary stats) plus independent + * hash-match booleans, to both console and a JSON report (D-06). + * + * Not CTest-gated (Pattern 5) -- a meaningful cross-machine pass/fail needs + * Phase 11's physical machines. + * + * Update (Phase 13 gap-closure, Plan 13-06): the per-element numeric-diff pass + * described above originally examined only the trailing combined-hash capture + * record. It now ADDITIONALLY numeric-diffs each individual per-chunk raw + * capture record (rawRecordsPerArtifact[0][j] for j < chunkHashCount) via the + * new `chunkDiffs` JSON output array (index-aligned with `chunkHashesMatch`), + * closing the blind spot where a `chunkHashesMatch[j]: false` result carried + * no magnitude information. The original trailing-record pass is unchanged. + * + * Usage: + * capture_diff --a --b --element-type + * [--json-output ] + * + * @brief Capture diff CLI (compares two .cap files, reports divergence stats) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "capture_file_format.hpp" +#include "util/diff_utils.hpp" + +namespace +{ + struct CliArgs + { + std::string pathA; + std::string pathB; + std::string elementType; // "float32" or "uint8" + std::string jsonOutput = "diff_report.json"; + }; + + void PrintUsage() + { + std::cerr << "Usage: capture_diff --a --b --element-type " + "[--json-output ]\n"; + } + + /// Parses argv into CliArgs. + /// @return true on success; false (with an error already printed) on any parse failure. + bool ParseArgs( int argc, char **argv, CliArgs &out ) + { + for ( int i = 1; i < argc; ++i ) + { + std::string arg = argv[i]; + if ( arg == "--a" && i + 1 < argc ) + { + out.pathA = argv[++i]; + } + else if ( arg == "--b" && i + 1 < argc ) + { + out.pathB = argv[++i]; + } + else if ( arg == "--element-type" && i + 1 < argc ) + { + out.elementType = argv[++i]; + } + else if ( arg == "--json-output" && i + 1 < argc ) + { + out.jsonOutput = argv[++i]; + } + else + { + std::cerr << "capture_diff: unrecognized or incomplete argument: " << arg << "\n"; + return false; + } + } + + if ( out.pathA.empty() || out.pathB.empty() ) + { + std::cerr << "capture_diff: --a and --b are required\n"; + return false; + } + if ( out.elementType != "float32" && out.elementType != "uint8" ) + { + std::cerr << "capture_diff: --element-type must be exactly \"float32\" or \"uint8\", got \"" + << out.elementType << "\"\n"; + return false; + } + return true; + } + + /// Reads an entire file into a byte vector. + /// @return true on success; false (with an error already printed) if the file cannot be opened. + bool ReadFileBytes( const std::string &path, std::vector &out ) + { + std::ifstream stream( path, std::ios::binary ); + if ( !stream.is_open() ) + { + std::cerr << "capture_diff: could not open file " << path << "\n"; + return false; + } + out.assign( std::istreambuf_iterator( stream ), std::istreambuf_iterator() ); + return true; + } + +} // namespace + +int main( int argc, char **argv ) +{ + CliArgs args; + if ( !ParseArgs( argc, argv, args ) ) + { + PrintUsage(); + return 1; + } + + std::vector bytesA; + std::vector bytesB; + if ( !ReadFileBytes( args.pathA, bytesA ) || !ReadFileBytes( args.pathB, bytesB ) ) + { + return 1; + } + + sgns::sgproccapture::CaptureFile captureA; + sgns::sgproccapture::CaptureFile captureB; + if ( !sgns::sgproccapture::DeserializeCaptureFile( bytesA, captureA ) ) + { + std::cerr << "capture_diff: failed to parse capture file " << args.pathA << " (malformed or truncated)\n"; + return 1; + } + if ( !sgns::sgproccapture::DeserializeCaptureFile( bytesB, captureB ) ) + { + std::cerr << "capture_diff: failed to parse capture file " << args.pathB << " (malformed or truncated)\n"; + return 1; + } + + if ( captureA.artifacts.size() != 1 || captureB.artifacts.size() != 1 ) + { + std::cerr << "capture_diff: expected exactly one artifact per capture file (Phase 10 scope), got " + << captureA.artifacts.size() << " in " << args.pathA << " and " << captureB.artifacts.size() + << " in " << args.pathB << "\n"; + return 1; + } + + const auto &artifactA = captureA.artifacts[0]; + const auto &artifactB = captureB.artifacts[0]; + + if ( artifactA.chunkHashCount != artifactB.chunkHashCount ) + { + std::cerr << "capture_diff: chunkHashCount mismatch (a=" << artifactA.chunkHashCount + << ", b=" << artifactB.chunkHashCount + << ") -- the two capture files are not comparable (likely different fixtures or a " + "structural divergence, not a numeric one); distinct failure mode, not reported as " + "0% divergence\n"; + return 1; + } + + // DIFF-03: hash-match booleans, computed purely from artifact/manifest metadata. + bool contentHashMatch = std::equal( artifactA.contentHash, + artifactA.contentHash + sgns::sgprocessing::SHA256_HASH_SIZE, + artifactB.contentHash ); + + std::vector chunkHashesMatch; + uint32_t sharedChunkCount = std::min( artifactA.chunkHashCount, artifactB.chunkHashCount ); + chunkHashesMatch.reserve( sharedChunkCount ); + for ( uint32_t j = 0; j < sharedChunkCount; ++j ) + { + chunkHashesMatch.push_back( std::equal( artifactA.chunkHashes[j], + artifactA.chunkHashes[j] + sgns::sgprocessing::SHA256_HASH_SIZE, + artifactB.chunkHashes[j] ) ); + } + + bool combinedHashMatch = captureA.combinedHash == captureB.combinedHash; + + // DIFF-01/02: per-element numeric divergence over the LAST CaptureRecord's + // quantizedBytes -- the same bytes that fed each run's contentHash. + // Update (Phase 13 gap-closure, Plan 13-06): capture_diff now ALSO + // numeric-diffs each individual per-chunk raw record below (see + // `chunkStats`/`chunkDiffs`) -- this trailing-record-only pass is + // preserved unchanged as its own distinct stat. + sgns::sgprocmanagerdiff::ElementDiffStats stats; + bool haveRecords = !captureA.rawRecordsPerArtifact.empty() && !captureB.rawRecordsPerArtifact.empty() && + !captureA.rawRecordsPerArtifact[0].empty() && !captureB.rawRecordsPerArtifact[0].empty(); + + if ( !haveRecords ) + { + std::cerr << "capture_diff: one or both capture files have no raw capture records for artifact 0 -- " + "skipping per-element numeric pass\n"; + stats.sizeMismatch = true; + } + else + { + const auto &lastRecordA = captureA.rawRecordsPerArtifact[0].back(); + const auto &lastRecordB = captureB.rawRecordsPerArtifact[0].back(); + + if ( args.elementType == "float32" ) + { + stats = sgns::sgprocmanagerdiff::ComputeFloat32Diff( lastRecordA.quantizedBytes, lastRecordB.quantizedBytes ); + } + else + { + stats = sgns::sgprocmanagerdiff::ComputeUint8Diff( lastRecordA.quantizedBytes, lastRecordB.quantizedBytes ); + } + + if ( stats.sizeMismatch ) + { + std::cerr << "capture_diff: size mismatch between the two files' final capture record bytes (" + << lastRecordA.quantizedBytes.size() << " vs " << lastRecordB.quantizedBytes.size() + << ") -- skipping per-element numeric pass (hash-match booleans above are still valid)\n"; + } + } + + // DIFF-01/02 extension (Phase 13 gap-closure, Plan 13-06): per-chunk numeric + // divergence over each individual rawRecordsPerArtifact[0][j] record + // (j < chunkHashCount), closing the blind spot where chunkHashesMatch[j] + // could report a divergence without ever reporting its magnitude. Reuses + // ComputeFloat32Diff/ComputeUint8Diff unmodified -- only the caller loop + // and its per-chunk inputs are new. + std::vector chunkStats; + chunkStats.reserve( chunkHashesMatch.size() ); + bool haveArtifactZeroRecords = !captureA.rawRecordsPerArtifact.empty() && !captureB.rawRecordsPerArtifact.empty(); + for ( size_t j = 0; j < chunkHashesMatch.size(); ++j ) + { + bool haveChunkRecords = haveArtifactZeroRecords && captureA.rawRecordsPerArtifact[0].size() > j && + captureB.rawRecordsPerArtifact[0].size() > j; + if ( !haveChunkRecords ) + { + std::cerr << "capture_diff: chunk " << j + << " has no raw capture record in one or both files -- skipping its per-chunk numeric pass\n"; + sgns::sgprocmanagerdiff::ElementDiffStats missing; + missing.sizeMismatch = true; + chunkStats.push_back( missing ); + continue; + } + + const auto &chunkRecordA = captureA.rawRecordsPerArtifact[0][j]; + const auto &chunkRecordB = captureB.rawRecordsPerArtifact[0][j]; + + if ( args.elementType == "float32" ) + { + chunkStats.push_back( sgns::sgprocmanagerdiff::ComputeFloat32Diff( chunkRecordA.quantizedBytes, chunkRecordB.quantizedBytes ) ); + } + else + { + chunkStats.push_back( sgns::sgprocmanagerdiff::ComputeUint8Diff( chunkRecordA.quantizedBytes, chunkRecordB.quantizedBytes ) ); + } + } + + // Console output. + std::cout << "capture_diff: comparing " << args.pathA << " vs " << args.pathB << " (element-type " + << args.elementType << ")\n"; + std::cout << " contentHashMatch: " << ( contentHashMatch ? "true" : "false" ) << "\n"; + std::cout << " combinedHashMatch: " << ( combinedHashMatch ? "true" : "false" ) << "\n"; + std::cout << " chunkHashesMatch: ["; + for ( size_t j = 0; j < chunkHashesMatch.size(); ++j ) + { + std::cout << ( chunkHashesMatch[j] ? "true" : "false" ); + if ( j + 1 < chunkHashesMatch.size() ) + { + std::cout << ", "; + } + } + std::cout << "]\n"; + for ( size_t j = 0; j < chunkStats.size(); ++j ) + { + std::cout << " chunk[" << j << "] match=" << ( chunkHashesMatch[j] ? "true" : "false" ); + if ( chunkStats[j].sizeMismatch ) + { + std::cout << " sizeMismatch=true (per-chunk numeric pass skipped)\n"; + } + else + { + std::cout << " maxAbsDelta=" << chunkStats[j].maxAbsDelta << " maxRelDelta=" << chunkStats[j].maxRelDelta + << " maxUlpDistance=" << chunkStats[j].maxUlpDistance << "\n"; + } + } + if ( stats.sizeMismatch ) + { + std::cout << " sizeMismatch: true (per-element numeric pass skipped)\n"; + } + else + { + std::cout << " elementCount: " << stats.elementCount << "\n"; + std::cout << " maxAbsDelta: " << stats.maxAbsDelta << "\n"; + std::cout << " maxRelDelta: " << stats.maxRelDelta << "\n"; + std::cout << " maxUlpDistance: " << stats.maxUlpDistance << "\n"; + std::cout << " percentExceedingThreshold: " << stats.percentExceedingThreshold << "%\n"; + } + + // JSON output (D-06 -- both console and JSON, not deferred). + nlohmann::json report; + report["elementType"] = args.elementType; + report["elementCount"] = stats.elementCount; + report["maxAbsDelta"] = stats.maxAbsDelta; + report["maxRelDelta"] = stats.maxRelDelta; + report["maxUlpDistance"] = stats.maxUlpDistance; + report["percentExceedingThreshold"] = stats.percentExceedingThreshold; + report["sizeMismatch"] = stats.sizeMismatch; + report["contentHashMatch"] = contentHashMatch; + report["combinedHashMatch"] = combinedHashMatch; + report["chunkHashesMatch"] = chunkHashesMatch; + + report["chunkDiffs"] = nlohmann::json::array(); + for ( size_t j = 0; j < chunkStats.size(); ++j ) + { + nlohmann::json chunkEntry; + chunkEntry["chunkIndex"] = j; + chunkEntry["elementCount"] = chunkStats[j].elementCount; + chunkEntry["maxAbsDelta"] = chunkStats[j].maxAbsDelta; + chunkEntry["maxRelDelta"] = chunkStats[j].maxRelDelta; + chunkEntry["maxUlpDistance"] = chunkStats[j].maxUlpDistance; + chunkEntry["percentExceedingThreshold"] = chunkStats[j].percentExceedingThreshold; + chunkEntry["sizeMismatch"] = chunkStats[j].sizeMismatch; + report["chunkDiffs"].push_back( chunkEntry ); + } + + std::ofstream jsonStream( args.jsonOutput ); + if ( !jsonStream.is_open() ) + { + std::cerr << "capture_diff: failed to open " << args.jsonOutput << " for writing\n"; + return 1; + } + jsonStream << report.dump( 2 ); + jsonStream.close(); + + std::cout << "capture_diff: wrote " << args.jsonOutput << "\n"; + + return 0; +} diff --git a/tools/capture/capture_file_format.cpp b/tools/capture/capture_file_format.cpp new file mode 100644 index 0000000..f8ba385 --- /dev/null +++ b/tools/capture/capture_file_format.cpp @@ -0,0 +1,309 @@ +/** + * Implementation of the capture file binary format (see capture_file_format.hpp + * for the full layout doc comment). Calls SerializeArtifact/SerializeManifest/ + * DeserializeArtifact/DeserializeManifest unmodified for the metadata+hash + * portion; only the new raw-bytes section (per rawOutputCapture invocation) + * and the outer magic/machine/fixture/combinedHash framing are implemented here. + */ + +#include "tools/capture/capture_file_format.hpp" +#include "artifacts/artifact_serializer.hpp" +#include + +namespace sgns::sgproccapture +{ + + namespace + { + constexpr char kMagic[4] = { 'S', 'G', 'C', '1' }; + + /// Hard cap on any single declared length/count field before it is + /// trusted to size an allocation or read (T-10-02). 1 GiB. + constexpr uint64_t kMaxSectionBytes = 1073741824ULL; + + // ── Bounded-count guard ───────────────────────────────────────── + // Rejects a declared item count BEFORE any per-item allocation + // (reserve/push_back) if it could not possibly fit in the bytes + // actually remaining in the input buffer, given a lower bound on + // how many bytes each item must occupy on the wire. Closes the + // "malformed count triggers unbounded allocation" risk (T-10-02) + // for both the artifact count and each artifact's record count. + bool CountFitsRemaining( uint64_t count, uint64_t minBytesPerItem, uint64_t remainingBytes ) + { + if ( minBytesPerItem == 0 ) + { + return true; + } + return count <= ( remainingBytes / minBytesPerItem ); + } + + // ── Write helpers ─────────────────────────────────────────────── + + void AppendU32( std::vector &out, uint32_t value ) + { + uint8_t bytes[sizeof( uint32_t )]; + std::memcpy( bytes, &value, sizeof( uint32_t ) ); + out.insert( out.end(), bytes, bytes + sizeof( uint32_t ) ); + } + + void AppendU64( std::vector &out, uint64_t value ) + { + uint8_t bytes[sizeof( uint64_t )]; + std::memcpy( bytes, &value, sizeof( uint64_t ) ); + out.insert( out.end(), bytes, bytes + sizeof( uint64_t ) ); + } + + void AppendString( std::vector &out, const std::string &value ) + { + AppendU32( out, static_cast( value.size() ) ); + out.insert( out.end(), value.begin(), value.end() ); + } + + /// Length-prefixed byte section with an 8-byte (uint64) length prefix -- + /// used for the per-record preQuantizeBytes/quantizedBytes sections. + void AppendBytesU64( std::vector &out, const std::vector &value ) + { + AppendU64( out, static_cast( value.size() ) ); + out.insert( out.end(), value.begin(), value.end() ); + } + + /// Length-prefixed byte section with a 4-byte (uint32) length prefix -- + /// used for the trailing combinedHash section. + void AppendBytesU32( std::vector &out, const std::vector &value ) + { + AppendU32( out, static_cast( value.size() ) ); + out.insert( out.end(), value.begin(), value.end() ); + } + + // ── Bounds-checked read helpers ───────────────────────────────── + // Every helper validates `offset + needed <= bytes.size()` (and, for + // length-prefixed sections, the declared length against + // kMaxSectionBytes) BEFORE reading or allocating (T-10-02, T-10-01a). + + bool ReadU32( const std::vector &bytes, size_t &offset, uint32_t &value ) + { + if ( offset + sizeof( uint32_t ) > bytes.size() ) + { + return false; + } + std::memcpy( &value, bytes.data() + offset, sizeof( uint32_t ) ); + offset += sizeof( uint32_t ); + return true; + } + + bool ReadU64( const std::vector &bytes, size_t &offset, uint64_t &value ) + { + if ( offset + sizeof( uint64_t ) > bytes.size() ) + { + return false; + } + std::memcpy( &value, bytes.data() + offset, sizeof( uint64_t ) ); + offset += sizeof( uint64_t ); + return true; + } + + bool ReadString( const std::vector &bytes, size_t &offset, std::string &value ) + { + uint32_t len = 0; + if ( !ReadU32( bytes, offset, len ) ) + { + return false; + } + if ( len > kMaxSectionBytes || offset + len > bytes.size() ) + { + return false; + } + value.assign( reinterpret_cast( bytes.data() + offset ), len ); + offset += len; + return true; + } + + bool ReadBytesU64( const std::vector &bytes, size_t &offset, std::vector &value ) + { + uint64_t len = 0; + if ( !ReadU64( bytes, offset, len ) ) + { + return false; + } + if ( len > kMaxSectionBytes || offset + len > bytes.size() ) + { + return false; + } + value.assign( bytes.begin() + static_cast( offset ), + bytes.begin() + static_cast( offset + len ) ); + offset += len; + return true; + } + + bool ReadBytesU32( const std::vector &bytes, size_t &offset, std::vector &value ) + { + uint32_t len = 0; + if ( !ReadU32( bytes, offset, len ) ) + { + return false; + } + if ( len > kMaxSectionBytes || offset + len > bytes.size() ) + { + return false; + } + value.assign( bytes.begin() + static_cast( offset ), + bytes.begin() + static_cast( offset + len ) ); + offset += len; + return true; + } + + bool ReadFixedRegion( const std::vector &bytes, size_t &offset, size_t regionSize, + std::vector ®ion ) + { + if ( offset + regionSize > bytes.size() ) + { + return false; + } + region.assign( bytes.begin() + static_cast( offset ), + bytes.begin() + static_cast( offset + regionSize ) ); + offset += regionSize; + return true; + } + + } // namespace + + std::vector SerializeCaptureFile( const CaptureFile &capture ) + { + std::vector out; + out.insert( out.end(), kMagic, kMagic + sizeof( kMagic ) ); + + AppendString( out, capture.machineIdTag ); + AppendString( out, capture.fixtureLabel ); + + AppendU32( out, static_cast( capture.artifacts.size() ) ); + + static const std::vector kNoRecords; + + for ( size_t i = 0; i < capture.artifacts.size(); ++i ) + { + const auto artifactBytes = sgns::sgprocessing::SerializeArtifact( capture.artifacts[i] ); + out.insert( out.end(), artifactBytes.begin(), artifactBytes.end() ); + + const std::vector &records = + ( i < capture.rawRecordsPerArtifact.size() ) ? capture.rawRecordsPerArtifact[i] : kNoRecords; + + AppendU32( out, static_cast( records.size() ) ); + for ( const auto &record : records ) + { + AppendBytesU64( out, record.preQuantizeBytes ); + AppendBytesU64( out, record.quantizedBytes ); + } + } + + const auto manifestBytes = sgns::sgprocessing::SerializeManifest( capture.manifest ); + out.insert( out.end(), manifestBytes.begin(), manifestBytes.end() ); + + AppendBytesU32( out, capture.combinedHash ); + + return out; + } + + bool DeserializeCaptureFile( const std::vector &bytes, CaptureFile &out ) + { + size_t offset = 0; + + if ( bytes.size() < sizeof( kMagic ) || std::memcmp( bytes.data(), kMagic, sizeof( kMagic ) ) != 0 ) + { + return false; + } + offset += sizeof( kMagic ); + + CaptureFile parsed; + + if ( !ReadString( bytes, offset, parsed.machineIdTag ) ) + { + return false; + } + if ( !ReadString( bytes, offset, parsed.fixtureLabel ) ) + { + return false; + } + + uint32_t artifactCount = 0; + if ( !ReadU32( bytes, offset, artifactCount ) ) + { + return false; + } + // Minimum wire size for one artifact entry: the fixed artifact region + // plus its 4-byte recordCount field (records themselves are validated + // individually below). + const uint64_t kMinBytesPerArtifact = sgns::sgprocessing::ARTIFACT_SERIALIZED_SIZE + sizeof( uint32_t ); + if ( !CountFitsRemaining( artifactCount, kMinBytesPerArtifact, bytes.size() - offset ) ) + { + return false; + } + + parsed.artifacts.reserve( artifactCount ); + parsed.rawRecordsPerArtifact.reserve( artifactCount ); + + for ( uint32_t i = 0; i < artifactCount; ++i ) + { + std::vector artifactRegion; + if ( !ReadFixedRegion( bytes, offset, sgns::sgprocessing::ARTIFACT_SERIALIZED_SIZE, artifactRegion ) ) + { + return false; + } + sgns::sgprocessing::Artifact artifact{}; + if ( !sgns::sgprocessing::DeserializeArtifact( artifactRegion, artifact ) ) + { + return false; + } + + uint32_t recordCount = 0; + if ( !ReadU32( bytes, offset, recordCount ) ) + { + return false; + } + // Minimum wire size for one record: two 8-byte length prefixes + // (the byte payloads themselves are validated individually below). + constexpr uint64_t kMinBytesPerRecord = 2 * sizeof( uint64_t ); + if ( !CountFitsRemaining( recordCount, kMinBytesPerRecord, bytes.size() - offset ) ) + { + return false; + } + + std::vector records; + records.reserve( recordCount ); + for ( uint32_t r = 0; r < recordCount; ++r ) + { + CaptureRecord record; + if ( !ReadBytesU64( bytes, offset, record.preQuantizeBytes ) ) + { + return false; + } + if ( !ReadBytesU64( bytes, offset, record.quantizedBytes ) ) + { + return false; + } + records.push_back( std::move( record ) ); + } + + parsed.artifacts.push_back( artifact ); + parsed.rawRecordsPerArtifact.push_back( std::move( records ) ); + } + + std::vector manifestRegion; + if ( !ReadFixedRegion( bytes, offset, sgns::sgprocessing::MANIFEST_SERIALIZED_SIZE, manifestRegion ) ) + { + return false; + } + if ( !sgns::sgprocessing::DeserializeManifest( manifestRegion, parsed.manifest ) ) + { + return false; + } + + if ( !ReadBytesU32( bytes, offset, parsed.combinedHash ) ) + { + return false; + } + + out = std::move( parsed ); + return true; + } + +} // namespace sgns::sgproccapture diff --git a/tools/capture/capture_file_format.hpp b/tools/capture/capture_file_format.hpp new file mode 100644 index 0000000..62427c7 --- /dev/null +++ b/tools/capture/capture_file_format.hpp @@ -0,0 +1,101 @@ +/** + * Capture file binary format for Phase 10: Capture Harness & Diff Tool. + * + * Wraps the existing artifact_serializer.hpp binary convention + * (SerializeArtifact/SerializeManifest, called unmodified) for the + * metadata+hash portion, and appends one new length-prefixed raw-bytes + * section per rawOutputCapture invocation (D-01 -- capture files must not + * invent a second serialization convention alongside the one that already + * exists). Fixed-field regions (artifact, manifest) keep their existing + * little-endian, fixed-offset layout unchanged; the new sections added by + * this file are little-endian, length-prefixed, variable-length -- mirroring + * artifact_serializer.hpp's own "fixed-field, little-endian" doc convention + * where a fixed layout applies, and falling back to explicit length + * prefixes only where the data itself is inherently variable-length. + * + * Binary layout (little-endian throughout): + * [4] magic "SGC1" + * [4] machineIdTag byte length (uint32) + that many UTF-8 bytes + * [4] fixtureLabel byte length (uint32) + that many UTF-8 bytes + * [4] artifactCount (uint32) + * per artifact: + * [ARTIFACT_SERIALIZED_SIZE] SerializeArtifact(artifact) bytes, unmodified + * [4] recordCount for this artifact (uint32) + * per record: + * [8] preQuantizeBytes.size() (uint64) + that many raw bytes + * [8] quantizedBytes.size() (uint64) + that many raw bytes + * [MANIFEST_SERIALIZED_SIZE] SerializeManifest(manifest) bytes, unmodified + * [4] combinedHash.size() (uint32) + that many raw bytes + * + * DeserializeCaptureFile validates every declared length/count against the + * bytes actually remaining in the input buffer, and rejects (returns false) + * any single declared length exceeding kMaxSectionBytes (1 GiB), BEFORE + * allocating or reading that many bytes (T-10-02). Never throws; returns + * false rather than partially populating `out` on any malformed, truncated, + * or oversized input -- mirroring DeserializeArtifact/DeserializeManifest's + * existing bool-return-false-on-malformed-input convention (T-10-01a). + * + * @brief Capture file binary format (per-run raw output bytes + hashes + manifest) + */ +#ifndef SGPROCMGR_CAPTURE_FILE_FORMAT_HPP +#define SGPROCMGR_CAPTURE_FILE_FORMAT_HPP + +#include +#include +#include +#include "artifacts/artifact_types.hpp" +#include "artifacts/execution_manifest.hpp" + +namespace sgns::sgproccapture +{ + + /// One rawOutputCapture invocation's worth of bytes -- either a per-chunk + /// capture (paired with one entry of Artifact::chunkHashes) or the trailing + /// combined-hash capture (paired with Artifact::contentHash), in call order. + struct CaptureRecord + { + std::vector preQuantizeBytes; ///< Bytes offered to rawOutputCapture before Quantize*Buffer ran + std::vector quantizedBytes; ///< Bytes offered to rawOutputCapture after Quantize*Buffer ran (identity stub in Phase 10) + }; + + /// A single capture run: machine identity, fixture label, every output + /// artifact + the execution manifest from the run (via the existing + /// artifact_serializer.hpp convention, unmodified), plus the raw + /// pre-/post-quantization bytes captured at every rawOutputCapture call site. + struct CaptureFile + { + std::string machineIdTag; ///< Hostname + OS (D-03), e.g. "MacBook-Pro-M2 / macOS 15.1" + std::string fixtureLabel; ///< e.g. "render-happy-path" or "mnn-float" (D-02) + + std::vector artifacts; ///< One per job output + + /// Index-aligned with `artifacts`: rawRecordsPerArtifact[i] is the ordered + /// list of every rawOutputCapture call that fed hashes for artifacts[i]. + /// For a single-output job, records[0 .. artifacts[i].chunkHashCount - 1] + /// correspond 1:1 to artifacts[i].chunkHashes[0 .. chunkHashCount - 1], and + /// an optional trailing record (if present) corresponds to + /// artifacts[i].contentHash -- this pairing is what Wave 3's + /// capture_harness self-check (CAPT-02) verifies. + std::vector> rawRecordsPerArtifact; + + sgns::sgprocessing::ExecutionManifest manifest; ///< The job's execution manifest + + std::vector combinedHash; ///< The job's ProcessOutput.combinedHash + }; + + /// Serialize a CaptureFile to bytes: SerializeArtifact/SerializeManifest calls, + /// unmodified, for the metadata+hash portion, plus one new length-prefixed + /// raw-bytes section per rawOutputCapture invocation (D-01). + /// @return Serialized bytes per the layout documented above. + std::vector SerializeCaptureFile( const CaptureFile &capture ); + + /// Deserialize bytes back into a CaptureFile. Never throws; returns false + /// (without partially populating `out`) on wrong/missing magic, any length + /// or count field pointing past the buffer's end or exceeding the 1 GiB cap, + /// or a truncated SerializeArtifact/SerializeManifest region (T-10-01a, T-10-02). + /// @return true on success and a fully populated `out`; false otherwise. + bool DeserializeCaptureFile( const std::vector &bytes, CaptureFile &out ); + +} // namespace sgns::sgproccapture + +#endif // SGPROCMGR_CAPTURE_FILE_FORMAT_HPP diff --git a/tools/capture/capture_harness.cpp b/tools/capture/capture_harness.cpp new file mode 100644 index 0000000..7275c08 --- /dev/null +++ b/tools/capture/capture_harness.cpp @@ -0,0 +1,527 @@ +/** + * capture_harness -- standalone CLI tool (Phase 10, Plan 10-05, CAPT-01/02/03). + * + * Runs a Phase 09 fixture --repeat N times via ProcessingManager::Process()'s + * 5-argument ExecutionContext overload, captures per-run raw output bytes via + * ExecutionContext::rawOutputCapture, independently re-hashes every captured + * buffer to prove it is the literal pre-hash bytes production hashing saw + * (CAPT-02 self-check), and verifies same-node stability across all N runs + * (CAPT-03/D-04) before writing a single .cap file. Writes NO file if either + * check fails (D-05). + * + * Not CTest-gated (Pattern 5) -- a meaningful cross-machine pass/fail needs + * Phase 11's physical machines. + * + * Usage: + * capture_harness --fixture-root --fixture --label + * [--repeat N] [--output-dir ] [--model-input-source ] + * [--write-render-vertex-fixture] + * + * @brief Capture harness CLI (runs a fixture N times, self-checks, writes a .cap file) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if defined( __APPLE__ ) +#include +#endif + +#include +#include +#include +#include + +#include "capture_file_format.hpp" + +namespace +{ + /// Hand-rolled CLI argument bundle -- no new parsing dependency, per + /// Plan 10-CONTEXT.md's discretion note. + struct CliArgs + { + std::string fixtureRoot; + std::string fixture; + std::string label; + int repeat = 3; + std::string outputDir = "."; + std::string modelInputSource; + bool writeRenderVertexFixture = false; + }; + + void PrintUsage() + { + std::cerr << "Usage: capture_harness --fixture-root --fixture " + "--label [--repeat N] [--output-dir ] " + "[--model-input-source ] [--write-render-vertex-fixture]\n"; + } + + /// Parses argv into CliArgs. + /// @return true on success; false (with an error already printed) on any parse failure. + bool ParseArgs( int argc, char **argv, CliArgs &out ) + { + for ( int i = 1; i < argc; ++i ) + { + std::string arg = argv[i]; + if ( arg == "--fixture-root" && i + 1 < argc ) + { + out.fixtureRoot = argv[++i]; + } + else if ( arg == "--fixture" && i + 1 < argc ) + { + out.fixture = argv[++i]; + } + else if ( arg == "--label" && i + 1 < argc ) + { + out.label = argv[++i]; + } + else if ( arg == "--repeat" && i + 1 < argc ) + { + out.repeat = std::atoi( argv[++i] ); + } + else if ( arg == "--output-dir" && i + 1 < argc ) + { + out.outputDir = argv[++i]; + } + else if ( arg == "--model-input-source" && i + 1 < argc ) + { + out.modelInputSource = argv[++i]; + } + else if ( arg == "--write-render-vertex-fixture" ) + { + out.writeRenderVertexFixture = true; + } + else + { + std::cerr << "capture_harness: unrecognized or incomplete argument: " << arg << "\n"; + return false; + } + } + + if ( out.fixtureRoot.empty() || out.fixture.empty() || out.label.empty() ) + { + std::cerr << "capture_harness: --fixture-root, --fixture, and --label are required\n"; + return false; + } + if ( out.repeat < 2 ) + { + std::cerr << "capture_harness: --repeat must be >= 2 (CAPT-03/D-04 requires at least " + "two runs to self-check stability), got " + << out.repeat << "\n"; + return false; + } + return true; + } + + /// Standalone (no dependency -- this is CLI tooling, not a GTest + /// binary) local copy of processing_conformance_fixture.hpp's + /// PatchJsonUrisToAbsolute, using --fixture-root in place of that function's + /// bin_path parameter. + std::string PatchJsonUrisToAbsolute( const std::string &jsonStr, const std::string &fixtureRoot ) + { + std::string normalizedRoot = fixtureRoot; + for ( auto &c : normalizedRoot ) + { + if ( c == '\\' ) + { + c = '/'; + } + } + if ( !normalizedRoot.empty() && normalizedRoot.back() != '/' ) + { + normalizedRoot += '/'; + } + + std::string result; + std::regex relativeFileUriPattern( R"delim("(file://(?!/)(?![A-Za-z]:)[^"]+)")delim" ); + size_t lastPos = 0; + std::sregex_iterator iter( jsonStr.begin(), jsonStr.end(), relativeFileUriPattern ); + std::sregex_iterator end; + + while ( iter != end ) + { + result += jsonStr.substr( lastPos, iter->position() - lastPos ); + + std::string originalUri = ( *iter )[1].str(); + std::string relativePath = originalUri.substr( 7 ); // skip "file://" + result += "\"file://" + normalizedRoot + relativePath + "\""; + + lastPos = iter->position() + iter->length(); + ++iter; + } + result += jsonStr.substr( lastPos ); + + return result; + } + + /// Writes the render-pass-happy-path fixture's vertex data (3 scalar floats), + /// mirroring processing_dispatch_test.cpp's WriteHappyPathVertexData() exactly -- + /// this raw binary is never checked into source control. + /// @return true on success; false (with an error already printed) on failure. + bool WriteHappyPathVertexData( const std::string &fixtureRoot ) + { + std::error_code ec; + std::filesystem::path dir = std::filesystem::path( fixtureRoot ) / "processing_dispatch"; + std::filesystem::create_directories( dir, ec ); + if ( ec ) + { + std::cerr << "capture_harness: failed to create directory " << dir.string() << ": " << ec.message() + << "\n"; + return false; + } + + std::filesystem::path file = dir / "happy-path-vertex-data.raw"; + std::ofstream stream( file, std::ios::binary ); + if ( !stream.is_open() ) + { + std::cerr << "capture_harness: failed to open " << file.string() << " for writing\n"; + return false; + } + + float values[3] = { -0.5f, 0.0f, 0.5f }; + stream.write( reinterpret_cast( values ), sizeof( values ) ); + return true; + } + + /// Compile-time platform name (D-03 -- hostname + OS only, no GPU vendor/driver detail). + const char *PlatformName() + { +#if defined( _WIN32 ) + return "Windows"; +#elif defined( __ANDROID__ ) + return "Android"; +#elif defined( __APPLE__ ) +#if defined( TARGET_OS_IPHONE ) && TARGET_OS_IPHONE + return "iOS"; +#else + return "macOS"; +#endif +#elif defined( __linux__ ) + return "Linux"; +#else + return "Unknown"; +#endif + } + + /// Machine-identity tag (D-03): " / ". + std::string MachineIdTag() + { + boost::system::error_code ec; + std::string hostname = boost::asio::ip::host_name( ec ); + if ( ec || hostname.empty() ) + { + hostname = "unknown-host"; + } + return hostname + " / " + PlatformName(); + } + + /// Sanitizes a machine-identity tag for filesystem safety (D-02): spaces and '/' + /// become '-'. + std::string SanitizeForFilename( const std::string &tag ) + { + std::string result = tag; + for ( auto &c : result ) + { + if ( c == ' ' || c == '/' ) + { + c = '-'; + } + } + return result; + } + + /// UTC timestamp formatted yyyymmddThhmmss (D-02). + std::string UtcTimestampNow() + { + auto now = std::chrono::system_clock::now(); + std::time_t t = std::chrono::system_clock::to_time_t( now ); + std::tm tmUtc{}; +#if defined( _WIN32 ) + gmtime_s( &tmUtc, &t ); +#else + gmtime_r( &t, &tmUtc ); +#endif + std::ostringstream oss; + oss << std::put_time( &tmUtc, "%Y%m%dT%H%M%S" ); + return oss.str(); + } + + /// One run's captured data: the structured ProcessOutput plus every + /// rawOutputCapture record collected during that run, in call order. + struct IterationResult + { + sgns::sgprocessing::ProcessOutput output; + std::vector records; + }; + + /// CAPT-02 self-check: independently re-hashes every captured buffer and confirms + /// it equals the paired chunk/combined hash from the same run -- this is the + /// literal proof that captured bytes are the same bytes production hashing + /// actually saw, not a downstream copy (Pitfall 6). + /// @return true if every check passes; false (with an error already printed) otherwise. + bool SelfCheckCapturedBytes( const sgns::sgprocessing::Artifact &artifact, + const std::vector &records, + int iterationIndex ) + { + if ( records.size() < static_cast( artifact.chunkHashCount ) ) + { + std::cerr << "capture_harness: iteration " << iterationIndex << " self-check failed -- captured " + << records.size() << " records but artifact declares " << artifact.chunkHashCount + << " chunk hashes\n"; + return false; + } + + for ( uint32_t j = 0; j < artifact.chunkHashCount; ++j ) + { + auto hash = sgns::sgprocmanagersha::sha256( records[j].quantizedBytes.data(), + records[j].quantizedBytes.size() ); + if ( !std::equal( hash.begin(), hash.end(), artifact.chunkHashes[j] ) ) + { + std::cerr << "capture_harness: iteration " << iterationIndex + << " self-check failed -- re-hashed captured chunk " << j + << " does not match artifact.chunkHashes[" << j << "]\n"; + return false; + } + } + + if ( records.size() == static_cast( artifact.chunkHashCount ) + 1 ) + { + auto hash = sgns::sgprocmanagersha::sha256( records.back().quantizedBytes.data(), + records.back().quantizedBytes.size() ); + if ( !std::equal( hash.begin(), hash.end(), artifact.contentHash ) ) + { + std::cerr << "capture_harness: iteration " << iterationIndex + << " self-check failed -- re-hashed trailing combined-level capture does not " + "match artifact.contentHash\n"; + return false; + } + } + + return true; + } + + /// CAPT-03/D-04/D-05 stability check: compares iteration 0's contentHash, + /// chunkHashes, and combinedHash against every other iteration's same fields. + /// @return true if every iteration matches iteration 0; false (with an error + /// already printed) on any divergence. + bool CheckStability( const std::vector &iterations ) + { + const auto &baseArtifact = iterations[0].output.artifacts[0]; + const auto &baseCombined = iterations[0].output.combinedHash; + + for ( size_t i = 1; i < iterations.size(); ++i ) + { + const auto &curArtifact = iterations[i].output.artifacts[0]; + + if ( !std::equal( baseArtifact.contentHash, + baseArtifact.contentHash + sgns::sgprocessing::SHA256_HASH_SIZE, + curArtifact.contentHash ) ) + { + std::cerr << "capture_harness: instability detected -- iteration " << i + << "'s contentHash diverged from iteration 0's; aborting, no capture file written\n"; + return false; + } + + if ( baseArtifact.chunkHashCount != curArtifact.chunkHashCount ) + { + std::cerr << "capture_harness: instability detected -- iteration " << i << "'s chunkHashCount (" + << curArtifact.chunkHashCount << ") diverged from iteration 0's (" + << baseArtifact.chunkHashCount << "); aborting, no capture file written\n"; + return false; + } + + for ( uint32_t j = 0; j < baseArtifact.chunkHashCount; ++j ) + { + if ( !std::equal( baseArtifact.chunkHashes[j], + baseArtifact.chunkHashes[j] + sgns::sgprocessing::SHA256_HASH_SIZE, + curArtifact.chunkHashes[j] ) ) + { + std::cerr << "capture_harness: instability detected -- iteration " << i << "'s chunkHashes[" + << j << "] diverged from iteration 0's; aborting, no capture file written\n"; + return false; + } + } + + if ( iterations[i].output.combinedHash != baseCombined ) + { + std::cerr << "capture_harness: instability detected -- iteration " << i + << "'s combinedHash diverged from iteration 0's; aborting, no capture file written\n"; + return false; + } + } + + return true; + } + +} // namespace + +int main( int argc, char **argv ) +{ + CliArgs args; + if ( !ParseArgs( argc, argv, args ) ) + { + PrintUsage(); + return 1; + } + + if ( args.writeRenderVertexFixture ) + { + if ( !WriteHappyPathVertexData( args.fixtureRoot ) ) + { + return 1; + } + } + + std::filesystem::path fixturePath = std::filesystem::path( args.fixtureRoot ) / args.fixture; + std::ifstream fixtureStream( fixturePath ); + if ( !fixtureStream.is_open() ) + { + std::cerr << "capture_harness: could not open fixture file " << fixturePath.string() << "\n"; + return 1; + } + std::string rawJson( ( std::istreambuf_iterator( fixtureStream ) ), std::istreambuf_iterator() ); + if ( rawJson.empty() ) + { + std::cerr << "capture_harness: fixture file " << fixturePath.string() << " is empty\n"; + return 1; + } + + std::string patchedJson = PatchJsonUrisToAbsolute( rawJson, args.fixtureRoot ); + + std::vector iterations; + iterations.reserve( static_cast( args.repeat ) ); + + for ( int i = 0; i < args.repeat; ++i ) + { + auto mgrResult = sgns::sgprocessing::ProcessingManager::Create( patchedJson ); + if ( !mgrResult.has_value() ) + { + std::cerr << "capture_harness: iteration " << i << ": ProcessingManager::Create failed\n"; + return 1; + } + auto manager = mgrResult.value(); + + auto processingData = manager->GetProcessingData(); + const auto &passes = processingData.get_passes(); + if ( passes.empty() ) + { + std::cerr << "capture_harness: iteration " << i << ": fixture has no passes\n"; + return 1; + } + + sgns::ModelNode modelNode; + auto modelOpt = passes[0].get_model(); + if ( modelOpt.has_value() ) + { + auto model = modelOpt.value(); + const auto &inputNodes = model.get_input_nodes(); + if ( inputNodes.empty() ) + { + std::cerr << "capture_harness: iteration " << i << ": model has no input nodes\n"; + return 1; + } + modelNode = inputNodes[0]; + } + else + { + if ( args.modelInputSource.empty() ) + { + std::cerr << "capture_harness: fixture's pass has no model -- pass " + "--model-input-source (e.g. input:renderInput)\n"; + return 1; + } + modelNode.set_source( args.modelInputSource ); + } + + sgns::sgprocessing::ExecutionContext execCtx; + execCtx.cancelToken.SetCallback( []() {} ); + + std::vector captured; + execCtx.rawOutputCapture = [&captured]( const std::vector &quantizedBytes, + const std::vector &preQuantizeBytes ) + { + sgns::sgproccapture::CaptureRecord record; + record.quantizedBytes = quantizedBytes; + record.preQuantizeBytes = preQuantizeBytes; + captured.push_back( std::move( record ) ); + }; + + auto ioc = std::make_shared(); + std::vector> chunkhashes; + std::vector outputLocations; + + auto processResult = manager->Process( ioc, chunkhashes, modelNode, outputLocations, execCtx ); + if ( !processResult.has_value() ) + { + std::cerr << "capture_harness: iteration " << i + << ": Process() failed: " << processResult.error().message() << "\n"; + return 1; + } + + auto &output = processResult.value(); + if ( output.artifacts.empty() ) + { + std::cerr << "capture_harness: iteration " << i << ": Process() produced no artifacts\n"; + return 1; + } + + if ( !SelfCheckCapturedBytes( output.artifacts[0], captured, i ) ) + { + return 1; + } + + iterations.push_back( IterationResult{ std::move( output ), std::move( captured ) } ); + } + + if ( !CheckStability( iterations ) ) + { + return 1; + } + + sgns::sgproccapture::CaptureFile captureFile; + captureFile.machineIdTag = MachineIdTag(); + captureFile.fixtureLabel = args.label; + captureFile.artifacts = { iterations[0].output.artifacts[0] }; + captureFile.rawRecordsPerArtifact = { iterations[0].records }; + captureFile.manifest = iterations[0].output.manifest; + captureFile.combinedHash = iterations[0].output.combinedHash; + + auto serialized = sgns::sgproccapture::SerializeCaptureFile( captureFile ); + + std::error_code ec; + std::filesystem::create_directories( args.outputDir, ec ); // no-op if it already exists or is "." + + std::string filename = args.label + "_" + SanitizeForFilename( captureFile.machineIdTag ) + "_" + + UtcTimestampNow() + ".cap"; + std::filesystem::path outputPath = std::filesystem::path( args.outputDir ) / filename; + + std::ofstream outFile( outputPath, std::ios::binary ); + if ( !outFile.is_open() ) + { + std::cerr << "capture_harness: failed to open output file " << outputPath.string() << " for writing\n"; + return 1; + } + outFile.write( reinterpret_cast( serialized.data() ), + static_cast( serialized.size() ) ); + outFile.close(); + + std::cout << "capture_harness: wrote " << outputPath.string() << " (" << serialized.size() << " bytes, " + << args.repeat << "/" << args.repeat << " stable runs, " + << iterations[0].output.artifacts[0].chunkHashCount << " chunk hashes self-checked)\n"; + + return 0; +}