diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b2c05ef..ed227683 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,7 @@ option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OF option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF) option(Idefix_WERROR "Treat compiler warnings as errors" OFF) option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF) +option(Idefix_SUPPRESS_FMA "Disable FMA (fused multiply-add) contraction/codegen. Useful for code validation across architectures." OFF) set(Idefix_PROBLEM_DIR "${CMAKE_BINARY_DIR}" CACHE STRING "Problem directory to build for.") set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag") set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file") @@ -43,6 +44,7 @@ include(AddIdefixSource) include(SetIdefixProperty) include(SetRequiredBuildSettingsForGCC8) include(CheckHdf5ParallelSupport) +include(SuppressFMA) #Idefix requires Cuda Lambdas (experimental) if(Kokkos_ENABLE_CUDA) @@ -58,6 +60,8 @@ include_directories(${Kokkos_INCLUDE_DIRS_RET}) # Add Idefix CXX Flags add_compile_options(${Idefix_CXX_FLAGS}) + + # Add filesystem libraries for GCC8 set_required_build_settings_for_GCC8() @@ -181,15 +185,15 @@ endif() if(Idefix_EVOLVE_VECTOR_POTENTIAL) add_compile_definitions("EVOLVE_VECTOR_POTENTIAL") endif() -#update version.hpp if possible + +# determine idefix version from git git_describe(GIT_SHA1) set(Idefix_VERSION ${Idefix_VERSION_MAJOR}.${Idefix_VERSION_MINOR}.${Idefix_VERSION_PATCH}-${GIT_SHA1}) -file(WRITE ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_GIT_COMMIT \"${GIT_SHA1}\"\n") -file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_MAJOR \"${Idefix_VERSION_MAJOR}\"\n") -file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_MINOR \"${Idefix_VERSION_MINOR}\"\n") -file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_PATCH \"${Idefix_VERSION_PATCH}\"\n") -file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION \"${Idefix_VERSION}\"\n") - +configure_file( + ${CMAKE_SOURCE_DIR}/src/version.h.in + ${CMAKE_BINARY_DIR}/build/generated/version.h + @ONLY +) if(NOT ${Idefix_DEFS} STREQUAL "definitions.hpp") add_compile_definitions("DEFINITIONS_FILE=\"${Idefix_DEFS}\"") @@ -269,6 +273,21 @@ target_include_directories(idefix PUBLIC target_link_libraries(idefix Kokkos::kokkos) +# Generate header with compiler information (name, flags, path) +configure_file( + ${CMAKE_SOURCE_DIR}/src/compiler_info.h.in + ${CMAKE_BINARY_DIR}/build/generated/compiler_info.h + @ONLY +) +# Make sure the generated header is on the include path +target_include_directories(idefix PRIVATE ${CMAKE_BINARY_DIR}/build/generated) + +# disable FMA if needed +if(Idefix_SUPPRESS_FMA) + message(STATUS "FMA (fused multiply-add) contraction/codegen is disabled") + target_suppress_fma(idefix) +endif() + message(STATUS "Idefix final configuration") if(Idefix_EVOLVE_VECTOR_POTENTIAL) message(STATUS " MHD: ${Idefix_MHD} (Vector potential)") diff --git a/cmake/SuppressFMA.cmake b/cmake/SuppressFMA.cmake new file mode 100644 index 00000000..18dc10d4 --- /dev/null +++ b/cmake/SuppressFMA.cmake @@ -0,0 +1,102 @@ +#[[============================================================================ + SuppressFMA.cmake + + Provides an option and a function to optionally disable fused + multiply-add (FMA) code generation / contraction for a Kokkos-based CXX + target. + + Kokkos wraps the real device compiler behind nvcc_wrapper (CUDA) or + hipcc (HIP), which makes CMAKE_CXX_COMPILER_ID report the *underlying + host* compiler (e.g. "GNU" or "Clang") instead of "NVIDIA" or "Clang + as HIP". Backend detection therefore relies on the Kokkos_ENABLE_* + variables exported by KokkosConfig.cmake / set by the Kokkos build, + and only falls back to CMAKE_CXX_COMPILER_ID for the plain host + compilers (no CUDA/HIP backend active): + + - Kokkos_ENABLE_HIP ON -> AMD HIP (hipcc, clang-based) : -ffp-contract=off + - Kokkos_ENABLE_CUDA ON -> NVIDIA nvcc (via nvcc_wrapper) : --fmad=false + - otherwise, CMAKE_CXX_COMPILER_ID selects among: + GNU : gcc/g++ + Intel : classic icc/icpc + IntelLLVM : Intel oneAPI icx/icpx + Clang : LLVM clang++ + AppleClang : Xcode clang++ + NVHPC : NVIDIA HPC SDK (nvc++), e.g. for OpenMPTarget/OpenACC + + Usage (after find_package(Kokkos) so Kokkos_ENABLE_* are defined): + include(SuppressFMA.cmake) + add_library(mylib source.cpp) + target_link_libraries(mylib PUBLIC Kokkos::kokkos) + target_suppress_fma(mylib) + +============================================================================]] + +include_guard(GLOBAL) + +# Determine the CXX FMA-suppression flags for the active Kokkos backend / +# CXX compiler. Returns the list of flags (possibly empty) via out_var. +function(_fma_suppression_flags_cxx out_var) + set(flags "") + set(id "${CMAKE_CXX_COMPILER_ID}") + + # Kokkos backend takes priority: nvcc_wrapper/hipcc hide the real + # device compiler from CMAKE_CXX_COMPILER_ID. + if(Kokkos_ENABLE_HIP) + set(flags "-ffp-contract=off") + + elseif(Kokkos_ENABLE_CUDA) + set(flags "--fmad=false") + + elseif(id STREQUAL "GNU") + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-mno-fma" IDEFIX_HAS_MNO_FMA) + set(flags "-ffp-contract=off") + if(IDEFIX_HAS_MNO_FMA) + list(APPEND flags "-mno-fma") + endif() + elseif(id MATCHES "^(Clang|AppleClang|CrayClang)$") + set(flags "-ffp-contract=off") + + elseif(id STREQUAL "Intel") + # Intel classic compiler + set(flags "-fp-model=precise" "-no-fma") + + elseif(id STREQUAL "IntelLLVM") + # Intel oneAPI compiler (clang-based) + set(flags "-ffp-contract=off" "-fp-model=strict") + + elseif(id STREQUAL "NVHPC") + # NVIDIA HPC SDK (formerly PGI), e.g. OpenMPTarget/OpenACC backend + set(flags "-Mnofma") # untested + endif() + + set(${out_var} "${flags}" PARENT_SCOPE) +endfunction() + +# target_suppress_fma() +# +# Applies compiler-specific FMA-suppression flags to 's CXX +# sources, but only when the SUPPRESS_FMA option is ON. Safe to call +# unconditionally. +function(target_suppress_fma target) + + if(NOT TARGET ${target}) + message(FATAL_ERROR "target_suppress_fma: '${target}' is not a target") + endif() + + _fma_suppression_flags_cxx(cxx_flags) + + if(cxx_flags) + foreach(flag IN LISTS cxx_flags) + target_compile_options(${target} PRIVATE + $<$:${flag}> + ) + endforeach() + else() + message(VERBOSE + "target_suppress_fma: no FMA-suppression flag known for " + "CXX compiler '${CMAKE_CXX_COMPILER_ID}' " + "(Kokkos_ENABLE_CUDA=${Kokkos_ENABLE_CUDA}, " + "Kokkos_ENABLE_HIP=${Kokkos_ENABLE_HIP}) (target ${target})") + endif() +endfunction() diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py index ac72d6dd..ffbd0de3 100644 --- a/pytools/idfx_test.py +++ b/pytools/idfx_test.py @@ -225,21 +225,19 @@ def _genCmakeCommand(self, definitionFile=""): if self.cuda: comm.append("-DKokkos_ENABLE_CUDA=ON") - # disable fmad operations on Cuda to make it compatible with CPU arithmetics - comm.append("-DIdefix_CXX_FLAGS=--fmad=false") # disable Async cuda malloc for tests performed on old UCX implementations comm.append("-DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF") if self.intel: # disable fmad operations on Cuda to make it compatible with CPU arithmetics - comm.append("-DIdefix_CXX_FLAGS=-fp-model=strict") comm.append("-DCMAKE_CXX_COMPILER=icpx") comm.append("-DCMAKE_C_COMPILER=icx") if self.hip: comm.append("-DKokkos_ENABLE_HIP=ON") - # disable fmad operations on HIP to make it compatible with CPU arithmetics - comm.append("-DIdefix_CXX_FLAGS=-ffp-contract=off") + + # disable FMA for testing so that we have the same results on CPU and GPU (otherwise, the results are not bitwise identical) + comm.append("-DIdefix_SUPPRESS_FMA=ON") # if we use single precision if self.single: diff --git a/src/compiler_info.h.in b/src/compiler_info.h.in new file mode 100644 index 00000000..140eafd7 --- /dev/null +++ b/src/compiler_info.h.in @@ -0,0 +1,17 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef COMPILER_INFO_H +#define COMPILER_INFO_H + +namespace CompilerInfo { + inline constexpr const char* name = "@CMAKE_CXX_COMPILER_ID@"; + inline constexpr const char* version = "@CMAKE_CXX_COMPILER_VERSION@"; + inline constexpr const char* path = "@CMAKE_CXX_COMPILER@"; +} + +#endif diff --git a/src/dataBlock/dumpToFile.cpp b/src/dataBlock/dumpToFile.cpp index f20dde83..305b417a 100644 --- a/src/dataBlock/dumpToFile.cpp +++ b/src/dataBlock/dumpToFile.cpp @@ -9,8 +9,8 @@ #include #include "../idefix.hpp" #include "dataBlock.hpp" -#include "version.hpp" #include "fluid.hpp" +#include "version.h" #define NAMESIZE 16 #define HEADERSIZE 128 @@ -78,7 +78,7 @@ void DataBlock::DumpToFile(std::string filebase) { // Write Header char header[HEADERSIZE]; - std::snprintf(header, HEADERSIZE, "Idefix %s Debug DataBlock", IDEFIX_VERSION); + std::snprintf(header, HEADERSIZE, "Idefix %s Debug DataBlock", VersionInfo::version); fwrite (header, sizeof(char), HEADERSIZE, fileHdl); // Write Vc diff --git a/src/input.cpp b/src/input.cpp index 33641495..6be052f6 100644 --- a/src/input.cpp +++ b/src/input.cpp @@ -18,8 +18,9 @@ #include "idefix.hpp" #include "input.hpp" -#include "version.hpp" #include "profiler.hpp" +#include "version.h" +#include "compiler_info.h" // Flag will be set if a signal has been received bool Input::abortRequested = false; @@ -415,7 +416,10 @@ void Input::PrintOptions() { } void Input::PrintVersion() { - idfx::cout << " Idefix version " << IDEFIX_VERSION << std::endl; - idfx::cout << " Built against Kokkos " << KOKKOS_VERSION << std::endl; - idfx::cout << " Compiled on " << __DATE__ << " at " << __TIME__ << std::endl; + idfx::cout << "Idefix version " << VersionInfo::version << std::endl; + idfx::cout << "Built against Kokkos " << KOKKOS_VERSION << std::endl; + idfx::cout << "Compiled on " << __DATE__ << " at " << __TIME__ << std::endl; + idfx::cout << "Compiler name: " << CompilerInfo::name << std::endl; + idfx::cout << "Compiler version: " << CompilerInfo::version << std::endl; + idfx::cout << "Compiler path: " << CompilerInfo::path << std::endl; } diff --git a/src/loop.hpp b/src/loop.hpp index 42192fa8..319ad855 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -85,7 +85,7 @@ inline void idefix_for(const std::string & NAME, idfx::pushRegion("idefix_for("+NAME+")"); #endif const int NI = IE - IB; - Kokkos::parallel_for(NAME, NI, + Kokkos::parallel_for(NAME, Kokkos::RangePolicy>(0, NI), KOKKOS_LAMBDA (const int& IDX) { int i = IDX; i += IB; @@ -112,7 +112,7 @@ inline void idefix_for(const std::string & NAME, const int NJ = JE - JB; const int NI = IE - IB; const int NJNI = NJ * NI; - Kokkos::parallel_for(NAME, NJNI, + Kokkos::parallel_for(NAME, Kokkos::RangePolicy>(0, NJNI), KOKKOS_LAMBDA (const int& IDX) { int j = IDX / NI; int i = IDX - j*NI; @@ -172,7 +172,7 @@ inline void idefix_for(const std::string & NAME, const int NI = IE - IB; const int NKNJNI = NK*NJ*NI; const int NJNI = NJ * NI; - Kokkos::parallel_for(NAME,NKNJNI, + Kokkos::parallel_for(NAME,Kokkos::RangePolicy>(0, NKNJNI), KOKKOS_LAMBDA (const int& IDX) { int k = IDX / NJNI; int j = (IDX - k*NJNI) / NI; @@ -259,7 +259,7 @@ inline void idefix_for(const std::string & NAME, const int NNNKNJNI = NN*NK*NJ*NI; const int NKNJNI = NK*NJ*NI; const int NJNI = NJ * NI; - Kokkos::parallel_for(NAME,NNNKNJNI, + Kokkos::parallel_for(NAME,Kokkos::RangePolicy>(0, NNNKNJNI), KOKKOS_LAMBDA (const int& IDX) { int n = IDX / NKNJNI; int k = (IDX - n*NKNJNI) / NJNI; diff --git a/src/output/dump.cpp b/src/output/dump.cpp index 96a40b54..d20eec26 100644 --- a/src/output/dump.cpp +++ b/src/output/dump.cpp @@ -20,11 +20,11 @@ #include #include #include "dump.hpp" -#include "version.hpp" #include "dataBlockHost.hpp" #include "gridHost.hpp" #include "output.hpp" #include "fluid.hpp" +#include "version.h" // Max size of array name #define NAMESIZE 16 @@ -880,7 +880,7 @@ int Dump::Write(Output& output) { char header[HEADERSIZE]; std::snprintf(header, HEADERSIZE, "Idefix %s Dump Data %s endian", - IDEFIX_VERSION, endian.c_str()); + VersionInfo::version, endian.c_str()); WriteString(fileHdl, header, HEADERSIZE); for(int dir = 0; dir < 3 ; dir++) { diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index 98e4c257..fb04e03d 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -20,12 +20,12 @@ #else error "Missing the header." #endif -#include "version.hpp" #include "idefix.hpp" #include "dataBlock.hpp" #include "gridHost.hpp" #include "output.hpp" #include "fluid.hpp" +#include "version.h" #define VTK_RECTILINEAR_GRID 14 #define VTK_STRUCTURED_GRID 35 @@ -385,7 +385,7 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { 2. Header ------------------------------------------- */ - ssheader << "Idefix " << IDEFIX_VERSION << " VTK Data" << std::endl; + ssheader << "Idefix " << VersionInfo::version << " VTK Data" << std::endl; /* ------------------------------------------ 3. File format diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index f0881900..7d559a6d 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -21,11 +21,11 @@ #endif #include "xdmf.hpp" -#include "version.hpp" #include "idefix.hpp" #include "dataBlockHost.hpp" #include "gridHost.hpp" #include "output.hpp" +#include "version.h" // Whether or not we write the time in the XDMF file #define WRITE_TIME @@ -526,7 +526,7 @@ void Xdmf::WriteHeader( dimstr = 1; - ssheader << "Idefix " << IDEFIX_VERSION << " XDMF Data"; + ssheader << "Idefix " << VersionInfo::version << " XDMF Data"; strspace = H5Screate_simple(1, &dimstr, NULL); string_type = H5Tcopy(H5T_C_S1); H5Tset_size(string_type, strlen( ssheader.str().c_str() )); diff --git a/src/version.h.in b/src/version.h.in new file mode 100644 index 00000000..5561658a --- /dev/null +++ b/src/version.h.in @@ -0,0 +1,19 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef VERSION_H +#define VERSION_H + +namespace VersionInfo { + inline constexpr const char* gitCommit = "@GIT_SHA1@"; + inline constexpr const char* versionMajor = "@Idefix_VERSION_MAJOR@"; + inline constexpr const char* versionMinor = "@Idefix_VERSION_MINOR@"; + inline constexpr const char* versionPatch = "@Idefix_VERSION_PATCH@"; + inline constexpr const char* version = "@Idefix_VERSION@"; +} + +#endif