Add additional nonlinear solvers and prepare to run polyfem with hybrid solver

.gitignore

@@ -7,4 +7,6 @@ tests/data
 *.code-workspace
 PolySolveOptions.cmake
 .vs
-__cmake_systeminformation
+__cmake_systeminformation
+scaling-experiment
+restart-experiments

CMakeLists.txt

@@ -96,6 +96,7 @@ option(POLYSOLVE_WITH_PARDISO       "Enable Pardiso library"
 option(POLYSOLVE_WITH_HYPRE         "Enable hypre"                                       ON)
 option(POLYSOLVE_WITH_AMGCL         "Use AMGCL"                                          ON)
 option(POLYSOLVE_WITH_SPECTRA       "Enable Spectra library"                             ON)
+option(POLYSOLVE_WITH_STRUMPACK     "Enable STRUMPACK (experimental CPUHybrid subdomain factorization)" OFF)
 
 # Sanitizer options
 option(POLYSOLVE_SANITIZE_ADDRESS   "Sanitize Address"                                  OFF)
@@ -304,11 +305,12 @@ if(POLYSOLVE_WITH_CUDA)
   target_link_libraries(polysolve PUBLIC CCCL::CCCL)
 endif()
 
-# Cuda runtime and cuSparse
+# Cuda runtime, cuSparse and cuBLAS (cuBLAS is used by GPUHybridSolver's
+# contact_patch_schwarz dense batched subdomain solve)
 if (POLYSOLVE_WITH_CUDA)
     find_package(CUDAToolkit REQUIRED)
-    target_link_libraries(polysolve PUBLIC CUDA::cudart CUDA::cusparse)
-    target_link_libraries(polysolve_linear PUBLIC CUDA::cudart CUDA::cusparse)
+    target_link_libraries(polysolve PUBLIC CUDA::cudart CUDA::cusparse CUDA::cublas)
+    target_link_libraries(polysolve_linear PUBLIC CUDA::cudart CUDA::cusparse CUDA::cublas)
 endif()
 
 # ------
@@ -432,6 +434,13 @@ if(POLYSOLVE_WITH_SPECTRA)
     target_compile_definitions(polysolve_linear PUBLIC POLYSOLVE_WITH_SPECTRA)
 endif()
 
+# STRUMPACK (BSD 3-Clause), used only by CPUHybrid's experimental use_strumpack
+if(POLYSOLVE_WITH_STRUMPACK)
+    include(strumpack)
+    target_link_libraries(polysolve_linear PRIVATE STRUMPACK::strumpack)
+    target_compile_definitions(polysolve_linear PUBLIC POLYSOLVE_WITH_STRUMPACK)
+endif()
+
 # cuSolver solvers
 if(POLYSOLVE_WITH_CUSOLVER)
     if (NOT POLYSOLVE_WITH_CUDA)

PolySolveOptions.cmake.sample

@@ -34,6 +34,7 @@
 # option(POLYSOLVE_WITH_HYPRE         "Enable hypre"                                ON)
 # option(POLYSOLVE_WITH_AMGCL         "Use AMGCL"                                   ON)
 # option(POLYSOLVE_WITH_SPECTRA       "Enable Spectra library"                      ON)
+# option(POLYSOLVE_WITH_STRUMPACK     "Enable STRUMPACK (experimental CPUHybrid subdomain factorization)" OFF)
 
 # Options for third-party libraries
 # option(EIGEN_WITH_MKL "Use Eigen with MKL" ON)

cmake/recipes/amgcl.cmake

@@ -39,6 +39,22 @@ function(amgcl_import_target)
     # Prefer Config mode before Module mode to prevent lib from loading its own FindXXX.cmake
     set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
 
+    # AMGCL's own build optionally calls find_package(MPI) to enable its
+    # distributed-memory backend. When nano-mpi is in play, CMAKE_FIND_PACKAGE_PREFER_CONFIG
+    # above makes that call skip Module mode entirely -- bypassing
+    # CMAKE_MODULE_PATH and nano-mpi's FindMPI shim -- and resolve via Config
+    # mode straight to a real system MPI instead (Homebrew's Open MPI ships
+    # an MPIConfig.cmake). Since MPI::MPI_C is a single GLOBAL imported
+    # target, that unconditionally overwrites its INTERFACE_INCLUDE_DIRECTORIES
+    # for everyone, including hypre, silently pointing their #include <mpi.h>
+    # at a real MPI nano-mpi cannot run against. AMGCL's distributed backend
+    # would need a real launcher anyway, which nano-mpi does not provide, so
+    # disable that find_package(MPI) call here rather than let it corrupt the
+    # shared target.
+    if(TARGET nanompi::nanompi)
+        set(CMAKE_DISABLE_FIND_PACKAGE_MPI TRUE)
+    endif()
+
     # Ready to include third-party lib
     include(CPM)
     CPMAddPackage(

cmake/recipes/argparse.cmake

@@ -0,0 +1,16 @@
+if(TARGET argparse::argparse)
+    return()
+endif()
+
+message(STATUS "Third-party: creating target 'argparse::argparse'")
+
+include(CPM)
+CPMAddPackage(
+    NAME argparse
+    GITHUB_REPOSITORY p-ranav/argparse
+    GIT_TAG v3.2
+)
+
+if(TARGET argparse)
+    set_target_properties(argparse PROPERTIES SYSTEM ON)
+endif()

cmake/recipes/metis.cmake

@@ -0,0 +1,69 @@
+# METIS (https://github.com/KarypisLab/METIS), with GKlib (https://github.com/KarypisLab/GKlib)
+# License: Apache 2.0
+
+if(TARGET METIS::metis)
+    return()
+endif()
+
+message(STATUS "Third-party: creating target 'METIS::metis'")
+
+# Neither project can be add_subdirectory()'d: METIS's CMake expects GKlib to
+# be installed under CMAKE_INSTALL_PREFIX already, and a metis.h that its
+# `make config` step fills in. So both are fetched as plain sources and
+# compiled here into a single static library.
+include(CPM)
+CPMAddPackage(
+    NAME gklib
+    GITHUB_REPOSITORY KarypisLab/GKlib
+    # Pinned: GKlib's master has since moved its sources into src/ and include/.
+    GIT_TAG 8bd6bad750b2b0d90800c632cf18e8ee93ad72d7
+    DOWNLOAD_ONLY YES
+)
+CPMAddPackage(
+    NAME metis
+    GITHUB_REPOSITORY KarypisLab/METIS
+    GIT_TAG v5.2.1
+    DOWNLOAD_ONLY YES
+)
+
+enable_language(C)
+
+# metis.h ships with its index and real widths commented out, for `make config`
+# to fill in. Fill in 32 bits, matching the int indices of the Eigen matrices
+# every caller here passes in. file(CONFIGURE) rewrites the header only when
+# its content changes, so a reconfigure does not trigger a rebuild.
+file(READ "${metis_SOURCE_DIR}/include/metis.h" METIS_HEADER)
+string(REPLACE "//#define IDXTYPEWIDTH 32" "#define IDXTYPEWIDTH 32" METIS_HEADER "${METIS_HEADER}")
+string(REPLACE "//#define REALTYPEWIDTH 32" "#define REALTYPEWIDTH 32" METIS_HEADER "${METIS_HEADER}")
+file(CONFIGURE OUTPUT "${metis_BINARY_DIR}/include/metis.h" CONTENT "${METIS_HEADER}" @ONLY)
+
+file(GLOB GKLIB_SOURCES "${gklib_SOURCE_DIR}/*.c")
+file(GLOB METIS_SOURCES "${metis_SOURCE_DIR}/libmetis/*.c")
+
+add_library(METIS_metis STATIC ${GKLIB_SOURCES} ${METIS_SOURCES})
+add_library(METIS::metis ALIAS METIS_metis)
+set_target_properties(METIS_metis PROPERTIES FOLDER third_party/metis)
+
+target_include_directories(METIS_metis
+    PUBLIC
+        "${metis_BINARY_DIR}/include"
+    PRIVATE
+        "${gklib_SOURCE_DIR}"
+        "${metis_SOURCE_DIR}/libmetis"
+)
+
+# The definitions GKlib's own build passes (see its GKlibSystem.cmake), minus
+# -march=native and -Werror.
+if(MSVC)
+    target_include_directories(METIS_metis PRIVATE "${gklib_SOURCE_DIR}/win32")
+    target_compile_definitions(METIS_metis PRIVATE WIN32 MSC _CRT_SECURE_NO_DEPRECATE USE_GKREGEX "__thread=__declspec(thread)")
+elseif(MINGW)
+    target_compile_definitions(METIS_metis PRIVATE USE_GKREGEX)
+else()
+    target_compile_definitions(METIS_metis PRIVATE LINUX _FILE_OFFSET_BITS=64)
+endif()
+target_compile_definitions(METIS_metis PRIVATE NDEBUG NDEBUG2)
+
+if(UNIX)
+    target_link_libraries(METIS_metis PRIVATE m)
+endif()

cmake/recipes/nanompi.cmake

@@ -14,6 +14,17 @@
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/nanompi-as-mpi/")
 list(REMOVE_DUPLICATES CMAKE_MODULE_PATH)
 
+# Promote to CACHE so the shim is visible in every directory scope, not just
+# ones that inherit this list(APPEND) through add_subdirectory(). A dependency
+# added from a scope that never saw it (e.g. AMGCL's own optional
+# find_package(MPI) call) would otherwise fall through to CMake's real
+# FindMPI, which unconditionally overwrites MPI::MPI_C's
+# INTERFACE_INCLUDE_DIRECTORIES on every invocation (see _MPI_create_imported_target
+# in CMake's own FindMPI.cmake) -- silently re-pointing everyone's
+# #include <mpi.h>, hypre included, at a real system MPI nano-mpi cannot
+# actually run against.
+set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" CACHE STRING "" FORCE)
+
 if(TARGET nanompi::nanompi)
     return()
 endif()

cmake/recipes/strumpack.cmake

@@ -0,0 +1,134 @@
+# STRUMPACK (https://github.com/pghysels/STRUMPACK)
+# License: BSD 3-Clause (LBNL)
+#
+# Sparse multifrontal LU, with optional low-rank (BLR, HSS) compression of its
+# frontal matrices. CPUHybridSolver can use it to factorize its subdomains
+# (the experimental use_strumpack option).
+
+if(TARGET STRUMPACK::strumpack)
+    return()
+endif()
+
+message(STATUS "Third-party: creating target 'STRUMPACK::strumpack'")
+
+# STRUMPACK's project() declares Fortran (for BLAS name mangling and its
+# Fortran interface), so it cannot even configure without a Fortran compiler.
+include(CheckLanguage)
+check_language(Fortran)
+if(NOT CMAKE_Fortran_COMPILER)
+    message(FATAL_ERROR
+        "STRUMPACK needs a Fortran compiler, and none was found. Install one "
+        "(e.g. gfortran) or turn POLYSOLVE_WITH_STRUMPACK off.")
+endif()
+
+include(metis)
+include(blas)
+include(lapack)
+
+include(CPM)
+CPMAddPackage(
+    NAME strumpack
+    GITHUB_REPOSITORY pghysels/STRUMPACK
+    GIT_TAG v8.0.0
+    DOWNLOAD_ONLY YES
+)
+
+# Unlike the other recipes, STRUMPACK is not add_subdirectory()'d: its CMake
+# only works as the top-level project. It finds its own modules and test
+# sources through CMAKE_SOURCE_DIR, which would point at polysolve instead, and
+# unconditionally adds its ~200 tests to CTest, which would land in polysolve's.
+# So it is configured and built in a CMake run of its own, and only the
+# resulting static library is imported below.
+set(STRUMPACK_INSTALL_DIR "${FETCHCONTENT_BASE_DIR}/strumpack-install")
+set(STRUMPACK_LIBRARY "${STRUMPACK_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}strumpack${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+# Follow polysolve's build type; with none set, STRUMPACK would be built
+# unoptimized.
+if(CMAKE_BUILD_TYPE)
+    set(STRUMPACK_BUILD_TYPE ${CMAKE_BUILD_TYPE})
+else()
+    set(STRUMPACK_BUILD_TYPE Release)
+endif()
+
+get_target_property(STRUMPACK_METIS_INCLUDE_DIR METIS::metis INTERFACE_INCLUDE_DIRECTORIES)
+
+set(STRUMPACK_CMAKE_ARGS
+    -DCMAKE_BUILD_TYPE=${STRUMPACK_BUILD_TYPE}
+    -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
+    -DCMAKE_INSTALL_LIBDIR=lib
+    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+    -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER}
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+    -DBUILD_SHARED_LIBS=OFF
+
+    # Each subdomain is factorized by a single nanompi rank -- a thread of
+    # this process -- so STRUMPACK runs sequentially inside it, with neither
+    # MPI nor OpenMP of its own.
+    -DSTRUMPACK_USE_MPI=OFF
+    -DSTRUMPACK_USE_OPENMP=OFF
+    -DSTRUMPACK_USE_CUDA=OFF
+    -DSTRUMPACK_USE_HIP=OFF
+    -DSTRUMPACK_USE_SYCL=OFF
+
+    # No optional third-party libraries: BLR and HSS compression are built in.
+    -DTPL_ENABLE_SLATE=OFF
+    -DTPL_ENABLE_PARMETIS=OFF
+    -DTPL_ENABLE_SCOTCH=OFF
+    -DTPL_ENABLE_PTSCOTCH=OFF
+    -DTPL_ENABLE_BPACK=OFF
+    -DTPL_ENABLE_ZFP=OFF
+    -DTPL_ENABLE_SZ3=OFF
+    -DTPL_ENABLE_MAGMA=OFF
+    -DTPL_ENABLE_KBLAS=OFF
+    -DTPL_ENABLE_COMBBLAS=OFF
+    -DTPL_ENABLE_PAPI=OFF
+    -DTPL_ENABLE_MATLAB=OFF
+
+    # METIS, the one required dependency, is the one built by the metis
+    # recipe. Setting METIS_INCLUDE_DIR outright keeps STRUMPACK's FindMETIS
+    # from settling on a system metis.h instead. The library itself is only
+    # needed once polysolve links, so it does not have to be built yet.
+    -DMETIS_INCLUDE_DIR=${STRUMPACK_METIS_INCLUDE_DIR}
+    -DTPL_METIS_LIBRARIES=$<TARGET_FILE:METIS::metis>
+
+    # BLAS and LAPACK are left for STRUMPACK's configure to find: it uses
+    # them only to check that a test program links. What STRUMPACK actually
+    # calls is whatever polysolve links it with, below.
+)
+
+# Build the library only: STRUMPACK's default target also builds its test
+# programs. With a Makefile generator, $(MAKE) lets the sub-build share the
+# outer make's job slots instead of running serially.
+if(CMAKE_GENERATOR MATCHES "Makefiles")
+    set(STRUMPACK_BUILD_COMMAND $(MAKE) strumpack)
+else()
+    set(STRUMPACK_BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target strumpack --config ${STRUMPACK_BUILD_TYPE})
+endif()
+
+include(ExternalProject)
+ExternalProject_Add(strumpack_build
+    SOURCE_DIR "${strumpack_SOURCE_DIR}"
+    BINARY_DIR "${strumpack_BINARY_DIR}"
+    INSTALL_DIR "${STRUMPACK_INSTALL_DIR}"
+    CMAKE_ARGS ${STRUMPACK_CMAKE_ARGS}
+    BUILD_COMMAND ${STRUMPACK_BUILD_COMMAND}
+    # `cmake --install`, unlike the install target, does not first build
+    # everything else.
+    INSTALL_COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR> --config ${STRUMPACK_BUILD_TYPE}
+    BUILD_BYPRODUCTS "${STRUMPACK_LIBRARY}"
+)
+
+# An imported target's include directory has to exist at generate time, well
+# before the ExternalProject has installed anything into it.
+file(MAKE_DIRECTORY "${STRUMPACK_INSTALL_DIR}/include")
+
+add_library(STRUMPACK::strumpack STATIC IMPORTED GLOBAL)
+set_target_properties(STRUMPACK::strumpack PROPERTIES
+    IMPORTED_LOCATION "${STRUMPACK_LIBRARY}"
+    INTERFACE_INCLUDE_DIRECTORIES "${STRUMPACK_INSTALL_DIR}/include"
+)
+# Link polysolve's BLAS and LAPACK (MKL, Accelerate, ...), not whichever ones
+# STRUMPACK's configure happened to find.
+target_link_libraries(STRUMPACK::strumpack INTERFACE METIS::metis LAPACK::LAPACK BLAS::BLAS)
+add_dependencies(STRUMPACK::strumpack strumpack_build)