From c1ef5c2227e0168472e012eb4992e2eb3a683ee3 Mon Sep 17 00:00:00 2001 From: Josh Romero Date: Tue, 6 Oct 2026 21:43:22 -0700 Subject: [PATCH 1/3] Detect NVSHMEM static heap fallback to enable symmetric heap size hints. Signed-off-by: Josh Romero --- src/cudecomp.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/cudecomp.cc b/src/cudecomp.cc index 68db55a..30060fc 100644 --- a/src/cudecomp.cc +++ b/src/cudecomp.cc @@ -750,6 +750,23 @@ static void inspectNvshmemEnvVars(nvshmemRuntimeState& runtime) { char* vmm_str = std::getenv("NVSHMEM_DISABLE_CUDA_VMM"); if (vmm_str) { runtime.nvshmem_vmm = std::strtol(vmm_str, nullptr, 10) == 0; } + if (runtime.nvshmem_vmm) { + int dev; + CUdevice cu_dev; + CHECK_CUDA(cudaGetDevice(&dev)); + CHECK_CUDA_DRV(cuDeviceGet(&cu_dev, dev)); + + int vmm_supported = 0; + int rdma_vmm_supported = 0; + CHECK_CUDA_DRV( + cuDeviceGetAttribute(&vmm_supported, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, cu_dev)); + CHECK_CUDA_DRV(cuDeviceGetAttribute(&rdma_vmm_supported, + CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED, cu_dev)); + // NVSHMEM falls back to a static heap if either capability is unavailable. + // Passing these checks establishes VMM eligibility, not the active heap mode. + runtime.nvshmem_vmm = vmm_supported && rdma_vmm_supported; + } + // Check NVSHMEM_SYMMETRIC_SIZE char* symmetric_size_str = std::getenv("NVSHMEM_SYMMETRIC_SIZE"); if (symmetric_size_str) { From 7e3fb3b1d0d7e1ee4ab0d4d8056cfc4dd638e303 Mon Sep 17 00:00:00 2001 From: Josh Romero Date: Tue, 6 Oct 2026 21:51:35 -0700 Subject: [PATCH 2/3] Formatting. Signed-off-by: Josh Romero --- src/cudecomp.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cudecomp.cc b/src/cudecomp.cc index 30060fc..3943616 100644 --- a/src/cudecomp.cc +++ b/src/cudecomp.cc @@ -760,8 +760,8 @@ static void inspectNvshmemEnvVars(nvshmemRuntimeState& runtime) { int rdma_vmm_supported = 0; CHECK_CUDA_DRV( cuDeviceGetAttribute(&vmm_supported, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, cu_dev)); - CHECK_CUDA_DRV(cuDeviceGetAttribute(&rdma_vmm_supported, - CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED, cu_dev)); + CHECK_CUDA_DRV( + cuDeviceGetAttribute(&rdma_vmm_supported, CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED, cu_dev)); // NVSHMEM falls back to a static heap if either capability is unavailable. // Passing these checks establishes VMM eligibility, not the active heap mode. runtime.nvshmem_vmm = vmm_supported && rdma_vmm_supported; From 2b14c9613ebc29fcb82a54a9e293a34000ce0402 Mon Sep 17 00:00:00 2001 From: Josh Romero Date: Tue, 6 Oct 2026 22:01:23 -0700 Subject: [PATCH 3/3] Add CUDA version guards. Signed-off-by: Josh Romero --- src/cudecomp.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cudecomp.cc b/src/cudecomp.cc index 3943616..f1f4cd3 100644 --- a/src/cudecomp.cc +++ b/src/cudecomp.cc @@ -751,6 +751,7 @@ static void inspectNvshmemEnvVars(nvshmemRuntimeState& runtime) { if (vmm_str) { runtime.nvshmem_vmm = std::strtol(vmm_str, nullptr, 10) == 0; } if (runtime.nvshmem_vmm) { +#if CUDART_VERSION >= 11030 int dev; CUdevice cu_dev; CHECK_CUDA(cudaGetDevice(&dev)); @@ -765,6 +766,10 @@ static void inspectNvshmemEnvVars(nvshmemRuntimeState& runtime) { // NVSHMEM falls back to a static heap if either capability is unavailable. // Passing these checks establishes VMM eligibility, not the active heap mode. runtime.nvshmem_vmm = vmm_supported && rdma_vmm_supported; +#else + // NVSHMEM's VMM heap requires CUDA 11.3 or newer. + runtime.nvshmem_vmm = false; +#endif } // Check NVSHMEM_SYMMETRIC_SIZE