Optional CUDA builds for UCX and OpenMPI; add UCX 1.22.0 and OpenMPI 5.0.11

ansible/roles/compiled/tasks/common/main.yml

@@ -183,6 +183,8 @@
       - { version_number: "1.15.0", cuda_root: "/software/common/cuda/12.2.0_535.54.03" }
       - { version_number: "1.16.0", cuda_root: "/software/common/cuda/12.2.0_535.54.03" }
       - { version_number: "1.19.0", cuda_root: "/software/common/cuda/12.9.1_575.57.08" }
+      - version_number: "1.22.0"
+        cuda_root: "/software/common/cuda/12.9.1_575.57.08"
   ansible.builtin.include_tasks:
     file: ucx.yaml
   loop: "{{ ucx_versions }}"
@@ -241,43 +243,45 @@
 - name: Install openmpis
   vars:
     openmpi_versions:
+      - version_number: "5.0.11"
+        checksum: "sha256:e668a3c4acd50c41dc204c8a6dd98a611e0f26af89cf677577fa9be8a2698003"
+        url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.11.tar.bz2"
+        ucx_version: "1.22.0"
+        cuda_root: "/software/common/cuda/12.9.1_575.57.08"
+        cuda_module: "cuda/12.9.1_575.57.08"
+        pmix: "/software/common/pmix/5.0.9"
       - version_number: "5.0.8"
         checksum: "sha256:53131e1a57e7270f645707f8b0b65ba56048f5b5ac3f68faabed3eb0d710e449"
         url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.8.tar.bz2"
         ucx_version: "1.19.0"
-        singularity_version: "4.2.2"
         cuda_root: "/software/common/cuda/12.9.1_575.57.08"
         cuda_module: "cuda/12.9.1_575.57.08"
         pmix: "/software/common/pmix/5.0.9"
       - version_number: "5.0.3"
         checksum: "sha256:990582f206b3ab32e938aa31bbf07c639368e4405dca196fabe7f0f76eeda90b"
         url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.3.tar.bz2"
         ucx_version: "1.16.0"
-        singularity_version: "3.11.3"
         cuda_root: "/software/common/cuda/12.2.0_535.54.03"
         cuda_module: "cuda/12.2.0_535.54.03"
         pmix: "/software/common/pmix/5.0.2"
       - version_number: "5.0.2"
         checksum: "sha256:ee46ad8eeee2c3ff70772160bff877cbf38c330a0bc3b3ddc811648b3396698f"
         url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.2.tar.bz2"
         ucx_version: "1.15.0"
-        singularity_version: "3.11.3"
         cuda_root: "/software/common/cuda/12.1.0_530.30.02"
         cuda_module: "cuda/12.1.0_530.30.02"
         pmix: "/software/common/pmix/4.2.9"
       - version_number: "4.1.5"
         checksum: "sha256:a640986bc257389dd379886fdae6264c8cfa56bc98b71ce3ae3dfbd8ce61dbe3"
         url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.5.tar.bz2"
         ucx_version: "1.15.0"
-        singularity_version: "3.11.3"
         cuda_root: "/software/common/cuda/12.2.0_535.54.03"
         cuda_module: "cuda/12.2.0_535.54.03"
         pmix: "/software/common/pmix/4.1.1"
       - version_number: "4.1.4"
         checksum: "sha256:92912e175fd1234368c8730c03f4996fe5942e7479bb1d10059405e7f2b3930d"
         url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.bz2"
         ucx_version: "1.13.0"
-        singularity_version: "3.9.1"
         cuda_root: "/software/common/cuda/11.6.0_510.39.01"
         cuda_module: "cuda/11.6.0_510.39.01"
       - version_number: "4.1.0"

ansible/roles/compiled/tasks/common/openmpi.yaml

@@ -11,7 +11,11 @@
     unzipdir: "/dev/shm/openmpi-{{ version_number }}"
     build_dir: "{{ unzipdir }}/build"
     ucx_dir: "{{ common_dir }}/ucx/{{ item.ucx_version }}"
-    singularity_dir: "{{ common_dir }}/singularity/{{ item.singularity_version }}"
+    # Follow the cluster's current Singularity (group_vars `singularity`) rather
+    # than pinning a version per entry. OpenMPI 5 dropped --with-singularity
+    # (its configure no longer recognises the option), so only 4.x gets the flag.
+    singularity_dir: "{{ singularity | dirname | dirname }}"
+    singularity_configure_flag: "{{ '--with-singularity=' ~ singularity_dir if version_number is version('5.0', '<') else '' }}"
     cuda_root: "{{ item.cuda_root }}"
     cuda_module: "{{ item.cuda_module }}"
   tags:
@@ -42,16 +46,21 @@
                 mode: u=rwx,g=rwx,o=rx
             - name: Run OpenMPI configure ( 4.1.4 and below ) {{ version_number }}
               when: item.version_number < "4.1.5"
-              ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir }}
+              ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static {{ singularity_configure_flag }} --with-ucx={{ ucx_dir }}
                 --with-cuda={{ cuda_root }} --with-pmix=/opt/slurm --with-pmi-libdir=/opt/slurm/lib"
               args:
                 chdir: "{{ build_dir }}"
                 creates: "{{ build_dir }}/Makefile"
-            - name: Run OpenMPI configure ( 4.1.5 and above ) {{ version_number }}
+            # Run with --skip-tags openmpi_gpu to build only the plain variant, e.g.
+            # on a host without the CUDA toolkit. The gpu module task carries the
+            # same tag so no module is left pointing at a missing install.
+            - name: Build OpenMPI with gpu ( 4.1.5 and above ) {{ version_number }}
               when: item.version_number >= "4.1.5"
+              tags:
+                - openmpi_gpu
               block:
                 - name: Configure OpenMPI for gpu {{ version_number }}
-                  ansible.builtin.command: "../configure --prefix={{ install_dir_gpu }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir
+                  ansible.builtin.command: "../configure --prefix={{ install_dir_gpu }} --enable-static {{ singularity_configure_flag }} --with-ucx={{ ucx_dir
                     }} --with-cuda={{ cuda_root }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib"
                   args:
                     chdir: "{{ build_dir }}"
@@ -71,8 +80,11 @@
                   ansible.builtin.file:
                     path: "{{ build_dir }}/Makefile"
                     state: absent
+            - name: Build OpenMPI with no gpu ( 4.1.5 and above ) {{ version_number }}
+              when: item.version_number >= "4.1.5"
+              block:
                 - name: Configure OpenMPI for no gpu {{ version_number }}
-                  ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir
+                  ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static {{ singularity_configure_flag }} --with-ucx={{ ucx_dir
                     }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib"
                   args:
                     chdir: "{{ build_dir }}"
@@ -93,6 +105,8 @@
             state: directory
             mode: u=rwx,g=rwx,o=rx
         - name: Install OpenMPI gpu module file
+          tags:
+            - openmpi_gpu
           vars:
             gpu: true
           ansible.builtin.template:

ansible/roles/compiled/tasks/common/ucx.yaml

@@ -9,7 +9,10 @@
     install_dir: "{{ ucx_dir }}/{{ version_number }}"
     unzipdir: "/dev/shm/ucx"
     build_dir: "{{ unzipdir }}/build"
-    cuda_root: "{{ item.cuda_root }}"
+    cuda_root: "{{ item.cuda_root | default('') }}"
+    # Run with --skip-tags ucx_cuda to build without CUDA support, e.g. on a host
+    # without the toolkit. Entries with no cuda_root (1.9.0) never get the flag.
+    cuda_configure_flag: "{{ '--with-cuda=' ~ cuda_root if cuda_root and 'ucx_cuda' not in ansible_skip_tags else '' }}"
   tags:
     - never
     - ucx{{ item.version_number }}
@@ -35,7 +38,7 @@
                 chdir: "{{ unzipdir }}"
                 creates: "{{ unzipdir }}/configure"
             - name: Run UCX configure {{ version_number }}
-              ansible.builtin.command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations --with-cuda={{ cuda_root }}"
+              ansible.builtin.command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations {{ cuda_configure_flag }}"
               args:
                 chdir: "{{ build_dir }}"
                 creates: "{{ build_dir }}/Makefile"