From a726a03a2e26edb72a99f0468a8691457e70e810 Mon Sep 17 00:00:00 2001 From: Dane Kennedy Date: Wed, 23 Sep 2026 11:19:35 +0200 Subject: [PATCH 1/5] Make the UCX CUDA build optional via --skip-tags ucx_cuda UCX has a single install dir, so rather than duplicating the build block (as PR #168 did, where both blocks installed to the same place), the one configure task drops --with-cuda when ucx_cuda is in ansible_skip_tags. Entries without a cuda_root, such as 1.9.0, no longer fail on an undefined variable and simply build without CUDA. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FLJRERiNvARR3e8ftXi5TG --- ansible/roles/compiled/tasks/common/ucx.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ansible/roles/compiled/tasks/common/ucx.yaml b/ansible/roles/compiled/tasks/common/ucx.yaml index 76ce9be..1cbf2b1 100644 --- a/ansible/roles/compiled/tasks/common/ucx.yaml +++ b/ansible/roles/compiled/tasks/common/ucx.yaml @@ -9,7 +9,10 @@ install_dir: "{{ ucx_dir }}/{{ version_number }}" unzipdir: "/dev/shm/ucx" build_dir: "{{ unzipdir }}/build" - cuda_root: "{{ item.cuda_root }}" + cuda_root: "{{ item.cuda_root | default('') }}" + # Run with --skip-tags ucx_cuda to build without CUDA support, e.g. on a host + # without the toolkit. Entries with no cuda_root (1.9.0) never get the flag. + cuda_configure_flag: "{{ '--with-cuda=' ~ cuda_root if cuda_root and 'ucx_cuda' not in ansible_skip_tags else '' }}" tags: - never - ucx{{ item.version_number }} @@ -35,7 +38,7 @@ chdir: "{{ unzipdir }}" creates: "{{ unzipdir }}/configure" - name: Run UCX configure {{ version_number }} - ansible.builtin.command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations --with-cuda={{ cuda_root }}" + ansible.builtin.command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations {{ cuda_configure_flag }}" args: chdir: "{{ build_dir }}" creates: "{{ build_dir }}/Makefile" From 7916cf755cf14b786a965b0103a74c4e04a0d31c Mon Sep 17 00:00:00 2001 From: Dane Kennedy Date: Wed, 23 Sep 2026 11:19:35 +0200 Subject: [PATCH 2/5] Make the OpenMPI GPU build optional via --skip-tags openmpi_gpu Split the 4.1.5+ build into a gpu block tagged openmpi_gpu and a plain block, so --skip-tags openmpi_gpu builds only the plain variant. The gpu module task carries the same tag so no module is left pointing at a missing install. Verified with a throwaway playbook that a skip tag on an inner block inside a tagged include drops exactly those tasks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FLJRERiNvARR3e8ftXi5TG --- ansible/roles/compiled/tasks/common/openmpi.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ansible/roles/compiled/tasks/common/openmpi.yaml b/ansible/roles/compiled/tasks/common/openmpi.yaml index 73f001b..48395f3 100644 --- a/ansible/roles/compiled/tasks/common/openmpi.yaml +++ b/ansible/roles/compiled/tasks/common/openmpi.yaml @@ -47,8 +47,13 @@ args: chdir: "{{ build_dir }}" creates: "{{ build_dir }}/Makefile" - - name: Run OpenMPI configure ( 4.1.5 and above ) {{ version_number }} + # Run with --skip-tags openmpi_gpu to build only the plain variant, e.g. + # on a host without the CUDA toolkit. The gpu module task carries the + # same tag so no module is left pointing at a missing install. + - name: Build OpenMPI with gpu ( 4.1.5 and above ) {{ version_number }} when: item.version_number >= "4.1.5" + tags: + - openmpi_gpu block: - name: Configure OpenMPI for gpu {{ version_number }} ansible.builtin.command: "../configure --prefix={{ install_dir_gpu }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir @@ -71,6 +76,9 @@ ansible.builtin.file: path: "{{ build_dir }}/Makefile" state: absent + - name: Build OpenMPI with no gpu ( 4.1.5 and above ) {{ version_number }} + when: item.version_number >= "4.1.5" + block: - name: Configure OpenMPI for no gpu {{ version_number }} ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib" @@ -93,6 +101,8 @@ state: directory mode: u=rwx,g=rwx,o=rx - name: Install OpenMPI gpu module file + tags: + - openmpi_gpu vars: gpu: true ansible.builtin.template: From 3430338d9b804085881cc4457aaa01bc4438c5fb Mon Sep 17 00:00:00 2001 From: Dane Kennedy Date: Wed, 23 Sep 2026 11:20:07 +0200 Subject: [PATCH 3/5] Add UCX 1.22.0 Built against CUDA 12.9.1 like 1.19.0. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FLJRERiNvARR3e8ftXi5TG --- ansible/roles/compiled/tasks/common/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/compiled/tasks/common/main.yml b/ansible/roles/compiled/tasks/common/main.yml index 7d8cab6..f361a2e 100644 --- a/ansible/roles/compiled/tasks/common/main.yml +++ b/ansible/roles/compiled/tasks/common/main.yml @@ -183,6 +183,8 @@ - { version_number: "1.15.0", cuda_root: "/software/common/cuda/12.2.0_535.54.03" } - { version_number: "1.16.0", cuda_root: "/software/common/cuda/12.2.0_535.54.03" } - { version_number: "1.19.0", cuda_root: "/software/common/cuda/12.9.1_575.57.08" } + - version_number: "1.22.0" + cuda_root: "/software/common/cuda/12.9.1_575.57.08" ansible.builtin.include_tasks: file: ucx.yaml loop: "{{ ucx_versions }}" From 1ad123bb7ad695aaa12b046d845d01a1d1473449 Mon Sep 17 00:00:00 2001 From: Dane Kennedy Date: Wed, 23 Sep 2026 11:20:07 +0200 Subject: [PATCH 4/5] Add OpenMPI 5.0.11 Checksum computed from the release tarball. Companion versions mirror the 5.0.8 entry (Singularity 4.2.2, CUDA 12.9.1, PMIx 5.0.9) with UCX moved to 1.22.0. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FLJRERiNvARR3e8ftXi5TG --- ansible/roles/compiled/tasks/common/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/roles/compiled/tasks/common/main.yml b/ansible/roles/compiled/tasks/common/main.yml index f361a2e..ea081ac 100644 --- a/ansible/roles/compiled/tasks/common/main.yml +++ b/ansible/roles/compiled/tasks/common/main.yml @@ -243,6 +243,14 @@ - name: Install openmpis vars: openmpi_versions: + - version_number: "5.0.11" + checksum: "sha256:e668a3c4acd50c41dc204c8a6dd98a611e0f26af89cf677577fa9be8a2698003" + url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.11.tar.bz2" + ucx_version: "1.22.0" + singularity_version: "4.2.2" + cuda_root: "/software/common/cuda/12.9.1_575.57.08" + cuda_module: "cuda/12.9.1_575.57.08" + pmix: "/software/common/pmix/5.0.9" - version_number: "5.0.8" checksum: "sha256:53131e1a57e7270f645707f8b0b65ba56048f5b5ac3f68faabed3eb0d710e449" url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.8.tar.bz2" From 8654939a1b5e2e3220a846c2382df100b5958b9d Mon Sep 17 00:00:00 2001 From: Dane Kennedy Date: Wed, 23 Sep 2026 11:27:52 +0200 Subject: [PATCH 5/5] Build OpenMPI against the cluster's current Singularity, not a pinned one Derive the Singularity directory from the group_vars `singularity` path instead of a per-entry singularity_version, so every build follows the cluster default (4.4.1 today). OpenMPI 5's configure no longer recognises --with-singularity, so the flag was a silent no-op for the 5.x entries; it is now passed only to 4.x builds. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FLJRERiNvARR3e8ftXi5TG --- ansible/roles/compiled/tasks/common/main.yml | 6 ------ ansible/roles/compiled/tasks/common/openmpi.yaml | 12 ++++++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ansible/roles/compiled/tasks/common/main.yml b/ansible/roles/compiled/tasks/common/main.yml index ea081ac..ecdf51a 100644 --- a/ansible/roles/compiled/tasks/common/main.yml +++ b/ansible/roles/compiled/tasks/common/main.yml @@ -247,7 +247,6 @@ checksum: "sha256:e668a3c4acd50c41dc204c8a6dd98a611e0f26af89cf677577fa9be8a2698003" url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.11.tar.bz2" ucx_version: "1.22.0" - singularity_version: "4.2.2" cuda_root: "/software/common/cuda/12.9.1_575.57.08" cuda_module: "cuda/12.9.1_575.57.08" pmix: "/software/common/pmix/5.0.9" @@ -255,7 +254,6 @@ checksum: "sha256:53131e1a57e7270f645707f8b0b65ba56048f5b5ac3f68faabed3eb0d710e449" url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.8.tar.bz2" ucx_version: "1.19.0" - singularity_version: "4.2.2" cuda_root: "/software/common/cuda/12.9.1_575.57.08" cuda_module: "cuda/12.9.1_575.57.08" pmix: "/software/common/pmix/5.0.9" @@ -263,7 +261,6 @@ checksum: "sha256:990582f206b3ab32e938aa31bbf07c639368e4405dca196fabe7f0f76eeda90b" url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.3.tar.bz2" ucx_version: "1.16.0" - singularity_version: "3.11.3" cuda_root: "/software/common/cuda/12.2.0_535.54.03" cuda_module: "cuda/12.2.0_535.54.03" pmix: "/software/common/pmix/5.0.2" @@ -271,7 +268,6 @@ checksum: "sha256:ee46ad8eeee2c3ff70772160bff877cbf38c330a0bc3b3ddc811648b3396698f" url: "https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.2.tar.bz2" ucx_version: "1.15.0" - singularity_version: "3.11.3" cuda_root: "/software/common/cuda/12.1.0_530.30.02" cuda_module: "cuda/12.1.0_530.30.02" pmix: "/software/common/pmix/4.2.9" @@ -279,7 +275,6 @@ checksum: "sha256:a640986bc257389dd379886fdae6264c8cfa56bc98b71ce3ae3dfbd8ce61dbe3" url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.5.tar.bz2" ucx_version: "1.15.0" - singularity_version: "3.11.3" cuda_root: "/software/common/cuda/12.2.0_535.54.03" cuda_module: "cuda/12.2.0_535.54.03" pmix: "/software/common/pmix/4.1.1" @@ -287,7 +282,6 @@ checksum: "sha256:92912e175fd1234368c8730c03f4996fe5942e7479bb1d10059405e7f2b3930d" url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.bz2" ucx_version: "1.13.0" - singularity_version: "3.9.1" cuda_root: "/software/common/cuda/11.6.0_510.39.01" cuda_module: "cuda/11.6.0_510.39.01" - version_number: "4.1.0" diff --git a/ansible/roles/compiled/tasks/common/openmpi.yaml b/ansible/roles/compiled/tasks/common/openmpi.yaml index 48395f3..288454f 100644 --- a/ansible/roles/compiled/tasks/common/openmpi.yaml +++ b/ansible/roles/compiled/tasks/common/openmpi.yaml @@ -11,7 +11,11 @@ unzipdir: "/dev/shm/openmpi-{{ version_number }}" build_dir: "{{ unzipdir }}/build" ucx_dir: "{{ common_dir }}/ucx/{{ item.ucx_version }}" - singularity_dir: "{{ common_dir }}/singularity/{{ item.singularity_version }}" + # Follow the cluster's current Singularity (group_vars `singularity`) rather + # than pinning a version per entry. OpenMPI 5 dropped --with-singularity + # (its configure no longer recognises the option), so only 4.x gets the flag. + singularity_dir: "{{ singularity | dirname | dirname }}" + singularity_configure_flag: "{{ '--with-singularity=' ~ singularity_dir if version_number is version('5.0', '<') else '' }}" cuda_root: "{{ item.cuda_root }}" cuda_module: "{{ item.cuda_module }}" tags: @@ -42,7 +46,7 @@ mode: u=rwx,g=rwx,o=rx - name: Run OpenMPI configure ( 4.1.4 and below ) {{ version_number }} when: item.version_number < "4.1.5" - ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir }} + ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static {{ singularity_configure_flag }} --with-ucx={{ ucx_dir }} --with-cuda={{ cuda_root }} --with-pmix=/opt/slurm --with-pmi-libdir=/opt/slurm/lib" args: chdir: "{{ build_dir }}" @@ -56,7 +60,7 @@ - openmpi_gpu block: - name: Configure OpenMPI for gpu {{ version_number }} - ansible.builtin.command: "../configure --prefix={{ install_dir_gpu }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir + ansible.builtin.command: "../configure --prefix={{ install_dir_gpu }} --enable-static {{ singularity_configure_flag }} --with-ucx={{ ucx_dir }} --with-cuda={{ cuda_root }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib" args: chdir: "{{ build_dir }}" @@ -80,7 +84,7 @@ when: item.version_number >= "4.1.5" block: - name: Configure OpenMPI for no gpu {{ version_number }} - ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir + ansible.builtin.command: "../configure --prefix={{ install_dir }} --enable-static {{ singularity_configure_flag }} --with-ucx={{ ucx_dir }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib" args: chdir: "{{ build_dir }}"