fix(mori,docker): unblock MORI smoke on containerized RDMA clusters

cvs/input/config_file/preflight/preflight_config.json

@@ -11,7 +11,7 @@
       "_comment_gpus_per_node": "Exact number of AMD GPUs expected on every node.",
 
       "_example_expected_rocm_version": "6.2.0",
-      "expected_rocm_version": "10.0.0",
+      "expected_rocm_version": "<changeme>",
       "_comment_expected_rocm_version": "Expected ROCm version across all cluster nodes. Must match the output of 'amd-smi version' on all nodes. Format: 'major.minor.patch' (e.g., '6.2.0', '5.7.1')."
     },
 
@@ -25,11 +25,11 @@
         "_comment_connectivity_mode": "RDMA connectivity testing mode. Options: 'basic' (test adjacent node pairs, fast, ~14% coverage for 8 nodes), 'full_mesh' (test all possible node pairs, comprehensive, 100% coverage), 'skip' (skip RDMA connectivity testing entirely).",
 
         "_example_gid_index": "3",
-        "gid_index": "3",
+        "gid_index": "<changeme>",
         "_comment_gid_index": "GID index to check on all RDMA interfaces. Typically '3' for RoCE (RDMA over Converged Ethernet). Must be a valid GID index for your InfiniBand/RoCE setup.",
 
         "_example_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0", "rocep158s0", "rocep190s0", "rocep206s0", "rocep222s0"],
-        "interfaces": ["rdma0", "rdma1", "rdma2", "rdma3", "rdma4", "rdma5", "rdma6", "rdma7"],
+        "interfaces": ["<changeme>"],
         "_comment_interfaces": "List of specific RDMA interface names that should be present on all cluster nodes. Examples: ['rocep28s0', 'rocep62s0'] for standard setup, ['mlx5_0', 'mlx5_1'] for Mellanox, ['ib0', 'ib1'] for generic InfiniBand.",
 
         "nodes_per_full_mesh_group": 32,

cvs/input/config_file/training/jaxmaxtext/README.md

@@ -6,7 +6,7 @@ lives in the CVS docs and is the single source of truth:
 - **Configuration & threshold reference:**
   [`docs/reference/configuration-files/training/jaxmaxtext.rst`](../../../../../docs/reference/configuration-files/training/jaxmaxtext.rst)
 - **Running the suites (lifecycle, metrics, reports):**
-  [`docs/how-to/test-suites/training/jax.rst`](../../../../../docs/how-to/test-suites/training/jax.rst)
+  [`docs/how-to/test-suites/training/jaxmaxtext.rst`](../../../../../docs/how-to/test-suites/training/jaxmaxtext.rst)
 
 
 The config files and their sibling `_threshold.json` files live in this

cvs/input/config_file/training/jaxmaxtext/mi3xx_jaxmaxtext_deepseek-v4-284b_distributed.json

@@ -91,6 +91,9 @@
       "steps": 30,
       "enable_checkpointing": false,
       "attention": "dot_product",
+      "dtype": "bfloat16",
+      "weight_dtype": "bfloat16",
+      "quantization": "",
       "dataset_type": "synthetic",
       "remat_policy": "full",
       "per_device_batch_size": 1,

cvs/lib/docker_lib.py

@@ -143,6 +143,7 @@ def launch_docker_container(
     shm_size='64G',
     timeout=60 * 10,
     ulimit_nofile=None,
+    ulimit_memlock=None,
 ):
     cmd = f'docker run -d --network {network} --ipc {network} \
             --cap-add=IPC_LOCK --security-opt seccomp=unconfined --privileged '
@@ -165,6 +166,11 @@ def launch_docker_container(
             )
     if ulimit_nofile:
         cmd = cmd + f' --ulimit nofile={ulimit_nofile} '
+    if ulimit_memlock is not None:
+        # RDMA tests register GPU memory via ibv_reg_mr, charged against RLIMIT_MEMLOCK.
+        # Docker defaults the container to 8MB even when the host is unlimited
+        # (--privileged does NOT lift ulimits), so registrations >8MB fail with ENOMEM.
+        cmd = cmd + f' --ulimit memlock={ulimit_memlock} '
     for device in device_list:
         cmd = cmd + f' --device {device} '
     for src_vol in volume_dict.keys():

cvs/lib/mori_lib.py

@@ -185,7 +185,7 @@ def create_env_script(
     ):
         cmd = f'''docker exec {self.container_name} /bin/bash -c "echo '
               export PYTHONPATH={self.mori_dir}:$PYTHONPATH
-              #export LD_LIBRARY_PATH={self.torchlib_dir}:$LD_LIBRARY_PATH
+              export LD_LIBRARY_PATH={self.torchlib_dir}:$LD_LIBRARY_PATH
               export NCCL_SOCKET_IFNAME={self.oob_port}
               export GLOO_SOCKET_IFNAME={self.oob_port}
               export GLOO_TCP_IFNAME={self.oob_port}
@@ -328,7 +328,13 @@ def run_concurrent_put_threads(
                     mpiexec --allow-run-as-root -np 2 ./build/examples/concurrent_put_thread" '''
         out_dict = self.phdl.exec(cmd)
         for node in out_dict.keys():
-            if not re.search('PASSED', out_dict[node], re.I):
+            # NOTE: concurrent_put_thread / concurrent_put_imm_thread perform NO data
+            # verification - they launch the kernel, barrier, and print "test done!".
+            # They never emit "PASSED" (unlike concurrent_put_signal_thread, which does
+            # validate and prints "...tests passed!"). Accepting the completion marker
+            # makes this a SMOKE TEST: it proves the binary ran to completion without
+            # crashing, NOT that the transferred data is correct.
+            if not re.search(r'PASSED|test done!', out_dict[node], re.I):
                 fail_test('ERROR - test concurrent_put_thread did not run properly, no PASSED test results seen')
             if re.search('FAIL', out_dict[node], re.I):
                 fail_test('ERROR - one or more concurrent_put_thread tests failed')
@@ -342,7 +348,13 @@ def run_concurrent_put_imm_threads(
                     mpiexec --allow-run-as-root -np 2 ./build/examples/concurrent_put_imm_thread" '''
         out_dict = self.phdl.exec(cmd)
         for node in out_dict.keys():
-            if not re.search('PASSED', out_dict[node], re.I):
+            # NOTE: concurrent_put_thread / concurrent_put_imm_thread perform NO data
+            # verification - they launch the kernel, barrier, and print "test done!".
+            # They never emit "PASSED" (unlike concurrent_put_signal_thread, which does
+            # validate and prints "...tests passed!"). Accepting the completion marker
+            # makes this a SMOKE TEST: it proves the binary ran to completion without
+            # crashing, NOT that the transferred data is correct.
+            if not re.search(r'PASSED|test done!', out_dict[node], re.I):
                 fail_test('ERROR - test concurrent_put_imm_thread did not run properly, no PASSED test results seen')
             if re.search('FAIL', out_dict[node], re.I):
                 fail_test('ERROR - one or more concurrent_put_imm_thread tests failed')

cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py

@@ -519,6 +519,30 @@ def test_sweep_steps_override_drives_yaml_and_poll(self):
         self.assertIn("steps: 7", written)
         self.assertNotIn("steps: 3", written)
 
+    def test_smoke_steps_override_wins_over_base_maxtext_config(self):
+        # Smoke passes training.smoke.steps via maxtext_overrides (not training.steps
+        # alone); base maxtext_config.steps must not leak into the run.
+        base, _ = _make_job()
+        base.variant.training.maxtext_config["steps"] = 30
+        sweep = SimpleNamespace(
+            name="SMOKE",
+            maxtext_overrides={
+                "steps": 5,
+                "per_device_batch_size": 1,
+                "max_target_length": 2048,
+                "dtype": "bfloat16",
+                "weight_dtype": "bfloat16",
+                "quantization": "",
+            },
+        )
+        job = MaxTextTrainingJob(base.orch, base.variant, hf_token="dummy", sweep=sweep)
+        self.assertEqual(job.steps, 5)
+        self.assertEqual(job._poll_count, 50)
+        job._write_maxtext_yaml()
+        written = " ".join(str(c.args[0]) for c in job.orch.exec.call_args_list)
+        self.assertIn("steps: 5", written)
+        self.assertNotIn("steps: 30", written)
+
     def test_empty_string_rendered_as_quoted_not_bare(self):
         # An empty-string maxtext param (e.g. profiler) must render as 'key: ""',
         # never bare 'key:' (which YAML reads as null and breaks MaxText enums).

cvs/tests/mori/mori_benchmark_test.py

@@ -175,6 +175,7 @@ def test_launch_mori_container(phdl, mori_dict):
         mori_dict['container_config']['env_dict'],
         shm_size='48G',
         timeout=60 * 20,
+        ulimit_memlock='-1',
     )
     # ADD verifications ..
     time.sleep(30)

cvs/tests/training/jaxmaxtext/README.md

@@ -4,7 +4,7 @@ The documentation for the JAX MaxText **test suites** now lives in the CVS docs
 and is the single source of truth:
 
 - **Running the suites (lifecycle, metrics, reports):**
-  [`docs/how-to/test-suites/training/jax.rst`](../../../../docs/how-to/test-suites/training/jax.rst)
+  [`docs/how-to/test-suites/training/jaxmaxtext.rst`](../../../../docs/how-to/test-suites/training/jaxmaxtext.rst)
 - **Configuration & threshold reference:**
   [`docs/reference/configuration-files/training/jaxmaxtext.rst`](../../../../docs/reference/configuration-files/training/jaxmaxtext.rst)
 

cvs/tests/training/jaxmaxtext/_common.py

@@ -276,6 +276,7 @@ def smoke(orch, variant_config, hf_token, lifecycle, request):
     smoke_sweep = SimpleNamespace(
         name="SMOKE",
         maxtext_overrides={
+            "steps": steps,
             "per_device_batch_size": batch,
             "max_target_length": seqlen,
             "dtype": "bfloat16",
@@ -439,6 +440,7 @@ def _mk_job(total_steps, extra_overrides, enable_ckpt):
         v.training.enable_checkpointing = enable_ckpt
         ov = dict(base_overrides)
         ov.update(extra_overrides or {})
+        ov["steps"] = total_steps
         sweep = SimpleNamespace(name="CKPT", maxtext_overrides=ov)
         return MaxTextTrainingJob(orch, v, hf_token, sweep=sweep)
 

docs/concepts/what-is-cvs.rst

@@ -11,7 +11,7 @@ Use CVS to verify GPU cluster health, GPU/CPU node health, host OS configuration
 Here are the tests available in the CVS:
 
 - **Platform tests**: Perform host OS configuration, BIOS, firmware/driver, and network configuration checks.
-- **Burn-in health tests**: Perform `AMD GPU Field Health Check (AGFHC) <https://instinct.docs.amd.com/projects/gpu-operator/en/latest/test/agfhc.html>`_, `TransferBench <https://rocm.docs.amd.com/projects/TransferBench/en/latest/install/install.html#install-transferbench>`_, and `ROCm Validation Suite (RVS) <https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/latest/install/installation.html>`_.
+- **Burn-in health tests**: Perform `AMD GPU Field Health Check (AGFHC) <https://instinct.docs.amd.com/projects/gpu-operator/en/latest/test/agfhc.html>`_, `TransferBench <https://rocm.docs.amd.com/projects/TransferBench/en/latest/install/install.html#install-transferbench>`_, and `ROCm Validation Suite (RVS) <https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/latest/install/installation.html>`_. For MI4XX platforms, the suite also includes AMD Node Check (ANC).
 - **InfiniBand (IB Perf)**: These tests are low-level network performance benchmarks that validate the raw communication capabilities of InfiniBand adapters and interconnects. These tests measure the fundamental building blocks on which RCCL and other high-level libraries depend.
 - **Network tests**: Perform ping checks and multi-node `ROCm Communication Collectives Library (RCCL) <https://rocm.docs.amd.com/projects/rccl/en/latest/install/installation.html>`_ validations for different collectives.
 - **Distributed training tests**: Run and validate Llama and DeepSeek model trainings on single-node or multi-node clusters with the `JAX <https://rocm.docs.amd.com/en/latest/compatibility/ml-compatibility/jax-compatibility.html>`_ MaxText and `Megatron <https://rocm.docs.amd.com/en/latest/compatibility/ml-compatibility/stanford-megatron-lm-compatibility.html>`_ frameworks.

docs/how-to/configure/test-suite-config/pick-config-file.rst

@@ -36,6 +36,11 @@ Burn-in / Diag
      - ``cvs config list preflight``
 
        `README_preflight_config.md <https://github.com/ROCm/cvs/blob/main/cvs/input/config_file/preflight/README_preflight_config.md>`_
+   * - ANC
+     - ``input/config_file/anc/anc_config.json``
+     - ``cvs config list anc``
+
+       `cvs/tests/anc/README.md <https://github.com/ROCm/cvs/blob/main/cvs/tests/anc/README.md>`_
 
 Network
 =======

docs/how-to/run-tests/index.rst

@@ -76,7 +76,7 @@ Test suites
 
 Per-suite **Set up config** and **Run tests** steps are grouped by category:
 
-- :doc:`Burn-in / Diag tests </how-to/test-suites/burn-in-diag/index>` — platform, health, preflight
+- :doc:`Burn-in / Diag tests </how-to/test-suites/burn-in-diag/index>` — platform, health, preflight, ANC
 - :doc:`Network tests </how-to/test-suites/network/index>` — IB Perf, RCCL, MORI
 - :doc:`Training tests </how-to/test-suites/training/index>` — Aorta, JAX MaxText, Megatron
 - :doc:`Inference tests </how-to/test-suites/inference/index>` — vLLM, ATOM, SGLang, xDiT

docs/how-to/test-suites/burn-in-diag/anc.rst

@@ -0,0 +1,140 @@
+.. meta::
+  :description: Run AMD Node Check (ANC) CPU and GPU diagnostic suites
+  :keywords: CVS, ANC, AMD Node Check, burn-in, diag
+
+*********
+ANC tests
+*********
+
+AMD Node Check (ANC) runs CPU and GPU diagnostic groups on every node in the cluster. CVS installs ANC when needed, invokes each group as ``sudo ./anc.py -g <group>``, and collects logs and HTML reports.
+
+ANC requires **root**. The runner must have passwordless SSH and passwordless ``sudo`` on every target node. Without passwordless ``sudo``, group runs and log collection fail.
+
+.. _anc-set-up-config:
+
+Set up config
+=============
+
+1. Copy the ANC configuration file:
+
+   .. code:: bash
+
+     cvs config copy anc/anc_config.json --output ~/cvs_workspace/anc/anc_config.json
+
+2. Replace every ``<changeme>`` placeholder:
+
+   - ``anc_release_url`` — URL of the ANC release archive to download and install
+   - ``log_folder_path`` — controller-side directory prefix for collected logs and auto-generated HTML reports
+   - ``anc_version`` — expected ANC version (must match the version in ``anc_release_url``)
+
+3. Optionally set ``ANC_INSTALL_PATH`` for relocatable **tar** installs. Deb and rpm packages ignore this key and always install under ``/opt/amdtools``.
+
+Full parameter list: :doc:`/reference/configuration-files/burn-in-diag/anc`.
+
+.. _anc-run-tests:
+
+Run tests
+=========
+
+List the install suite and the CPU / GPU group suites:
+
+.. code:: bash
+
+  cvs list anc_installation
+  cvs list anc_test_cpu
+  cvs list anc_test_gpu
+
+Install ANC
+~~~~~~~~~~~
+
+Every CPU and GPU group run installs ANC as a session-cached pre-task, so a separate install step is optional. Run ``anc_installation`` when you want to install or refresh ANC without running a validation group:
+
+.. code:: bash
+
+  cvs run anc_installation \
+    --cluster_file ~/cvs_workspace/cluster.json \
+    --config_file ~/cvs_workspace/anc/anc_config.json \
+    --html=/var/www/html/cvs/anc.html --capture=tee-sys --self-contained-html \
+    --log-file=/tmp/anc.log -vvv -s
+
+CPU groups
+~~~~~~~~~~
+
+``cvs list anc_test_cpu`` reports one ``test_<group>`` function per CPU group:
+
+.. code:: text
+
+  Available tests in anc_test_cpu:
+    - test_ampttk_full
+    - test_cachewalker_full
+    - test_cpu_all
+    - test_cpu_content_check
+    - test_cpu_mfg_l10
+    - test_cpu_sanity
+    - test_difect_full
+    - test_fpdeluge_full
+    - test_hdrt_full
+    - test_maxcorestim_full
+    - test_memtest_full
+    - test_miidct_full
+    - test_mithac_full
+    - test_weighted_sanity
+
+Run every CPU group (install + ldconfig once, then each group as its own test):
+
+.. code:: bash
+
+  cvs run anc_test_cpu \
+    --cluster_file ~/cvs_workspace/cluster.json \
+    --config_file ~/cvs_workspace/anc/anc_config.json \
+    --html=/var/www/html/cvs/anc_cpu.html --capture=tee-sys --self-contained-html \
+    --log-file=/tmp/anc_cpu.log -vvv -s
+
+Run a single group by function name:
+
+.. code:: bash
+
+  cvs run anc_test_cpu test_cpu_all \
+    --cluster_file ~/cvs_workspace/cluster.json \
+    --config_file ~/cvs_workspace/anc/anc_config.json
+
+GPU groups
+~~~~~~~~~~
+
+``cvs list anc_test_gpu`` reports one ``test_<group>`` function per GPU group:
+
+.. code:: text
+
+  Available tests in anc_test_gpu:
+    - test_gpu_content_check
+    - test_gpu_mfg_l10
+    - test_hbm_lvl1
+    - test_hbm_lvl2
+    - test_hbm_lvl3
+    - test_hbm_lvl4
+    - test_hbm_lvl5
+
+Run every GPU group:
+
+.. code:: bash
+
+  cvs run anc_test_gpu \
+    --cluster_file ~/cvs_workspace/cluster.json \
+    --config_file ~/cvs_workspace/anc/anc_config.json \
+    --html=/var/www/html/cvs/anc_gpu.html --capture=tee-sys --self-contained-html \
+    --log-file=/tmp/anc_gpu.log -vvv -s
+
+Run a single GPU group:
+
+.. code:: bash
+
+  cvs run anc_test_gpu test_hbm_lvl1 \
+    --cluster_file ~/cvs_workspace/cluster.json \
+    --config_file ~/cvs_workspace/anc/anc_config.json
+
+Pass and fail
+=============
+
+A node passes only when ANC started (a ``Log directory`` line is present), ``console.log`` was collected, and the **final** return-code line in ``console.log`` is ``ANC_SUCCESS [0]``. Failures on multiple nodes are aggregated into a single test failure.
+
+Logs land under ``<log_folder_path>/anc_logs/<ip>_<hostname>/<test_name>/<timestamp>/``. When ``COLLECT_HTML_REPORTS`` is ``True`` (the default), CVS also writes a pytest-html report under ``log_folder_path`` even if you omit ``--html``. An explicit ``--html`` on the command line always wins.

docs/how-to/test-suites/burn-in-diag/index.rst

@@ -1,12 +1,12 @@
 .. meta::
   :description: Run burn-in and diagnostic CVS test suites
-  :keywords: CVS, platform, health, preflight, burn-in, diag
+  :keywords: CVS, platform, health, preflight, ANC, burn-in, diag
 
 ********************
 Burn-in / Diag tests
 ********************
 
-Host validation, GPU burn-in, and preflight checks run before network, training, or inference workloads.
+Host validation, GPU burn-in, preflight checks, and AMD Node Check (ANC) diagnostics run before network, training, or inference workloads.
 
 .. list-table::
    :header-rows: 1
@@ -24,5 +24,8 @@ Host validation, GPU burn-in, and preflight checks run before network, training,
    * - Preflight
      - :doc:`/how-to/test-suites/burn-in-diag/preflight`
      - :doc:`/reference/configuration-files/burn-in-diag/preflight`
+   * - ANC (AMD Node Check)
+     - :doc:`/how-to/test-suites/burn-in-diag/anc`
+     - :doc:`/reference/configuration-files/burn-in-diag/anc`
 
 See also :doc:`/how-to/run-tests/index` for common ``cvs run`` flags and workflow.