diff --git a/cvs/input/config_file/preflight/preflight_config.json b/cvs/input/config_file/preflight/preflight_config.json index dbc54bea9..6f8bad2e8 100644 --- a/cvs/input/config_file/preflight/preflight_config.json +++ b/cvs/input/config_file/preflight/preflight_config.json @@ -11,7 +11,7 @@ "_comment_gpus_per_node": "Exact number of AMD GPUs expected on every node.", "_example_expected_rocm_version": "6.2.0", - "expected_rocm_version": "10.0.0", + "expected_rocm_version": "", "_comment_expected_rocm_version": "Expected ROCm version across all cluster nodes. Must match the output of 'amd-smi version' on all nodes. Format: 'major.minor.patch' (e.g., '6.2.0', '5.7.1')." }, @@ -25,11 +25,11 @@ "_comment_connectivity_mode": "RDMA connectivity testing mode. Options: 'basic' (test adjacent node pairs, fast, ~14% coverage for 8 nodes), 'full_mesh' (test all possible node pairs, comprehensive, 100% coverage), 'skip' (skip RDMA connectivity testing entirely).", "_example_gid_index": "3", - "gid_index": "3", + "gid_index": "", "_comment_gid_index": "GID index to check on all RDMA interfaces. Typically '3' for RoCE (RDMA over Converged Ethernet). Must be a valid GID index for your InfiniBand/RoCE setup.", "_example_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0", "rocep158s0", "rocep190s0", "rocep206s0", "rocep222s0"], - "interfaces": ["rdma0", "rdma1", "rdma2", "rdma3", "rdma4", "rdma5", "rdma6", "rdma7"], + "interfaces": [""], "_comment_interfaces": "List of specific RDMA interface names that should be present on all cluster nodes. Examples: ['rocep28s0', 'rocep62s0'] for standard setup, ['mlx5_0', 'mlx5_1'] for Mellanox, ['ib0', 'ib1'] for generic InfiniBand.", "nodes_per_full_mesh_group": 32, diff --git a/cvs/input/config_file/training/jaxmaxtext/README.md b/cvs/input/config_file/training/jaxmaxtext/README.md index 2a0ac0957..3b79a2ee8 100644 --- a/cvs/input/config_file/training/jaxmaxtext/README.md +++ b/cvs/input/config_file/training/jaxmaxtext/README.md @@ -6,7 +6,7 @@ lives in the CVS docs and is the single source of truth: - **Configuration & threshold reference:** [`docs/reference/configuration-files/training/jaxmaxtext.rst`](../../../../../docs/reference/configuration-files/training/jaxmaxtext.rst) - **Running the suites (lifecycle, metrics, reports):** - [`docs/how-to/test-suites/training/jax.rst`](../../../../../docs/how-to/test-suites/training/jax.rst) + [`docs/how-to/test-suites/training/jaxmaxtext.rst`](../../../../../docs/how-to/test-suites/training/jaxmaxtext.rst) The config files and their sibling `_threshold.json` files live in this diff --git a/cvs/input/config_file/training/jaxmaxtext/mi3xx_jaxmaxtext_deepseek-v4-284b_distributed.json b/cvs/input/config_file/training/jaxmaxtext/mi3xx_jaxmaxtext_deepseek-v4-284b_distributed.json index b8d472508..927a0e059 100644 --- a/cvs/input/config_file/training/jaxmaxtext/mi3xx_jaxmaxtext_deepseek-v4-284b_distributed.json +++ b/cvs/input/config_file/training/jaxmaxtext/mi3xx_jaxmaxtext_deepseek-v4-284b_distributed.json @@ -91,6 +91,9 @@ "steps": 30, "enable_checkpointing": false, "attention": "dot_product", + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "", "dataset_type": "synthetic", "remat_policy": "full", "per_device_batch_size": 1, diff --git a/cvs/lib/docker_lib.py b/cvs/lib/docker_lib.py index 62d5f5303..e3e600a93 100644 --- a/cvs/lib/docker_lib.py +++ b/cvs/lib/docker_lib.py @@ -143,6 +143,7 @@ def launch_docker_container( shm_size='64G', timeout=60 * 10, ulimit_nofile=None, + ulimit_memlock=None, ): cmd = f'docker run -d --network {network} --ipc {network} \ --cap-add=IPC_LOCK --security-opt seccomp=unconfined --privileged ' @@ -165,6 +166,11 @@ def launch_docker_container( ) if ulimit_nofile: cmd = cmd + f' --ulimit nofile={ulimit_nofile} ' + if ulimit_memlock is not None: + # RDMA tests register GPU memory via ibv_reg_mr, charged against RLIMIT_MEMLOCK. + # Docker defaults the container to 8MB even when the host is unlimited + # (--privileged does NOT lift ulimits), so registrations >8MB fail with ENOMEM. + cmd = cmd + f' --ulimit memlock={ulimit_memlock} ' for device in device_list: cmd = cmd + f' --device {device} ' for src_vol in volume_dict.keys(): diff --git a/cvs/lib/mori_lib.py b/cvs/lib/mori_lib.py index dac359ec7..a141dbf02 100644 --- a/cvs/lib/mori_lib.py +++ b/cvs/lib/mori_lib.py @@ -185,7 +185,7 @@ def create_env_script( ): cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' export PYTHONPATH={self.mori_dir}:$PYTHONPATH - #export LD_LIBRARY_PATH={self.torchlib_dir}:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH={self.torchlib_dir}:$LD_LIBRARY_PATH export NCCL_SOCKET_IFNAME={self.oob_port} export GLOO_SOCKET_IFNAME={self.oob_port} export GLOO_TCP_IFNAME={self.oob_port} @@ -328,7 +328,13 @@ def run_concurrent_put_threads( mpiexec --allow-run-as-root -np 2 ./build/examples/concurrent_put_thread" ''' out_dict = self.phdl.exec(cmd) for node in out_dict.keys(): - if not re.search('PASSED', out_dict[node], re.I): + # NOTE: concurrent_put_thread / concurrent_put_imm_thread perform NO data + # verification - they launch the kernel, barrier, and print "test done!". + # They never emit "PASSED" (unlike concurrent_put_signal_thread, which does + # validate and prints "...tests passed!"). Accepting the completion marker + # makes this a SMOKE TEST: it proves the binary ran to completion without + # crashing, NOT that the transferred data is correct. + if not re.search(r'PASSED|test done!', out_dict[node], re.I): fail_test('ERROR - test concurrent_put_thread did not run properly, no PASSED test results seen') if re.search('FAIL', out_dict[node], re.I): fail_test('ERROR - one or more concurrent_put_thread tests failed') @@ -342,7 +348,13 @@ def run_concurrent_put_imm_threads( mpiexec --allow-run-as-root -np 2 ./build/examples/concurrent_put_imm_thread" ''' out_dict = self.phdl.exec(cmd) for node in out_dict.keys(): - if not re.search('PASSED', out_dict[node], re.I): + # NOTE: concurrent_put_thread / concurrent_put_imm_thread perform NO data + # verification - they launch the kernel, barrier, and print "test done!". + # They never emit "PASSED" (unlike concurrent_put_signal_thread, which does + # validate and prints "...tests passed!"). Accepting the completion marker + # makes this a SMOKE TEST: it proves the binary ran to completion without + # crashing, NOT that the transferred data is correct. + if not re.search(r'PASSED|test done!', out_dict[node], re.I): fail_test('ERROR - test concurrent_put_imm_thread did not run properly, no PASSED test results seen') if re.search('FAIL', out_dict[node], re.I): fail_test('ERROR - one or more concurrent_put_imm_thread tests failed') diff --git a/cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py b/cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py index 9f5822b94..a82fba062 100644 --- a/cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py +++ b/cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py @@ -519,6 +519,30 @@ def test_sweep_steps_override_drives_yaml_and_poll(self): self.assertIn("steps: 7", written) self.assertNotIn("steps: 3", written) + def test_smoke_steps_override_wins_over_base_maxtext_config(self): + # Smoke passes training.smoke.steps via maxtext_overrides (not training.steps + # alone); base maxtext_config.steps must not leak into the run. + base, _ = _make_job() + base.variant.training.maxtext_config["steps"] = 30 + sweep = SimpleNamespace( + name="SMOKE", + maxtext_overrides={ + "steps": 5, + "per_device_batch_size": 1, + "max_target_length": 2048, + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "", + }, + ) + job = MaxTextTrainingJob(base.orch, base.variant, hf_token="dummy", sweep=sweep) + self.assertEqual(job.steps, 5) + self.assertEqual(job._poll_count, 50) + job._write_maxtext_yaml() + written = " ".join(str(c.args[0]) for c in job.orch.exec.call_args_list) + self.assertIn("steps: 5", written) + self.assertNotIn("steps: 30", written) + def test_empty_string_rendered_as_quoted_not_bare(self): # An empty-string maxtext param (e.g. profiler) must render as 'key: ""', # never bare 'key:' (which YAML reads as null and breaks MaxText enums). diff --git a/cvs/tests/mori/mori_benchmark_test.py b/cvs/tests/mori/mori_benchmark_test.py index b34e51b06..a3dcba866 100644 --- a/cvs/tests/mori/mori_benchmark_test.py +++ b/cvs/tests/mori/mori_benchmark_test.py @@ -175,6 +175,7 @@ def test_launch_mori_container(phdl, mori_dict): mori_dict['container_config']['env_dict'], shm_size='48G', timeout=60 * 20, + ulimit_memlock='-1', ) # ADD verifications .. time.sleep(30) diff --git a/cvs/tests/training/jaxmaxtext/README.md b/cvs/tests/training/jaxmaxtext/README.md index ca53f388c..b174ded81 100644 --- a/cvs/tests/training/jaxmaxtext/README.md +++ b/cvs/tests/training/jaxmaxtext/README.md @@ -4,7 +4,7 @@ The documentation for the JAX MaxText **test suites** now lives in the CVS docs and is the single source of truth: - **Running the suites (lifecycle, metrics, reports):** - [`docs/how-to/test-suites/training/jax.rst`](../../../../docs/how-to/test-suites/training/jax.rst) + [`docs/how-to/test-suites/training/jaxmaxtext.rst`](../../../../docs/how-to/test-suites/training/jaxmaxtext.rst) - **Configuration & threshold reference:** [`docs/reference/configuration-files/training/jaxmaxtext.rst`](../../../../docs/reference/configuration-files/training/jaxmaxtext.rst) diff --git a/cvs/tests/training/jaxmaxtext/_common.py b/cvs/tests/training/jaxmaxtext/_common.py index 5df56ee1b..1b7ee869e 100644 --- a/cvs/tests/training/jaxmaxtext/_common.py +++ b/cvs/tests/training/jaxmaxtext/_common.py @@ -276,6 +276,7 @@ def smoke(orch, variant_config, hf_token, lifecycle, request): smoke_sweep = SimpleNamespace( name="SMOKE", maxtext_overrides={ + "steps": steps, "per_device_batch_size": batch, "max_target_length": seqlen, "dtype": "bfloat16", @@ -439,6 +440,7 @@ def _mk_job(total_steps, extra_overrides, enable_ckpt): v.training.enable_checkpointing = enable_ckpt ov = dict(base_overrides) ov.update(extra_overrides or {}) + ov["steps"] = total_steps sweep = SimpleNamespace(name="CKPT", maxtext_overrides=ov) return MaxTextTrainingJob(orch, v, hf_token, sweep=sweep) diff --git a/docs/concepts/what-is-cvs.rst b/docs/concepts/what-is-cvs.rst index e436c7b8f..4e3a2cc5f 100644 --- a/docs/concepts/what-is-cvs.rst +++ b/docs/concepts/what-is-cvs.rst @@ -11,7 +11,7 @@ Use CVS to verify GPU cluster health, GPU/CPU node health, host OS configuration Here are the tests available in the CVS: - **Platform tests**: Perform host OS configuration, BIOS, firmware/driver, and network configuration checks. -- **Burn-in health tests**: Perform `AMD GPU Field Health Check (AGFHC) `_, `TransferBench `_, and `ROCm Validation Suite (RVS) `_. +- **Burn-in health tests**: Perform `AMD GPU Field Health Check (AGFHC) `_, `TransferBench `_, and `ROCm Validation Suite (RVS) `_. For MI4XX platforms, the suite also includes AMD Node Check (ANC). - **InfiniBand (IB Perf)**: These tests are low-level network performance benchmarks that validate the raw communication capabilities of InfiniBand adapters and interconnects. These tests measure the fundamental building blocks on which RCCL and other high-level libraries depend. - **Network tests**: Perform ping checks and multi-node `ROCm Communication Collectives Library (RCCL) `_ validations for different collectives. - **Distributed training tests**: Run and validate Llama and DeepSeek model trainings on single-node or multi-node clusters with the `JAX `_ MaxText and `Megatron `_ frameworks. diff --git a/docs/how-to/configure/test-suite-config/pick-config-file.rst b/docs/how-to/configure/test-suite-config/pick-config-file.rst index 1cdfe74f9..73b8ed3fb 100644 --- a/docs/how-to/configure/test-suite-config/pick-config-file.rst +++ b/docs/how-to/configure/test-suite-config/pick-config-file.rst @@ -36,6 +36,11 @@ Burn-in / Diag - ``cvs config list preflight`` `README_preflight_config.md `_ + * - ANC + - ``input/config_file/anc/anc_config.json`` + - ``cvs config list anc`` + + `cvs/tests/anc/README.md `_ Network ======= diff --git a/docs/how-to/run-tests/index.rst b/docs/how-to/run-tests/index.rst index 91de48294..08e5d915c 100644 --- a/docs/how-to/run-tests/index.rst +++ b/docs/how-to/run-tests/index.rst @@ -76,7 +76,7 @@ Test suites Per-suite **Set up config** and **Run tests** steps are grouped by category: -- :doc:`Burn-in / Diag tests ` — platform, health, preflight +- :doc:`Burn-in / Diag tests ` — platform, health, preflight, ANC - :doc:`Network tests ` — IB Perf, RCCL, MORI - :doc:`Training tests ` — Aorta, JAX MaxText, Megatron - :doc:`Inference tests ` — vLLM, ATOM, SGLang, xDiT diff --git a/docs/how-to/test-suites/burn-in-diag/anc.rst b/docs/how-to/test-suites/burn-in-diag/anc.rst new file mode 100644 index 000000000..8bab38740 --- /dev/null +++ b/docs/how-to/test-suites/burn-in-diag/anc.rst @@ -0,0 +1,140 @@ +.. meta:: + :description: Run AMD Node Check (ANC) CPU and GPU diagnostic suites + :keywords: CVS, ANC, AMD Node Check, burn-in, diag + +********* +ANC tests +********* + +AMD Node Check (ANC) runs CPU and GPU diagnostic groups on every node in the cluster. CVS installs ANC when needed, invokes each group as ``sudo ./anc.py -g ``, and collects logs and HTML reports. + +ANC requires **root**. The runner must have passwordless SSH and passwordless ``sudo`` on every target node. Without passwordless ``sudo``, group runs and log collection fail. + +.. _anc-set-up-config: + +Set up config +============= + +1. Copy the ANC configuration file: + + .. code:: bash + + cvs config copy anc/anc_config.json --output ~/cvs_workspace/anc/anc_config.json + +2. Replace every ```` placeholder: + + - ``anc_release_url`` — URL of the ANC release archive to download and install + - ``log_folder_path`` — controller-side directory prefix for collected logs and auto-generated HTML reports + - ``anc_version`` — expected ANC version (must match the version in ``anc_release_url``) + +3. Optionally set ``ANC_INSTALL_PATH`` for relocatable **tar** installs. Deb and rpm packages ignore this key and always install under ``/opt/amdtools``. + +Full parameter list: :doc:`/reference/configuration-files/burn-in-diag/anc`. + +.. _anc-run-tests: + +Run tests +========= + +List the install suite and the CPU / GPU group suites: + +.. code:: bash + + cvs list anc_installation + cvs list anc_test_cpu + cvs list anc_test_gpu + +Install ANC +~~~~~~~~~~~ + +Every CPU and GPU group run installs ANC as a session-cached pre-task, so a separate install step is optional. Run ``anc_installation`` when you want to install or refresh ANC without running a validation group: + +.. code:: bash + + cvs run anc_installation \ + --cluster_file ~/cvs_workspace/cluster.json \ + --config_file ~/cvs_workspace/anc/anc_config.json \ + --html=/var/www/html/cvs/anc.html --capture=tee-sys --self-contained-html \ + --log-file=/tmp/anc.log -vvv -s + +CPU groups +~~~~~~~~~~ + +``cvs list anc_test_cpu`` reports one ``test_`` function per CPU group: + +.. code:: text + + Available tests in anc_test_cpu: + - test_ampttk_full + - test_cachewalker_full + - test_cpu_all + - test_cpu_content_check + - test_cpu_mfg_l10 + - test_cpu_sanity + - test_difect_full + - test_fpdeluge_full + - test_hdrt_full + - test_maxcorestim_full + - test_memtest_full + - test_miidct_full + - test_mithac_full + - test_weighted_sanity + +Run every CPU group (install + ldconfig once, then each group as its own test): + +.. code:: bash + + cvs run anc_test_cpu \ + --cluster_file ~/cvs_workspace/cluster.json \ + --config_file ~/cvs_workspace/anc/anc_config.json \ + --html=/var/www/html/cvs/anc_cpu.html --capture=tee-sys --self-contained-html \ + --log-file=/tmp/anc_cpu.log -vvv -s + +Run a single group by function name: + +.. code:: bash + + cvs run anc_test_cpu test_cpu_all \ + --cluster_file ~/cvs_workspace/cluster.json \ + --config_file ~/cvs_workspace/anc/anc_config.json + +GPU groups +~~~~~~~~~~ + +``cvs list anc_test_gpu`` reports one ``test_`` function per GPU group: + +.. code:: text + + Available tests in anc_test_gpu: + - test_gpu_content_check + - test_gpu_mfg_l10 + - test_hbm_lvl1 + - test_hbm_lvl2 + - test_hbm_lvl3 + - test_hbm_lvl4 + - test_hbm_lvl5 + +Run every GPU group: + +.. code:: bash + + cvs run anc_test_gpu \ + --cluster_file ~/cvs_workspace/cluster.json \ + --config_file ~/cvs_workspace/anc/anc_config.json \ + --html=/var/www/html/cvs/anc_gpu.html --capture=tee-sys --self-contained-html \ + --log-file=/tmp/anc_gpu.log -vvv -s + +Run a single GPU group: + +.. code:: bash + + cvs run anc_test_gpu test_hbm_lvl1 \ + --cluster_file ~/cvs_workspace/cluster.json \ + --config_file ~/cvs_workspace/anc/anc_config.json + +Pass and fail +============= + +A node passes only when ANC started (a ``Log directory`` line is present), ``console.log`` was collected, and the **final** return-code line in ``console.log`` is ``ANC_SUCCESS [0]``. Failures on multiple nodes are aggregated into a single test failure. + +Logs land under ``/anc_logs/_///``. When ``COLLECT_HTML_REPORTS`` is ``True`` (the default), CVS also writes a pytest-html report under ``log_folder_path`` even if you omit ``--html``. An explicit ``--html`` on the command line always wins. diff --git a/docs/how-to/test-suites/burn-in-diag/index.rst b/docs/how-to/test-suites/burn-in-diag/index.rst index c4a0883a7..c6ed0c118 100644 --- a/docs/how-to/test-suites/burn-in-diag/index.rst +++ b/docs/how-to/test-suites/burn-in-diag/index.rst @@ -1,12 +1,12 @@ .. meta:: :description: Run burn-in and diagnostic CVS test suites - :keywords: CVS, platform, health, preflight, burn-in, diag + :keywords: CVS, platform, health, preflight, ANC, burn-in, diag ******************** Burn-in / Diag tests ******************** -Host validation, GPU burn-in, and preflight checks run before network, training, or inference workloads. +Host validation, GPU burn-in, preflight checks, and AMD Node Check (ANC) diagnostics run before network, training, or inference workloads. .. list-table:: :header-rows: 1 @@ -24,5 +24,8 @@ Host validation, GPU burn-in, and preflight checks run before network, training, * - Preflight - :doc:`/how-to/test-suites/burn-in-diag/preflight` - :doc:`/reference/configuration-files/burn-in-diag/preflight` + * - ANC (AMD Node Check) + - :doc:`/how-to/test-suites/burn-in-diag/anc` + - :doc:`/reference/configuration-files/burn-in-diag/anc` See also :doc:`/how-to/run-tests/index` for common ``cvs run`` flags and workflow. diff --git a/docs/reference/configuration-files/burn-in-diag/anc.rst b/docs/reference/configuration-files/burn-in-diag/anc.rst new file mode 100644 index 000000000..5a30f17de --- /dev/null +++ b/docs/reference/configuration-files/burn-in-diag/anc.rst @@ -0,0 +1,105 @@ +.. meta:: + :description: Configure the AMD Node Check (ANC) configuration file + :keywords: ANC, AMD Node Check, ROCm, cluster, validation, burn-in + +*************************** +ANC configuration file +*************************** + +AMD Node Check (ANC) suites install the ANC tool on every node, then run CPU or GPU diagnostic groups. Shared logic lives in ``cvs/lib/anc_lib.py``; group names come from ``CPU_GROUPS`` and ``GPU_GROUPS``. + +Configuration file location: ``cvs/input/config_file/anc/anc_config.json`` + +For the suite overview, install flavours, and artifact layout, see +`cvs/tests/anc/README.md `_ +in the repository. + +Run ANC +======= + +.. code:: bash + + cvs run anc_installation \ + --cluster_file cluster.json \ + --config_file cvs/input/config_file/anc/anc_config.json + + cvs run anc_test_cpu \ + --cluster_file cluster.json \ + --config_file cvs/input/config_file/anc/anc_config.json + + cvs run anc_test_gpu \ + --cluster_file cluster.json \ + --config_file cvs/input/config_file/anc/anc_config.json + +Replace ```` in ``anc_release_url`` and ``log_folder_path`` before running. An unresolved placeholder aborts the run before any node is contacted. + +Sample configuration +==================== + +Keys prefixed with ``_comment`` are documentation only and ignored at runtime. + +.. note:: + + In this configuration file, ``{home}`` and ``{user-id}`` are resolved at load time. A leading ``~`` is expanded to the runner user's home directory. + +.. dropdown:: ``anc_config.json`` (example) + + .. code:: json + + { + "anc": { + "description": "AMD Node Check", + "inactivity_timeout": 900, + "install_timeout": 1800, + "anc_version": "1.4.9", + "anc_release_url": "", + "ANC_INSTALL_PATH": "", + "print_all_to_console": "True", + "log_folder_path": "", + "ADD_ANC_LOGS_TO_HTML_REPORTS": "False", + "COLLECT_HTML_REPORTS": "True" + } + } + +Key parameters +============== + +.. list-table:: + :header-rows: 1 + :widths: 28 72 + + * - Key + - Meaning + * - ``inactivity_timeout`` + - Per-group inactivity timeout in seconds (default ``900``). A group is aborted only after this many seconds with **no new ANC output**. There is no total wall-clock cap. + * - ``install_timeout`` + - Download and install inactivity timeout in seconds (default ``1800``), used only by ``anc_installation`` / the install pre-task. Not a total budget: a progress heartbeat keeps a slow download alive; a genuine stall still fails. + * - ``anc_version`` + - Expected ANC version. Install skips when this version is already present and post-verifies the match. When set, it must equal the version in ``anc_release_url``. + * - ``anc_release_url`` + - URL of the ANC release archive. Packaging is auto-detected from the filename: **legacy (≤1.4.x)** outer tarballs include a ``-deb-`` / ``-rpm-`` / ``-tar-`` token; **direct (1.5.0+)** URLs point at a ``.deb`` / ``.rpm`` / ``.tar.gz`` with no flavour token. + * - ``ANC_INSTALL_PATH`` + - **Tar installs only:** relocatable prefix (entrypoint ``/anc/anc.py``). Deb/rpm packages ignore this key and install under ``/opt/amdtools``. Leave blank to keep the default ``/opt/amdtools``. + * - ``print_all_to_console`` + - ``True`` echoes ANC group output to the console; ``False`` suppresses it (install and ldconfig diagnostics still print). + * - ``log_folder_path`` + - Controller-side destination **prefix** for collected logs and the auto-collected HTML report. Required. CVS appends ``anc_logs///`` and ``html_reports////``. + * - ``ADD_ANC_LOGS_TO_HTML_REPORTS`` + - ``True`` always bundles each node's ANC log tarball into the pytest-html report. ``False`` (default) bundles tarballs only when the test fails. + * - ``COLLECT_HTML_REPORTS`` + - ``True`` (default) auto-generates a pytest-html report under ``log_folder_path`` even without ``--html``. An explicit ``--html`` always overrides that path. + +Install location +================ + +- **deb / rpm** — always ``/opt/amdtools/anc`` +- **tar** — ``ANC_INSTALL_PATH`` (default ``/opt/amdtools``), giving ``/anc/anc.py`` + +The download is staged in a private temp directory on each node and removed after install (success or failure). + +Further reading +=============== + +- :doc:`/how-to/test-suites/burn-in-diag/anc` — ``cvs run`` examples and group lists +- :doc:`/reference/cluster/cluster-file` — Cluster topology and SSH +- `cvs/tests/anc/README.md `_ — Install flavours, pass/fail criteria, and artifact layout diff --git a/docs/reference/configuration-files/burn-in-diag/index.rst b/docs/reference/configuration-files/burn-in-diag/index.rst index ccf0d487f..0c9b177a7 100644 --- a/docs/reference/configuration-files/burn-in-diag/index.rst +++ b/docs/reference/configuration-files/burn-in-diag/index.rst @@ -1,15 +1,16 @@ .. meta:: :description: Burn-in and diagnostic test configuration schemas - :keywords: CVS, platform, health, preflight, config schema + :keywords: CVS, platform, health, preflight, ANC, config schema **************** Burn-in / Diag **************** -JSON configuration schemas for host validation, GPU burn-in, and preflight suites under ``cvs/input/config_file/``. +JSON configuration schemas for host validation, GPU burn-in, preflight, and ANC suites under ``cvs/input/config_file/``. - :doc:`Platform ` — host OS, BIOS, firmware, and PCIe checks - :doc:`Health ` — AGFHC, TransferBench, and RVS burn-in configs - :doc:`Preflight ` — node smoke and cluster preflight checks +- :doc:`ANC ` — AMD Node Check CPU and GPU diagnostic groups How to run these suites: :doc:`/how-to/test-suites/burn-in-diag/index`. diff --git a/docs/reference/configuration-files/index.rst b/docs/reference/configuration-files/index.rst index 95a7fa91c..24bf44f68 100644 --- a/docs/reference/configuration-files/index.rst +++ b/docs/reference/configuration-files/index.rst @@ -35,6 +35,7 @@ Burn-in / Diag - :doc:`Platform ` — host OS, BIOS, firmware, and PCIe checks - :doc:`Health ` — AGFHC, TransferBench, and RVS burn-in configs - :doc:`Preflight ` — node smoke and cluster preflight checks +- :doc:`ANC ` — AMD Node Check CPU and GPU diagnostic groups Network ------- diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index 993bedd76..d1414f2df 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -44,6 +44,8 @@ subtrees: title: Health (burn-in) tests - file: how-to/test-suites/burn-in-diag/preflight title: Preflight tests + - file: how-to/test-suites/burn-in-diag/anc + title: ANC tests - file: how-to/test-suites/network/index title: Network tests subtrees: @@ -119,6 +121,8 @@ subtrees: title: Health - file: reference/configuration-files/burn-in-diag/preflight title: Preflight + - file: reference/configuration-files/burn-in-diag/anc + title: ANC - file: reference/configuration-files/network/index title: Network subtrees: