diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index c7a630df5..935efac81 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -40,7 +40,13 @@ jobs: echo "printing conda config just in case" conda config --show + - name: Conda install and set libmamba conda solver + run: | + conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda config --set solver libmamba + - name: Conda install conda-build + timeout-minutes: 10 run: | echo "conda install conda-build" conda install conda-forge::conda-build @@ -88,11 +94,22 @@ jobs: echo "printing conda config just in case" conda config --show + - name: Conda install and set libmamba conda solver + run: | + conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda config --set solver libmamba + - name: Conda install conda-build run: | echo "conda install conda-build" conda install conda-forge::conda-build + - name: Check conda-build version and supported package formats + run: | + conda --version + conda build --version + conda build --help + - name: Download tarball uses: actions/download-artifact@v5 with: diff --git a/fre/list_/frelist.py b/fre/list_/frelist.py index afad49a07..bc1f65756 100644 --- a/fre/list_/frelist.py +++ b/fre/list_/frelist.py @@ -44,9 +44,14 @@ def platforms(yamlfile): type=str, help="Experiment to be post-processed", required=True) -def pp_components(yamlfile, experiment): +@click.option("-a", + "--application", + type=str, + help="Post-processing application name.", + required=True) +def pp_components(yamlfile, experiment, application): """ - List components to be post-processed for a defined experiment""" - list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment) + list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment, application) @list_cli.command() @click.option("-y", diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index ecf205868..237c6cd75 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -5,15 +5,7 @@ """ import logging -import yaml - -from fre.yamltools.info_parsers import pp_info_parser as ppip - -## to-do: figure out validation -# Imports associated with commented block -# from pathlib import Path -# from fre.yamltools import helpers -# from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy fre_logger = logging.getLogger(__name__) @@ -29,34 +21,15 @@ def list_experiments_subtool(yamlfile: str): platform = None target = None - # Combine model - # Create pp yaml instance - yamldict = ppip.InitPPYaml(yamlfile, exp, platform, target) - yaml_str = yamldict.combine_model() - yaml_dict = yaml.load(yaml_str, Loader = yaml.Loader) + yaml_dict = cy.yamltools_combine_subtool(yamls = yamlfile, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = True) -## COULD HAVE been one way to validate but section we'd want to parse was -## cleaned in final/"combined" yaml information -## Currently not a way to validate model yaml information because we only -## have schemas for the final "combined" compile or pp information (both -## of which remove the "experiments" section I believe -# exp = yamlfile.split("/")[-1].split(".")[0] -# platform = "None" -# target = "None" -# -# # Combine model / experiment -# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, -# experiment = exp, -# platform = platform, -# target = target, -# use = "compile", -# output = None) -# -# # Validate combined yaml information -# frelist_dir = Path(__file__).resolve().parents[2] -# schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json" -# # from fre.yamltools -# helpers.validate_yaml(yml_dict, schema_path) +#### TO-DO: CURRENTLY NO YAML PIECE VALIDATION +### No way to validate just the model yaml # set logger level to INFO former_log_level = fre_logger.level @@ -64,9 +37,9 @@ def list_experiments_subtool(yamlfile: str): # log the experiment names, which should show up on screen for sure fre_logger.info("Experiments found:") - for i in yaml_dict.get("experiments"): - fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") + + for i in yaml_dict["experiments"].keys(): + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index 2ebc0a0e5..7d36d3703 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -6,7 +6,8 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) @@ -22,17 +23,33 @@ def list_platforms_subtool(yamlfile: str): former_log_level = fre_logger.level fre_logger.setLevel(logging.INFO) - exp = yamlfile.split("/")[-1].split(".")[0] + exp = None platform = None target = None - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "compile", - output = None) +# model_yf_path = Path(yamlfile).resolve().parent +# with open(yamlfile, 'r') as yf: +# yml = yaml.safe_load(yf) +# +# platform_yf_path = f"{model_yf_path}/{yml['build']['platformYaml']}" + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = None) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "compile", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) # Validate the yaml fre_pkg_dir = Path(__file__).resolve().parents[1] @@ -41,8 +58,7 @@ def list_platforms_subtool(yamlfile: str): helpers.validate_yaml(yml_dict, schema_path) fre_logger.info("Platforms available:") - for i in yml_dict.get("platforms"): + for i in yml_dict["platforms"]: fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_pp_components_script.py b/fre/list_/list_pp_components_script.py index 0be6128c7..5f475c5fb 100644 --- a/fre/list_/list_pp_components_script.py +++ b/fre/list_/list_pp_components_script.py @@ -19,12 +19,13 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) -def list_ppcomps_subtool(yamlfile: str, experiment: str): +def list_ppcomps_subtool(yamlfile: str, experiment: str, application: str): """ List_ppcomps_subtool lists the components to be post-processed. @@ -32,6 +33,8 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): :type yamlfile: str :param experiment: is the experiment name defined in the model.yaml :type experiment: str + :param application: + :type applicatin: str """ # set logger level to INFO former_log_level = fre_logger.level @@ -40,14 +43,26 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): exp = experiment platform = None target = None + app = application + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = app) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "pp", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "pp", - output = None) # Validate combined yaml information frelist_dir = Path(__file__).resolve().parents[2] @@ -55,15 +70,15 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): # from fre.yamltools helpers.validate_yaml(yml_dict, schema_path) + comp_info = yml_dict["postprocess"]["components"] # log the experiment names, which should show up on screen for sure fre_logger.info("Components to be post-processed:") - for i in yml_dict["postprocess"]["components"]: - if "postprocess_on" in i: - if i.get("postprocess_on") is True: - fre_logger.info(' - %s', i.get("type")) + for i in comp_info.keys(): + if "postprocess_on" in comp_info[i]: + if comp_info[i].get("postprocess_on") is True: + fre_logger.info(' - %s', i) else: - fre_logger.info(' - %s', i.get("type")) - fre_logger.info("\n") + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index f78bf1f93..2b9c96f2d 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -4,40 +4,36 @@ from pathlib import Path import pytest -import yaml from fre.list_ import list_experiments_script from fre.yamltools import helpers - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") -YAMLFILE = "null_model.yaml" +TEST_DIR = Path("fre/list_/tests/yamls") +YAMLFILE = "model.yaml" EXP_NAME = "None" # yaml file checks def test_modelyaml_exists(): ''' Test that model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() # Test whole tool def test_exp_list(caplog): ''' Test fre list exps subtool ''' - list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = [ 'Experiments found:', - ' - null_model_full', - ' - null_model_0', - ' - null_model_1', - ' - null_model_2' ] + ' - experiment1', + ' - experiment2' ] for i in check_out: assert i in caplog.text # make sure the level is INFO for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation @pytest.mark.skip( @@ -48,10 +44,10 @@ def test_exp_list(caplog): ) def test_yamlvalidate(): ''' Test yaml is being validated ''' - yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yamlfilepath = Path(f"{TEST_DIR}/{YAMLFILE}") # Combine model / experiment - yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # Validate and capture output assert helpers.validate_yaml(yml_dict, VAL_SCHEMA) diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index 8dd7eba56..16e6ecd2f 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -4,63 +4,57 @@ from pathlib import Path import pytest -import yaml - from fre.list_ import list_platforms_script -from fre.yamltools import combine_yamls_script as cy - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") +TEST_DIR = Path("fre/list_/tests/yamls") PLATFORM = None TARGET = None -YAMLFILE = "null_model.yaml" -EXP_NAME = YAMLFILE.split(".")[0] +YAMLFILE = "model.yaml" +EXP_NAME = YAMLFILE.split(".", maxsplit=1)[0] VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_make.json") # Bad yaml example -BADYAMLFILE_PATH = f"{TEST_DIR}/{NM_EXAMPLE}/wrong_model/wrong_null_model.yaml" +BADYAMLFILE_PATH = f"{TEST_DIR}/wrong_null_model.yaml" # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() def test_compileyaml_exists(): ''' Test compile yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() + assert Path(f"{TEST_DIR}/compile.yaml").exists() def test_platformyaml_exists(): ''' Test platforms yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() + assert Path(f"{TEST_DIR}/platforms.yaml").exists() # Test whole tool def test_platforms_list_correct(caplog): ''' Test fre list platforms subtool ''' - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = ["Platforms available:", " - ncrc5.intel23", - " - hpcme.2023", + " - hpcmini.2025", " - ci.gnu", - " - con.twostep" ] + " - hpcme.intel25" ] for i in check_out: assert i in caplog.text # make sure level is INFO for record in caplog.records: - record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation def test_yamlvalidate(caplog): ''' Test yaml is being validated and is actually valid''' - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - # Combine model / experiment - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -69,7 +63,8 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" def test_not_valid_yaml(): ''' Test the correct output matches the ValueError raised when yaml is invalid ''' diff --git a/fre/list_/tests/test_list_pp_components_script.py b/fre/list_/tests/test_list_pp_components_script.py index f273d955c..fb0b30388 100644 --- a/fre/list_/tests/test_list_pp_components_script.py +++ b/fre/list_/tests/test_list_pp_components_script.py @@ -6,12 +6,13 @@ # SET-UP -TEST_DIR = Path("fre/pp/tests") -AM5_EXAMPLE = Path("AM5_example") -MODEL_YAMLFILE = "am5.yaml" +#TEST_DIR = Path("fre/pp/tests") +TEST_DIR = Path("fre/list_/tests/yamls") +#AM5_EXAMPLE = Path("AM5_example") +MODEL_YAMLFILE = "model.yaml" SETTINGS_YAMLFILE = "yaml_include/settings.yaml" -PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml", "yaml_include/settings.yaml"] -EXP_NAME = "c96L65_am5f7b12r1_amip" +PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml"] #, "yaml_include/settings.yaml"] +EXP_NAME = "experiment1" VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_pp.json") PLATFORM = "FOO" TARGET = "BAR" @@ -19,44 +20,45 @@ # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{MODEL_YAMLFILE}").exists() def test_settingsyaml_exist(): ''' Test settings yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{SETTINGS_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{SETTINGS_YAMLFILE}").exists() def test_ppyamls_exist(): ''' Test post-processing yamls exist ''' for pp_yaml in PP_YAMLFILES: - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{pp_yaml}").exists() + assert Path(f"{TEST_DIR}/{pp_yaml}").exists() # Test whole tool def test_pp_comp_list(caplog): ''' Test fre list pp-components subtool ''' - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") # check the logging output check_out = [ 'Components to be post-processed:', ' - atmos_cmip', ' - atmos', - ' - atmos_level_cmip', - ' - atmos_level', - ' - atmos_month_aer', ' - atmos_diurnal', - ' - atmos_scalar'] - + ' - atmos_scalar', + ' - atmos_cmip-TEST' ] for i in check_out: assert i in caplog.text - # make sure the level is WARNING for version mismatch and INFO otherwise for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] # Test validation def test_yamlvalidate(caplog): ''' Test yaml is being validated ''' # Combine model / experiment - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -65,5 +67,5 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] - + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] diff --git a/fre/list_/tests/test_list_yamls_script.py b/fre/list_/tests/test_list_yamls_script.py index e89ca0cbe..d909d355e 100644 --- a/fre/list_/tests/test_list_yamls_script.py +++ b/fre/list_/tests/test_list_yamls_script.py @@ -67,27 +67,24 @@ def test_list_yamls_runtime_only(self, model_yaml_path): def test_list_yamls_postprocess_only(self, model_yaml_path): """Test postprocess_only flag (without analysis) returns model YAML plus postprocessing YAMLs""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="experiment1", - application="postprocess" - ) + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="experiment1", + application="postprocess" + ) - yaml_names = [Path(y).name for y in result.split(",")] + yaml_names = [Path(y).name for y in result.split(",")] - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain settings and postprocessing YAMLs - assert "settings.yaml" in yaml_names - assert "pp.c96_amip.yaml" in yaml_names - assert "pp-test.c96_amip.yaml" in yaml_names - - # Should NOT contain compile, platform, run, or analysis YAMLs - assert "compile.yaml" not in yaml_names - assert "platforms.yaml" not in yaml_names - assert "run1.yaml" not in yaml_names - assert "analysis1.yaml" not in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain settings and postprocessing YAMLs + assert "settings.yaml" in yaml_names + assert "pp.c96_amip.yaml" in yaml_names + assert "pp-test.c96_amip.yaml" in yaml_names + # Should NOT contain compile, platform, run, or analysis YAMLs + assert "compile.yaml" not in yaml_names + assert "run1.yaml" not in yaml_names + assert "analysis1.yaml" not in yaml_names def test_list_yamls_analysis_only(self, model_yaml_path): """Test analysis_only flag (without postprocess) returns model YAML plus analysis YAMLs""" @@ -138,23 +135,23 @@ def test_list_yamls_postprocess_and_analysis(self, model_yaml_path): def test_list_yamls_no_experiment(self, model_yaml_path): """Test with no experiment name provided (default compile behavior)""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="", - application=None - ) +# with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="", + application=None + ) - yaml_names = [Path(y).name for y in result.split(",")] + yaml_names = [Path(y).name for y in result.split(",")] - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain compile and platform YAMLs - assert "compile.yaml" in yaml_names - assert "platforms.yaml" in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain compile and platform YAMLs + assert "compile.yaml" in yaml_names + assert "platforms.yaml" in yaml_names - # Should NOT contain experiment-specific YAMLs - assert "run1.yaml" not in yaml_names + # Should NOT contain experiment-specific YAMLs + assert "run1.yaml" not in yaml_names def test_list_yamls_returns_full_paths(self, model_yaml_path): """Test that returned YAMLs have full paths""" diff --git a/fre/list_/tests/yamls/compile.yaml b/fre/list_/tests/yamls/compile.yaml new file mode 100644 index 000000000..51a72b73c --- /dev/null +++ b/fre/list_/tests/yamls/compile.yaml @@ -0,0 +1,49 @@ +compile: + experiment: "null_model_full" + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + additionalInstructions: "echo \"Testing additionalInstructions\"" + cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" + otherFlags: "-fallow-argument-mismatch" # only needed for gcc + branch: "2025.04" + - component: "atmos_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/atmos_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "land_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/land_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_param" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ocean_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_null.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_null" + requires: ["FMS", "ice_param", "ocean_null"] + repo: "https://github.com/NOAA-GFDL/ice_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "coupler" + requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML" + paths: [ "coupler/full", + "coupler/shared" ] + diff --git a/fre/list_/tests/yamls/empty_platforms.yaml b/fre/list_/tests/yamls/empty_platforms.yaml new file mode 100644 index 000000000..9bceccc06 --- /dev/null +++ b/fre/list_/tests/yamls/empty_platforms.yaml @@ -0,0 +1 @@ +platforms: diff --git a/fre/list_/tests/yamls/model.yaml b/fre/list_/tests/yamls/model.yaml index 43cc46edb..035dbdcf3 100644 --- a/fre/list_/tests/yamls/model.yaml +++ b/fre/list_/tests/yamls/model.yaml @@ -1,22 +1,25 @@ # reusable variables fre_properties: - VERSION: "foo" - FRE_STEM: "test_list/{{ fre_properties.VERSION }}" + VERSION: "foo" + FRE_STEM: "test_list/{{ fre_properties.VERSION }}" # amip - EXP_AMIP_START: "19790101T0000Z" - EXP_AMIP_END: "20200101T0000Z" - ANA_AMIP_START: "1980" - ANA_AMIP_END: "2020" + EXP_AMIP_START: "19790101T0000Z" + EXP_AMIP_END: "20200101T0000Z" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" - PP_AMIP_CHUNK96: "P1Y" - PP_XYINTERP96: "180,288" - PP_XYINTERP384: "720,1152" + PP_AMIP_CHUNK96: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" # compile information - INTEL: "intel-classic" - FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + branch: "test" + + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" # compile information build: @@ -33,3 +36,4 @@ experiments: - "yaml_include/pp-test.c96_amip.yaml" analysis: - "analysis1.yaml" + experiment2: diff --git a/fre/list_/tests/yamls/platforms.yaml b/fre/list_/tests/yamls/platforms.yaml new file mode 100644 index 000000000..00d1c74fc --- /dev/null +++ b/fre/list_/tests/yamls/platforms.yaml @@ -0,0 +1,34 @@ +platforms: + - name: ncrc5.intel23 + compiler: intel + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", + "source $MODULESHOME/init/sh", + "module load {{ fre_properties.INTEL }}/2023.2.0", + "module load fre/bronx-23", + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" + modelRoot: ${HOME}/fremake_canopy/test + - name: ci.gnu + compiler: gnu + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test + - name: hpcmini.2025 + compiler: gnu + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "ghcr.io/rem1776/hpc-mini:alpine3gcc14.2" + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + - name: hpcme.intel25 + compiler: intel + RUNenv: "" + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3" + mkTemplate: "/apps/mkmf/templates/hpcme-intel2025.03-oneapi.mk" + container2step: True + container2base: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3rte" diff --git a/fre/list_/tests/yamls/wrong_null_model.yaml b/fre/list_/tests/yamls/wrong_null_model.yaml new file mode 100644 index 000000000..ccec12827 --- /dev/null +++ b/fre/list_/tests/yamls/wrong_null_model.yaml @@ -0,0 +1,15 @@ +# reusable variables +fre_properties: + VERSION: "full" + FRE_STEM: "null_model/{{ fre_properties.VERSION }}" + # compile information + branch: "main" + INTEL: "intel-classic" + FMSincludes: "-IFMS/include" + +build: + compileYaml: "compile.yaml" + platformYaml: "empty_platforms.yaml" + +experiments: + experiment1: diff --git a/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..b54f0fc77 --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "200,200" + +#c96_amip_postprocess: +postprocess: + components: + atmos_cmip-TEST: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.custom_interp}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos-TEST: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level_cmip-TEST: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml similarity index 50% rename from fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml rename to fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml index 3859217bd..a06f8162e 100644 --- a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml +++ b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml @@ -1,87 +1,59 @@ # local reusable variable overrides fre_properties: - - &custom_interp "180,360" + custom_interp: "180,360" #c96_amip_postprocess: postprocess: # main pp instructions components: - - type: "atmos_cmip" + atmos_cmip: sources: - history_file: "atmos_month_cmip" - history_file: "atmos_8xdaily_cmip" - history_file: "atmos_daily_cmip" sourceGrid: "cubedsphere" - xyInterp: *custom_interp + xyInterp: "{{fre_properties.custom_interp}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos" + atmos: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_level_cmip" + postprocess_on: True + atmos_level_cmip: sources: - history_file: "atmos_level_cmip" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_level" + postprocess_on: False + atmos_level: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_month_aer" + postprocess_on: False + atmos_month_aer: sources: - history_file: "atmos_month_aer" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order1" inputRealm: 'atmos' - - type: "atmos_diurnal" + postprocess_on: False + atmos_diurnal: sources: - history_file: "atmos_diurnal" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_scalar" + atmos_scalar: sources: - history_file: "atmos_scalar" - - type: "aerosol_cmip" - xyInterp: *PP_XYINTERP96 - sources: - - history_file: "aerosol_month_cmip" - sourceGrid: "cubedsphere" - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False - - type: "land" - sources: - - history_file: "land_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "land_cmip" - sources: - - history_file: "land_month_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "tracer_level" - sources: - - history_file: "atmos_tracer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False diff --git a/fre/pp/tests/AM5_example/yaml_include/settings.yaml b/fre/list_/tests/yamls/yaml_include/settings.yaml similarity index 52% rename from fre/pp/tests/AM5_example/yaml_include/settings.yaml rename to fre/list_/tests/yamls/yaml_include/settings.yaml index 1057fc6d5..b790137a0 100644 --- a/fre/pp/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/list_/tests/yamls/yaml_include/settings.yaml @@ -1,9 +1,9 @@ #c96_amip_directories: directories: - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - refined_history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history_dir_refined] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + refined_history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history_dir_refined" + pp_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "/nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" ptmp_dir: "/xtmp/$USER/ptmp" #c96_amip_postprocess: @@ -11,12 +11,12 @@ postprocess: settings: history_segment: "P1Y" site: "ppan" - pp_start: *ANA_AMIP_START - pp_stop: *ANA_AMIP_END + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" pp_chunks: - - *PP_AMIP_CHUNK96 - pp_grid_spec: *GRID_SPEC96 - switches: &shared_switches + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: clean_work: True do_atmos_plevel_masking: True preanalysis: diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 0b1c77286..7844eec50 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -11,7 +11,8 @@ from jsonschema import validate, SchemaError, ValidationError import metomi.rose.config -import fre.yamltools.combine_yamls_script as cy +import fre.yamltools.combine_yamls_script_new as cy +import fre.list_.list_yamls_script as ly fre_logger = logging.getLogger(__name__) @@ -265,10 +266,15 @@ def yaml_info(yamlfile: str = None, experiment: str = None, platform: str = None # Combine model, experiment, and analysis yamls cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") outfile = os.path.join(cylc_dir, f"{e}.yaml") - full_yamldict = cy.consolidate_yamls(yamlfile = yml, - experiment = e, platform = p, target = t, - use = "pp", - output = outfile) + + yamls = ly.list_yamls_subtool(yamlfile = yml, + experiment = e, + application= "postprocess") + full_yamldict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = e, + platform = p, + target = t, + output = outfile) # Validate yaml validate_yaml(full_yamldict) diff --git a/fre/pp/tests/AM5_example/am5.yaml b/fre/pp/tests/AM5_example/am5.yaml deleted file mode 100644 index 24d365b25..000000000 --- a/fre/pp/tests/AM5_example/am5.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "19790101T0000Z" - - &EXP_AMIP_END "20200101T0000Z" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -# compile information -build: - compileYaml: "compile.yaml" - platformYaml: "yaml_include/platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - settings: "yaml_include/settings.yaml" - pp: - - "yaml_include/pp.c96_amip.yaml" - - "yaml_include/pp-test.c96_amip.yaml" - - name: "c96L65_am5f7b12r1_pdclim1850F" - pp: - - "yaml_include/pp.c96_clim.yaml" diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index 870e727b2..20269ffef 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -16,17 +16,17 @@ import metomi.rose.config from fre.pp import configure_script_yaml as csy -from fre.yamltools import combine_yamls_script as cy - +from fre.yamltools import combine_yamls_script_new as cy +import fre.list_.list_yamls_script as ly # Set what would be click options -EXPERIMENT = "c96L65_am5f7b12r1_amip" +EXPERIMENT = "experiment_1" PLATFORM = "gfdl.ncrc5-intel22-classic" TARGET = "prod-openmp" # Set example yaml paths, input directory TEST_DIR = Path("fre/pp/tests") -TEST_YAML = Path("AM5_example/am5.yaml") +TEST_YAML = Path("yamls/model.yaml") def test_combinedyaml_exists(): """ @@ -34,15 +34,14 @@ def test_combinedyaml_exists(): """ assert Path(f"{TEST_DIR}/{TEST_YAML}").exists() -def test_configure_script(): +def test_configure_script(monkeypatch): """ Tests success of configure yaml script Creates rose-suite, regrid rose-app, remap rose-app TO-DO: will break this up for better tests """ # Set home for ~/cylc-src location in script - old_home = os.environ["HOME"] - os.environ["HOME"] = str(Path(f"{TEST_DIR}/configure_yaml_out")) + monkeypatch.setenv("HOME", str(Path(f"{TEST_DIR}/configure_yaml_out"))) # Set output directory OUT_DIR = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}") @@ -54,8 +53,6 @@ def test_configure_script(): # Invoke configure_yaml_script.py csy.yaml_info(model_yaml, EXPERIMENT, PLATFORM, TARGET) - os.environ["HOME"] = old_home - # Check for configuration creation and final combined yaml assert all([ Path(f"{OUT_DIR}/{EXPERIMENT}.yaml").exists(), Path(f"{OUT_DIR}/rose-suite.conf").exists()]) @@ -64,12 +61,15 @@ def test_validate(): """ Test the success of validation. """ - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", - experiment = EXPERIMENT, - platform = PLATFORM, - target = TARGET, - use = "pp", - output = None) + yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", + experiment = EXPERIMENT, + application= "postprocess") + yml_dict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = EXPERIMENT, + platform = PLATFORM, + target = TARGET, + output = None) + try: csy.validate_yaml(yml_dict) except: @@ -79,12 +79,15 @@ def test_validate_fail(): """ Test that validation fails when given the wrong yaml dictionary. """ - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", - experiment = EXPERIMENT, - platform = PLATFORM, - target = TARGET, - use = "pp", - output = f"{Path(__file__).parent}/csy_out.yaml") + yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", + experiment = EXPERIMENT, + application= "postprocess") + yml_dict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = EXPERIMENT, + platform = PLATFORM, + target = TARGET, + output = f"{Path(__file__).parent}/csy_out.yaml") + # Missing history_dir, ptmp_dir, and postprocess wrong_yml_dict = { diff --git a/fre/pp/tests/yamls/model.yaml b/fre/pp/tests/yamls/model.yaml new file mode 100644 index 000000000..4ea595842 --- /dev/null +++ b/fre/pp/tests/yamls/model.yaml @@ -0,0 +1,56 @@ +# reusable variables +fre_properties: + AM5_VERSION: "am5f7b12r1" + FRE_STEM: "am5/{{ fre_properties.AM5_VERSION }}" + + # amip + EXP_AMIP_START: "19790101T0000Z" + EXP_AMIP_END: "20200101T0000Z" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" + + PP_AMIP_CHUNK96: "P1Y" + PP_AMIP_CHUNK384: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" + + # climo + EXP_CLIMO_START96: "0001" + EXP_CLIMO_END96: "0011" + ANA_CLIMO_START96: "0002" + ANA_CLIMO_END96: "0011" + + EXP_CLIMO_START384: "0001" + EXP_CLIMO_END384: "0006" + ANA_CLIMO_START384: "0002" + ANA_CLIMO_END384: "0006" + + # coupled + PP_CPLD_CHUNK_A: "P5Y" + PP_CPLD_CHUNK_B: "P20Y" + + # grids + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + RELEASE: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: +# c96L65_am5f7b12r1_amip: + experiment_1: + settings: "yaml_include/settings.yaml" + postprocess: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" +# c96L65_am5f7b12r1_pdclim1850F: + experiment_2: + postprocess: + - "yaml_include/pp.c96_clim.yaml" diff --git a/fre/pp/tests/yamls/yaml_include/platforms.yaml b/fre/pp/tests/yamls/yaml_include/platforms.yaml new file mode 100644 index 000000000..01689dab7 --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/platforms.yaml @@ -0,0 +1,12 @@ +platforms: + - name: hpcme.intel25 + compiler: intel + RUNenv: "" + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3" + mkTemplate: "/apps/mkmf/templates/hpcme-intel2025.03-oneapi.mk" + container2step: True + container2base: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3rte" diff --git a/fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml similarity index 73% rename from fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml rename to fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml index 76f078523..25412100d 100644 --- a/fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml +++ b/fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml @@ -1,33 +1,32 @@ # local reusable variable overrides fre_properties: - - &custom_interp "200,200" + custom_interp_test: "200,200" -#c96_amip_postprocess: postprocess: components: - - type: "atmos_cmip-TEST" + atmos_cmip-TEST: sources: - history_file: "atmos_month_cmip" - history_file: "atmos_8xdaily_cmip" - history_file: "atmos_daily_cmip" sourceGrid: "cubedsphere" - xyInterp: *custom_interp + xyInterp: "{{ fre_properties.custom_interp_test }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos-TEST" + atmos-TEST: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos_level_cmip-TEST" + atmos_level_cmip-TEST: sources: - history_file: "atmos_level_cmip" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False diff --git a/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..b5e1b7a8a --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,54 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "180,360" + +postprocess: + # main pp instructions + components: + atmos_cmip: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.custom_interp }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_level_cmip: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_level: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_month_aer: + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + atmos_diurnal: + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_scalar: + sources: + - history_file: "atmos_scalar" diff --git a/fre/pp/tests/yamls/yaml_include/settings.yaml b/fre/pp/tests/yamls/yaml_include/settings.yaml new file mode 100644 index 000000000..9ccd6a755 --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/settings.yaml @@ -0,0 +1,35 @@ +directories: + history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + refined_history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history_dir_refined" + pp_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "/nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" + ptmp_dir: "/xtmp/$USER/ptmp" + +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" + pp_chunks: + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: + clean_work: True + do_atmos_plevel_masking: True + preanalysis: + vitals: + script: "/path/to/vitals-script" + inputs: ['atmos_month', 'aerosol_month'] + do_preanalysis: True + refinediag: + ocean: + script: "/path/to/ocean-script" + inputs: ['ocean_monthly', 'ocean_bgc'] + outputs: ['ocean_refined'] + do_refinediag: True + aerosol: + script: "/path/to/aerosol-script" + inputs: ['aerosol_month'] + outputs: ['aerosol_refined'] + do_refinediag: True diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index 82410cc07..7ce5f13a9 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -37,12 +37,13 @@ class LetsGo(): :ivar str output: is the file path to a file that will incude the final combined, resolved YAML configuration file """ - def __init__(self, yamls, experiment, platform, target, output): + def __init__(self, yamls, experiment, platform, target, output, no_clean): self.y = yamls.split(",") self.e = str(experiment) self.p = str(platform) self.t = str(target) self.o = output + self.no_clean = no_clean def list_check_yamls(self, init_file): """ @@ -93,10 +94,13 @@ def use_uwtools(self, yaml_list): print("This will not be displayed") combined_yaml_dict = config.compose(configs=yaml_list, realize=True).as_dict() - # CLEAN SERIALIZED YAML ## - cleaned_yaml_dict = clean_yaml(combined_yaml_dict) - if not cleaned_yaml_dict: - raise ValueError("YAML configuration could not be cleaned (experiments)") + if self.no_clean: + final_yaml_dict = combined_yaml_dict + else: + # CLEAN SERIALIZED YAML ## + final_yaml_dict = clean_yaml(combined_yaml_dict) + if not final_yaml_dict: + raise ValueError("YAML configuration could not be cleaned (experiments)") ### SHOLD BE RESOLVED BUT THIS IS TO CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## ## uw config realize: resolve final yaml @@ -110,7 +114,7 @@ def use_uwtools(self, yaml_list): uwlogger = use_uwtools_logger() - config.realize(input_config = cleaned_yaml_dict, + config.realize(input_config = final_yaml_dict, values_needed = True, total = True) @@ -121,16 +125,16 @@ def use_uwtools(self, yaml_list): if self.o: out_path = Path.cwd()/self.o fre_logger.info("Writing resolved YAML file: %s", out_path) - output_yaml(cleaned_yaml_dict, out_path) + output_yaml(final_yaml_dict, out_path) # Is this helpful? (if config.realize fails, it will not be written to output file, # but error will hopefully come up; this is just more output if the user wants to # see what the combined yaml dictonary would look like - #fre_logger.debug(pformat(cleaned_yaml_dict)) + #fre_logger.debug(pformat(final_yaml_dict)) else: fre_logger.info("Resolved YAML saved as dictionary. To display dictionary, pass fre -v ...") - fre_logger.info(cleaned_yaml_dict) + fre_logger.info(final_yaml_dict) - return cleaned_yaml_dict + return final_yaml_dict # def validate(self, final_dict) # """ @@ -155,7 +159,7 @@ def use_uwtools(self, yaml_list): #with open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json") as j1, open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_pp.json") as j2, open("j3.json", 'w') as j3: # dump([load(j1), load(j2)], j3) -def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None) -> dict: +def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None, no_clean: Optional[bool]=False) -> dict: """ :param yamls: is the list of YAML configuration files to combine :type yamls: str @@ -168,10 +172,12 @@ def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:st :param output: is the file path to a file that will incude the final combined, resolved YAML configuration file :type output: str + :param no_clean: is a True/False value to determine whether or not to remove any keys/sections from the YAML. Default is False. + :type no_clean: boolean """ # fre_logger.info('checking fre_cli_version compatibility...') # check_fre_version(combined) - init_obj = LetsGo(yamls, experiment, platform, target, output) + init_obj = LetsGo(yamls, experiment, platform, target, output, no_clean) init_file = f"{Path.cwd()}/init.yaml" ymls = init_obj.list_check_yamls(init_file) diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index aab992354..853679ec8 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -50,7 +50,9 @@ def combine_yamls(yamlfile, help="Target name") @click.option("-o", "--output", type=str, help="Output") -def combine(yamls, experiment, platform, target, output): +@click.option("--no-clean", type=bool, + help="Do not remove anything from the combinedm resolved YAML file.") +def combine(yamls, experiment, platform, target, output, no_clean): """ - Combine the model yaml with the compile, platform, experiment, and analysis yamls @@ -66,7 +68,7 @@ def combine(yamls, experiment, platform, target, output): # stdout from last tool adds a newline for some reason yaml_paths = sys.stdin.read().strip() - combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output) + combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output, no_clean) #@yamltools_cli.command() #@click.option("-y", "--yaml", diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 16af8e6ed..fe349bc2e 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -155,9 +155,12 @@ def test_check_expected_platformyamlcontent(): # EXP_NAME = YAMLFILE.split(".")[0] # Combine model / experiment - yamls = ("fre/make/tests/null_example/null_model.yaml," - "fre/make/tests/null_example/compile.yaml," - "fre/make/tests/null_example/platforms.yaml") +# yamls = ("fre/make/tests/null_example/null_model.yaml," +# "fre/make/tests/null_example/compile.yaml," +# "fre/make/tests/null_example/platforms.yaml") + yamls = ("fre/yamltools/tests/yamls/model.yaml," + "fre/yamltools/tests/yamls/compile_yamls/compile.yaml," + "fre/yamltools/tests/yamls/compile_yamls/platforms.yaml") yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None) # compare combined yaml info with some information that's supposed to be parsed @@ -185,7 +188,6 @@ def test_check_expected_platformyamlcontent(): 'containerBase': 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25', 'mkTemplate': '/apps/mkmf/templates/hpcme-intel21.mk', 'volume': '/gpfs/f5'} - for key,value in yml_dict.items(): if key == "platforms": assert expected_platform_info_1 in value diff --git a/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml index 144261ce1..639f54c33 100644 --- a/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml @@ -1,23 +1,13 @@ platforms: - - name: ncrc5.intel - compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", - "source $MODULESHOME/init/sh", - "module load {{ fre_properties.INTEL }}/2022.2.1", - "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" - modelRoot: ${HOME}/fremake_canopy/test - name: ncrc5.intel23 compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - "module load {{ fre_properties.INTEL }}/2023.1.0", + "module load {{ fre_properties.INTEL }}/2023.2.0", "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" modelRoot: ${HOME}/fremake_canopy/test - name: hpcme.2023 compiler: intel @@ -26,3 +16,6 @@ platforms: container: True containerBuild: "podman" containerRun: "apptainer" + containerBase: 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25' + mkTemplate: '/apps/mkmf/templates/hpcme-intel21.mk' + volume: '/gpfs/f5'