diff --git a/fre/gfdl_msd_schemas b/fre/gfdl_msd_schemas index 33a5412b2..837e5e777 160000 --- a/fre/gfdl_msd_schemas +++ b/fre/gfdl_msd_schemas @@ -1 +1 @@ -Subproject commit 33a5412b2354adde4a7832eb81d4b8cbd965ffcb +Subproject commit 837e5e777319c6a497abf1012100688e0ceb480c diff --git a/fre/list_/frelist.py b/fre/list_/frelist.py index afad49a07..bc1f65756 100644 --- a/fre/list_/frelist.py +++ b/fre/list_/frelist.py @@ -44,9 +44,14 @@ def platforms(yamlfile): type=str, help="Experiment to be post-processed", required=True) -def pp_components(yamlfile, experiment): +@click.option("-a", + "--application", + type=str, + help="Post-processing application name.", + required=True) +def pp_components(yamlfile, experiment, application): """ - List components to be post-processed for a defined experiment""" - list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment) + list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment, application) @list_cli.command() @click.option("-y", diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index ecf205868..237c6cd75 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -5,15 +5,7 @@ """ import logging -import yaml - -from fre.yamltools.info_parsers import pp_info_parser as ppip - -## to-do: figure out validation -# Imports associated with commented block -# from pathlib import Path -# from fre.yamltools import helpers -# from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy fre_logger = logging.getLogger(__name__) @@ -29,34 +21,15 @@ def list_experiments_subtool(yamlfile: str): platform = None target = None - # Combine model - # Create pp yaml instance - yamldict = ppip.InitPPYaml(yamlfile, exp, platform, target) - yaml_str = yamldict.combine_model() - yaml_dict = yaml.load(yaml_str, Loader = yaml.Loader) + yaml_dict = cy.yamltools_combine_subtool(yamls = yamlfile, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = True) -## COULD HAVE been one way to validate but section we'd want to parse was -## cleaned in final/"combined" yaml information -## Currently not a way to validate model yaml information because we only -## have schemas for the final "combined" compile or pp information (both -## of which remove the "experiments" section I believe -# exp = yamlfile.split("/")[-1].split(".")[0] -# platform = "None" -# target = "None" -# -# # Combine model / experiment -# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, -# experiment = exp, -# platform = platform, -# target = target, -# use = "compile", -# output = None) -# -# # Validate combined yaml information -# frelist_dir = Path(__file__).resolve().parents[2] -# schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json" -# # from fre.yamltools -# helpers.validate_yaml(yml_dict, schema_path) +#### TO-DO: CURRENTLY NO YAML PIECE VALIDATION +### No way to validate just the model yaml # set logger level to INFO former_log_level = fre_logger.level @@ -64,9 +37,9 @@ def list_experiments_subtool(yamlfile: str): # log the experiment names, which should show up on screen for sure fre_logger.info("Experiments found:") - for i in yaml_dict.get("experiments"): - fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") + + for i in yaml_dict["experiments"].keys(): + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index 2ebc0a0e5..7d36d3703 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -6,7 +6,8 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) @@ -22,17 +23,33 @@ def list_platforms_subtool(yamlfile: str): former_log_level = fre_logger.level fre_logger.setLevel(logging.INFO) - exp = yamlfile.split("/")[-1].split(".")[0] + exp = None platform = None target = None - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "compile", - output = None) +# model_yf_path = Path(yamlfile).resolve().parent +# with open(yamlfile, 'r') as yf: +# yml = yaml.safe_load(yf) +# +# platform_yf_path = f"{model_yf_path}/{yml['build']['platformYaml']}" + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = None) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "compile", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) # Validate the yaml fre_pkg_dir = Path(__file__).resolve().parents[1] @@ -41,8 +58,7 @@ def list_platforms_subtool(yamlfile: str): helpers.validate_yaml(yml_dict, schema_path) fre_logger.info("Platforms available:") - for i in yml_dict.get("platforms"): + for i in yml_dict["platforms"]: fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_pp_components_script.py b/fre/list_/list_pp_components_script.py index 0be6128c7..5f475c5fb 100644 --- a/fre/list_/list_pp_components_script.py +++ b/fre/list_/list_pp_components_script.py @@ -19,12 +19,13 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) -def list_ppcomps_subtool(yamlfile: str, experiment: str): +def list_ppcomps_subtool(yamlfile: str, experiment: str, application: str): """ List_ppcomps_subtool lists the components to be post-processed. @@ -32,6 +33,8 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): :type yamlfile: str :param experiment: is the experiment name defined in the model.yaml :type experiment: str + :param application: + :type applicatin: str """ # set logger level to INFO former_log_level = fre_logger.level @@ -40,14 +43,26 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): exp = experiment platform = None target = None + app = application + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = app) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "pp", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "pp", - output = None) # Validate combined yaml information frelist_dir = Path(__file__).resolve().parents[2] @@ -55,15 +70,15 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): # from fre.yamltools helpers.validate_yaml(yml_dict, schema_path) + comp_info = yml_dict["postprocess"]["components"] # log the experiment names, which should show up on screen for sure fre_logger.info("Components to be post-processed:") - for i in yml_dict["postprocess"]["components"]: - if "postprocess_on" in i: - if i.get("postprocess_on") is True: - fre_logger.info(' - %s', i.get("type")) + for i in comp_info.keys(): + if "postprocess_on" in comp_info[i]: + if comp_info[i].get("postprocess_on") is True: + fre_logger.info(' - %s', i) else: - fre_logger.info(' - %s', i.get("type")) - fre_logger.info("\n") + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_yamls_script.py b/fre/list_/list_yamls_script.py index 1820fe3d0..ed2078572 100644 --- a/fre/list_/list_yamls_script.py +++ b/fre/list_/list_yamls_script.py @@ -11,7 +11,9 @@ """ import logging from pathlib import Path +import sys import yaml +import click fre_logger = logging.getLogger(__name__) @@ -25,6 +27,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): :type experiment: str :param application: is the application name :type application: str + :return: is a comma separated string of yaml files (absolute paths) + :rtype: str :raise ValueError: if the experiment, application passed does not exist and if yaml files do not exist @@ -40,8 +44,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): with open(yamlfile, 'r', encoding="utf-8") as yf: yaml_dict = yaml.load(yf, Loader = yaml.Loader) - compile_data = yaml_dict["build"].get("compileYaml") platform_data = yaml_dict["build"].get("platformYaml") + compile_data = yaml_dict["build"].get("compileYaml") exp_data = yaml_dict["experiments"].get(experiment) yamls = [model_yaml] @@ -68,7 +72,7 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): else: yamls.append(exp_data[a]) else: - yamls.extend([compile_data, platform_data]) + yamls.extend([platform_data, compile_data]) for value in exp_data.values(): if isinstance(value, list): yamls.extend(value) @@ -76,7 +80,7 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): yamls.append(value) else: fre_logger.info("No experiment name passed. Will only provide YAMLs related to compilation.") - yamls.extend([compile_data, platform_data]) + yamls.extend([platform_data, compile_data]) yamls_full_path = "" # Add full path for yaml configurations @@ -94,13 +98,13 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): fre_logger.info(" - %s", y) ### Might add this in when fre yamltools combine-yamls is refactored -# fre_logger.info("") -# fre_logger.info('If combining these yamls, there are 2 options:') -# fre_logger.info(' 1. Pipe this tool to "fre yamltools combine"') -# fre_logger.info(' 2. Copy and paste this string (including quotes) -# as the -y option in "fre yamltools combine -y :') -# fre_logger.info(' "%s"', yamls_full_path) -# fre_logger.info("") + fre_logger.info("") + fre_logger.info('If combining these yamls, there are 2 options:') + fre_logger.info(' 1. Pipe this tool to "fre yamltools combine"') + fre_logger.info(' 2. Copy and paste this string (including quotes) ' + 'as the -y option in "fre yamltools combine -y :') + fre_logger.info(' "%s"', yamls_full_path) + fre_logger.info("") fre_logger.setLevel(former_log_level) # Check if the paths exist; give warning @@ -113,4 +117,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): if "True" in fail: raise ValueError(" *** PROVIDE THE MISSING YAML CONFIGURATIONS ***") + # if not piped to fre yamltools combine, no need to show all the yamls + if not sys.stdout.isatty(): + click.echo(yamls_full_path) + return yamls_full_path diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index f78bf1f93..2b9c96f2d 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -4,40 +4,36 @@ from pathlib import Path import pytest -import yaml from fre.list_ import list_experiments_script from fre.yamltools import helpers - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") -YAMLFILE = "null_model.yaml" +TEST_DIR = Path("fre/list_/tests/yamls") +YAMLFILE = "model.yaml" EXP_NAME = "None" # yaml file checks def test_modelyaml_exists(): ''' Test that model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() # Test whole tool def test_exp_list(caplog): ''' Test fre list exps subtool ''' - list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = [ 'Experiments found:', - ' - null_model_full', - ' - null_model_0', - ' - null_model_1', - ' - null_model_2' ] + ' - experiment1', + ' - experiment2' ] for i in check_out: assert i in caplog.text # make sure the level is INFO for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation @pytest.mark.skip( @@ -48,10 +44,10 @@ def test_exp_list(caplog): ) def test_yamlvalidate(): ''' Test yaml is being validated ''' - yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yamlfilepath = Path(f"{TEST_DIR}/{YAMLFILE}") # Combine model / experiment - yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # Validate and capture output assert helpers.validate_yaml(yml_dict, VAL_SCHEMA) diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index 8dd7eba56..16e6ecd2f 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -4,63 +4,57 @@ from pathlib import Path import pytest -import yaml - from fre.list_ import list_platforms_script -from fre.yamltools import combine_yamls_script as cy - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") +TEST_DIR = Path("fre/list_/tests/yamls") PLATFORM = None TARGET = None -YAMLFILE = "null_model.yaml" -EXP_NAME = YAMLFILE.split(".")[0] +YAMLFILE = "model.yaml" +EXP_NAME = YAMLFILE.split(".", maxsplit=1)[0] VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_make.json") # Bad yaml example -BADYAMLFILE_PATH = f"{TEST_DIR}/{NM_EXAMPLE}/wrong_model/wrong_null_model.yaml" +BADYAMLFILE_PATH = f"{TEST_DIR}/wrong_null_model.yaml" # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() def test_compileyaml_exists(): ''' Test compile yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() + assert Path(f"{TEST_DIR}/compile.yaml").exists() def test_platformyaml_exists(): ''' Test platforms yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() + assert Path(f"{TEST_DIR}/platforms.yaml").exists() # Test whole tool def test_platforms_list_correct(caplog): ''' Test fre list platforms subtool ''' - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = ["Platforms available:", " - ncrc5.intel23", - " - hpcme.2023", + " - hpcmini.2025", " - ci.gnu", - " - con.twostep" ] + " - hpcme.intel25" ] for i in check_out: assert i in caplog.text # make sure level is INFO for record in caplog.records: - record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation def test_yamlvalidate(caplog): ''' Test yaml is being validated and is actually valid''' - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - # Combine model / experiment - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -69,7 +63,8 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" def test_not_valid_yaml(): ''' Test the correct output matches the ValueError raised when yaml is invalid ''' diff --git a/fre/list_/tests/test_list_pp_components_script.py b/fre/list_/tests/test_list_pp_components_script.py index f273d955c..fb0b30388 100644 --- a/fre/list_/tests/test_list_pp_components_script.py +++ b/fre/list_/tests/test_list_pp_components_script.py @@ -6,12 +6,13 @@ # SET-UP -TEST_DIR = Path("fre/pp/tests") -AM5_EXAMPLE = Path("AM5_example") -MODEL_YAMLFILE = "am5.yaml" +#TEST_DIR = Path("fre/pp/tests") +TEST_DIR = Path("fre/list_/tests/yamls") +#AM5_EXAMPLE = Path("AM5_example") +MODEL_YAMLFILE = "model.yaml" SETTINGS_YAMLFILE = "yaml_include/settings.yaml" -PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml", "yaml_include/settings.yaml"] -EXP_NAME = "c96L65_am5f7b12r1_amip" +PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml"] #, "yaml_include/settings.yaml"] +EXP_NAME = "experiment1" VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_pp.json") PLATFORM = "FOO" TARGET = "BAR" @@ -19,44 +20,45 @@ # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{MODEL_YAMLFILE}").exists() def test_settingsyaml_exist(): ''' Test settings yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{SETTINGS_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{SETTINGS_YAMLFILE}").exists() def test_ppyamls_exist(): ''' Test post-processing yamls exist ''' for pp_yaml in PP_YAMLFILES: - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{pp_yaml}").exists() + assert Path(f"{TEST_DIR}/{pp_yaml}").exists() # Test whole tool def test_pp_comp_list(caplog): ''' Test fre list pp-components subtool ''' - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") # check the logging output check_out = [ 'Components to be post-processed:', ' - atmos_cmip', ' - atmos', - ' - atmos_level_cmip', - ' - atmos_level', - ' - atmos_month_aer', ' - atmos_diurnal', - ' - atmos_scalar'] - + ' - atmos_scalar', + ' - atmos_cmip-TEST' ] for i in check_out: assert i in caplog.text - # make sure the level is WARNING for version mismatch and INFO otherwise for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] # Test validation def test_yamlvalidate(caplog): ''' Test yaml is being validated ''' # Combine model / experiment - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -65,5 +67,5 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] - + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] diff --git a/fre/list_/tests/test_list_yamls_script.py b/fre/list_/tests/test_list_yamls_script.py index e89ca0cbe..d909d355e 100644 --- a/fre/list_/tests/test_list_yamls_script.py +++ b/fre/list_/tests/test_list_yamls_script.py @@ -67,27 +67,24 @@ def test_list_yamls_runtime_only(self, model_yaml_path): def test_list_yamls_postprocess_only(self, model_yaml_path): """Test postprocess_only flag (without analysis) returns model YAML plus postprocessing YAMLs""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="experiment1", - application="postprocess" - ) + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="experiment1", + application="postprocess" + ) - yaml_names = [Path(y).name for y in result.split(",")] + yaml_names = [Path(y).name for y in result.split(",")] - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain settings and postprocessing YAMLs - assert "settings.yaml" in yaml_names - assert "pp.c96_amip.yaml" in yaml_names - assert "pp-test.c96_amip.yaml" in yaml_names - - # Should NOT contain compile, platform, run, or analysis YAMLs - assert "compile.yaml" not in yaml_names - assert "platforms.yaml" not in yaml_names - assert "run1.yaml" not in yaml_names - assert "analysis1.yaml" not in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain settings and postprocessing YAMLs + assert "settings.yaml" in yaml_names + assert "pp.c96_amip.yaml" in yaml_names + assert "pp-test.c96_amip.yaml" in yaml_names + # Should NOT contain compile, platform, run, or analysis YAMLs + assert "compile.yaml" not in yaml_names + assert "run1.yaml" not in yaml_names + assert "analysis1.yaml" not in yaml_names def test_list_yamls_analysis_only(self, model_yaml_path): """Test analysis_only flag (without postprocess) returns model YAML plus analysis YAMLs""" @@ -138,23 +135,23 @@ def test_list_yamls_postprocess_and_analysis(self, model_yaml_path): def test_list_yamls_no_experiment(self, model_yaml_path): """Test with no experiment name provided (default compile behavior)""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="", - application=None - ) +# with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="", + application=None + ) - yaml_names = [Path(y).name for y in result.split(",")] + yaml_names = [Path(y).name for y in result.split(",")] - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain compile and platform YAMLs - assert "compile.yaml" in yaml_names - assert "platforms.yaml" in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain compile and platform YAMLs + assert "compile.yaml" in yaml_names + assert "platforms.yaml" in yaml_names - # Should NOT contain experiment-specific YAMLs - assert "run1.yaml" not in yaml_names + # Should NOT contain experiment-specific YAMLs + assert "run1.yaml" not in yaml_names def test_list_yamls_returns_full_paths(self, model_yaml_path): """Test that returned YAMLs have full paths""" diff --git a/fre/list_/tests/yamls/compile.yaml b/fre/list_/tests/yamls/compile.yaml new file mode 100644 index 000000000..51a72b73c --- /dev/null +++ b/fre/list_/tests/yamls/compile.yaml @@ -0,0 +1,49 @@ +compile: + experiment: "null_model_full" + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + additionalInstructions: "echo \"Testing additionalInstructions\"" + cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" + otherFlags: "-fallow-argument-mismatch" # only needed for gcc + branch: "2025.04" + - component: "atmos_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/atmos_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "land_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/land_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_param" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ocean_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_null.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_null" + requires: ["FMS", "ice_param", "ocean_null"] + repo: "https://github.com/NOAA-GFDL/ice_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "coupler" + requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML" + paths: [ "coupler/full", + "coupler/shared" ] + diff --git a/fre/list_/tests/yamls/empty_platforms.yaml b/fre/list_/tests/yamls/empty_platforms.yaml new file mode 100644 index 000000000..9bceccc06 --- /dev/null +++ b/fre/list_/tests/yamls/empty_platforms.yaml @@ -0,0 +1 @@ +platforms: diff --git a/fre/list_/tests/yamls/model.yaml b/fre/list_/tests/yamls/model.yaml index 43cc46edb..035dbdcf3 100644 --- a/fre/list_/tests/yamls/model.yaml +++ b/fre/list_/tests/yamls/model.yaml @@ -1,22 +1,25 @@ # reusable variables fre_properties: - VERSION: "foo" - FRE_STEM: "test_list/{{ fre_properties.VERSION }}" + VERSION: "foo" + FRE_STEM: "test_list/{{ fre_properties.VERSION }}" # amip - EXP_AMIP_START: "19790101T0000Z" - EXP_AMIP_END: "20200101T0000Z" - ANA_AMIP_START: "1980" - ANA_AMIP_END: "2020" + EXP_AMIP_START: "19790101T0000Z" + EXP_AMIP_END: "20200101T0000Z" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" - PP_AMIP_CHUNK96: "P1Y" - PP_XYINTERP96: "180,288" - PP_XYINTERP384: "720,1152" + PP_AMIP_CHUNK96: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" # compile information - INTEL: "intel-classic" - FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + branch: "test" + + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" # compile information build: @@ -33,3 +36,4 @@ experiments: - "yaml_include/pp-test.c96_amip.yaml" analysis: - "analysis1.yaml" + experiment2: diff --git a/fre/list_/tests/yamls/platforms.yaml b/fre/list_/tests/yamls/platforms.yaml new file mode 100644 index 000000000..00d1c74fc --- /dev/null +++ b/fre/list_/tests/yamls/platforms.yaml @@ -0,0 +1,34 @@ +platforms: + - name: ncrc5.intel23 + compiler: intel + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", + "source $MODULESHOME/init/sh", + "module load {{ fre_properties.INTEL }}/2023.2.0", + "module load fre/bronx-23", + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" + modelRoot: ${HOME}/fremake_canopy/test + - name: ci.gnu + compiler: gnu + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test + - name: hpcmini.2025 + compiler: gnu + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "ghcr.io/rem1776/hpc-mini:alpine3gcc14.2" + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + - name: hpcme.intel25 + compiler: intel + RUNenv: "" + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3" + mkTemplate: "/apps/mkmf/templates/hpcme-intel2025.03-oneapi.mk" + container2step: True + container2base: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3rte" diff --git a/fre/list_/tests/yamls/wrong_null_model.yaml b/fre/list_/tests/yamls/wrong_null_model.yaml new file mode 100644 index 000000000..ccec12827 --- /dev/null +++ b/fre/list_/tests/yamls/wrong_null_model.yaml @@ -0,0 +1,15 @@ +# reusable variables +fre_properties: + VERSION: "full" + FRE_STEM: "null_model/{{ fre_properties.VERSION }}" + # compile information + branch: "main" + INTEL: "intel-classic" + FMSincludes: "-IFMS/include" + +build: + compileYaml: "compile.yaml" + platformYaml: "empty_platforms.yaml" + +experiments: + experiment1: diff --git a/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..b54f0fc77 --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "200,200" + +#c96_amip_postprocess: +postprocess: + components: + atmos_cmip-TEST: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.custom_interp}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos-TEST: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level_cmip-TEST: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml similarity index 50% rename from fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml rename to fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml index 3859217bd..a06f8162e 100644 --- a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml +++ b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml @@ -1,87 +1,59 @@ # local reusable variable overrides fre_properties: - - &custom_interp "180,360" + custom_interp: "180,360" #c96_amip_postprocess: postprocess: # main pp instructions components: - - type: "atmos_cmip" + atmos_cmip: sources: - history_file: "atmos_month_cmip" - history_file: "atmos_8xdaily_cmip" - history_file: "atmos_daily_cmip" sourceGrid: "cubedsphere" - xyInterp: *custom_interp + xyInterp: "{{fre_properties.custom_interp}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos" + atmos: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_level_cmip" + postprocess_on: True + atmos_level_cmip: sources: - history_file: "atmos_level_cmip" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_level" + postprocess_on: False + atmos_level: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_month_aer" + postprocess_on: False + atmos_month_aer: sources: - history_file: "atmos_month_aer" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order1" inputRealm: 'atmos' - - type: "atmos_diurnal" + postprocess_on: False + atmos_diurnal: sources: - history_file: "atmos_diurnal" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{fre_properties.PP_XYINTERP96}}" interpMethod: "conserve_order2" inputRealm: 'atmos' - - type: "atmos_scalar" + atmos_scalar: sources: - history_file: "atmos_scalar" - - type: "aerosol_cmip" - xyInterp: *PP_XYINTERP96 - sources: - - history_file: "aerosol_month_cmip" - sourceGrid: "cubedsphere" - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False - - type: "land" - sources: - - history_file: "land_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "land_cmip" - sources: - - history_file: "land_month_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "tracer_level" - sources: - - history_file: "atmos_tracer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False diff --git a/fre/pp/tests/AM5_example/yaml_include/settings.yaml b/fre/list_/tests/yamls/yaml_include/settings.yaml similarity index 52% rename from fre/pp/tests/AM5_example/yaml_include/settings.yaml rename to fre/list_/tests/yamls/yaml_include/settings.yaml index 1057fc6d5..b790137a0 100644 --- a/fre/pp/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/list_/tests/yamls/yaml_include/settings.yaml @@ -1,9 +1,9 @@ #c96_amip_directories: directories: - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - refined_history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history_dir_refined] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + refined_history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history_dir_refined" + pp_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "/nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" ptmp_dir: "/xtmp/$USER/ptmp" #c96_amip_postprocess: @@ -11,12 +11,12 @@ postprocess: settings: history_segment: "P1Y" site: "ppan" - pp_start: *ANA_AMIP_START - pp_stop: *ANA_AMIP_END + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" pp_chunks: - - *PP_AMIP_CHUNK96 - pp_grid_spec: *GRID_SPEC96 - switches: &shared_switches + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: clean_work: True do_atmos_plevel_masking: True preanalysis: diff --git a/fre/make/tests/null_example/null_model.yaml b/fre/make/tests/null_example/null_model.yaml index a5a06d523..2834341c6 100644 --- a/fre/make/tests/null_example/null_model.yaml +++ b/fre/make/tests/null_example/null_model.yaml @@ -1,23 +1,23 @@ # reusable variables fre_properties: - - &VERSION "full" - - &FRE_STEM !join [null_model/, *VERSION] + VERSION: &VERSION "full" + FRE_STEM: &FRE_STEM !join [null_model/, *VERSION] # compile information - - &branch "main" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/include" - - &VOLUME "/gpfs/f5" + BRANCH: &branch "main" + INTEL: &INTEL "intel-classic" + FMSincludes: &FMSincludes "-IFMS/include" + VOLUME: &VOLUME "/gpfs/f5" build: compileYaml: "compile.yaml" platformYaml: "platforms.yaml" experiments: - - name: "null_model_full" + null_model_full: pp: - - name: "null_model_0" + null_model_0: pp: - - name: "null_model_1" + null_model_1: pp: - - name: "null_model_2" + null_model_2: pp: diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 29d675e82..f8aa61476 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -21,7 +21,8 @@ from jsonschema import validate, SchemaError, ValidationError import metomi.rose.config -import fre.yamltools.combine_yamls_script as cy +import fre.yamltools.combine_yamls_script_new as cy +import fre.list_.list_yamls_script as ly fre_logger = logging.getLogger(__name__) @@ -276,10 +277,15 @@ def yaml_info(yamlfile: str = None, experiment: str = None, platform: str = None # Combine input YAMLs and save consolidated output to cylc-src cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") outfile = os.path.join(cylc_dir, f"{e}.yaml") - full_yamldict = cy.consolidate_yamls(yamlfile = yml, - experiment = e, platform = p, target = t, - use = "pp", - output = outfile) + + yamls = ly.list_yamls_subtool(yamlfile = yml, + experiment = e, + application= "postprocess") + full_yamldict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = e, + platform = p, + target = t, + output = outfile) # Validate combined YAML dictionary against schema validate_yaml(full_yamldict) diff --git a/fre/pp/tests/AM5_example/am5.yaml b/fre/pp/tests/AM5_example/am5.yaml deleted file mode 100644 index 24d365b25..000000000 --- a/fre/pp/tests/AM5_example/am5.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "19790101T0000Z" - - &EXP_AMIP_END "20200101T0000Z" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -# compile information -build: - compileYaml: "compile.yaml" - platformYaml: "yaml_include/platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - settings: "yaml_include/settings.yaml" - pp: - - "yaml_include/pp.c96_amip.yaml" - - "yaml_include/pp-test.c96_amip.yaml" - - name: "c96L65_am5f7b12r1_pdclim1850F" - pp: - - "yaml_include/pp.c96_clim.yaml" diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index 870e727b2..20269ffef 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -16,17 +16,17 @@ import metomi.rose.config from fre.pp import configure_script_yaml as csy -from fre.yamltools import combine_yamls_script as cy - +from fre.yamltools import combine_yamls_script_new as cy +import fre.list_.list_yamls_script as ly # Set what would be click options -EXPERIMENT = "c96L65_am5f7b12r1_amip" +EXPERIMENT = "experiment_1" PLATFORM = "gfdl.ncrc5-intel22-classic" TARGET = "prod-openmp" # Set example yaml paths, input directory TEST_DIR = Path("fre/pp/tests") -TEST_YAML = Path("AM5_example/am5.yaml") +TEST_YAML = Path("yamls/model.yaml") def test_combinedyaml_exists(): """ @@ -34,15 +34,14 @@ def test_combinedyaml_exists(): """ assert Path(f"{TEST_DIR}/{TEST_YAML}").exists() -def test_configure_script(): +def test_configure_script(monkeypatch): """ Tests success of configure yaml script Creates rose-suite, regrid rose-app, remap rose-app TO-DO: will break this up for better tests """ # Set home for ~/cylc-src location in script - old_home = os.environ["HOME"] - os.environ["HOME"] = str(Path(f"{TEST_DIR}/configure_yaml_out")) + monkeypatch.setenv("HOME", str(Path(f"{TEST_DIR}/configure_yaml_out"))) # Set output directory OUT_DIR = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}") @@ -54,8 +53,6 @@ def test_configure_script(): # Invoke configure_yaml_script.py csy.yaml_info(model_yaml, EXPERIMENT, PLATFORM, TARGET) - os.environ["HOME"] = old_home - # Check for configuration creation and final combined yaml assert all([ Path(f"{OUT_DIR}/{EXPERIMENT}.yaml").exists(), Path(f"{OUT_DIR}/rose-suite.conf").exists()]) @@ -64,12 +61,15 @@ def test_validate(): """ Test the success of validation. """ - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", - experiment = EXPERIMENT, - platform = PLATFORM, - target = TARGET, - use = "pp", - output = None) + yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", + experiment = EXPERIMENT, + application= "postprocess") + yml_dict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = EXPERIMENT, + platform = PLATFORM, + target = TARGET, + output = None) + try: csy.validate_yaml(yml_dict) except: @@ -79,12 +79,15 @@ def test_validate_fail(): """ Test that validation fails when given the wrong yaml dictionary. """ - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", - experiment = EXPERIMENT, - platform = PLATFORM, - target = TARGET, - use = "pp", - output = f"{Path(__file__).parent}/csy_out.yaml") + yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", + experiment = EXPERIMENT, + application= "postprocess") + yml_dict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = EXPERIMENT, + platform = PLATFORM, + target = TARGET, + output = f"{Path(__file__).parent}/csy_out.yaml") + # Missing history_dir, ptmp_dir, and postprocess wrong_yml_dict = { diff --git a/fre/pp/tests/yamls/model.yaml b/fre/pp/tests/yamls/model.yaml new file mode 100644 index 000000000..4ea595842 --- /dev/null +++ b/fre/pp/tests/yamls/model.yaml @@ -0,0 +1,56 @@ +# reusable variables +fre_properties: + AM5_VERSION: "am5f7b12r1" + FRE_STEM: "am5/{{ fre_properties.AM5_VERSION }}" + + # amip + EXP_AMIP_START: "19790101T0000Z" + EXP_AMIP_END: "20200101T0000Z" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" + + PP_AMIP_CHUNK96: "P1Y" + PP_AMIP_CHUNK384: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" + + # climo + EXP_CLIMO_START96: "0001" + EXP_CLIMO_END96: "0011" + ANA_CLIMO_START96: "0002" + ANA_CLIMO_END96: "0011" + + EXP_CLIMO_START384: "0001" + EXP_CLIMO_END384: "0006" + ANA_CLIMO_START384: "0002" + ANA_CLIMO_END384: "0006" + + # coupled + PP_CPLD_CHUNK_A: "P5Y" + PP_CPLD_CHUNK_B: "P20Y" + + # grids + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + RELEASE: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: +# c96L65_am5f7b12r1_amip: + experiment_1: + settings: "yaml_include/settings.yaml" + postprocess: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" +# c96L65_am5f7b12r1_pdclim1850F: + experiment_2: + postprocess: + - "yaml_include/pp.c96_clim.yaml" diff --git a/fre/pp/tests/yamls/yaml_include/platforms.yaml b/fre/pp/tests/yamls/yaml_include/platforms.yaml new file mode 100644 index 000000000..01689dab7 --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/platforms.yaml @@ -0,0 +1,12 @@ +platforms: + - name: hpcme.intel25 + compiler: intel + RUNenv: "" + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3" + mkTemplate: "/apps/mkmf/templates/hpcme-intel2025.03-oneapi.mk" + container2step: True + container2base: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3rte" diff --git a/fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml similarity index 73% rename from fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml rename to fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml index 76f078523..25412100d 100644 --- a/fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml +++ b/fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml @@ -1,33 +1,32 @@ # local reusable variable overrides fre_properties: - - &custom_interp "200,200" + custom_interp_test: "200,200" -#c96_amip_postprocess: postprocess: components: - - type: "atmos_cmip-TEST" + atmos_cmip-TEST: sources: - history_file: "atmos_month_cmip" - history_file: "atmos_8xdaily_cmip" - history_file: "atmos_daily_cmip" sourceGrid: "cubedsphere" - xyInterp: *custom_interp + xyInterp: "{{ fre_properties.custom_interp_test }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos-TEST" + atmos-TEST: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos_level_cmip-TEST" + atmos_level_cmip-TEST: sources: - history_file: "atmos_level_cmip" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False diff --git a/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..b5e1b7a8a --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,54 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "180,360" + +postprocess: + # main pp instructions + components: + atmos_cmip: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.custom_interp }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_level_cmip: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_level: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_month_aer: + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + atmos_diurnal: + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_scalar: + sources: + - history_file: "atmos_scalar" diff --git a/fre/pp/tests/yamls/yaml_include/settings.yaml b/fre/pp/tests/yamls/yaml_include/settings.yaml new file mode 100644 index 000000000..9ccd6a755 --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/settings.yaml @@ -0,0 +1,35 @@ +directories: + history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + refined_history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history_dir_refined" + pp_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "/nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" + ptmp_dir: "/xtmp/$USER/ptmp" + +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" + pp_chunks: + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: + clean_work: True + do_atmos_plevel_masking: True + preanalysis: + vitals: + script: "/path/to/vitals-script" + inputs: ['atmos_month', 'aerosol_month'] + do_preanalysis: True + refinediag: + ocean: + script: "/path/to/ocean-script" + inputs: ['ocean_monthly', 'ocean_bgc'] + outputs: ['ocean_refined'] + do_refinediag: True + aerosol: + script: "/path/to/aerosol-script" + inputs: ['aerosol_month'] + outputs: ['aerosol_refined'] + do_refinediag: True diff --git a/fre/tests/test_fre_list_cli.py b/fre/tests/test_fre_list_cli.py index 7315e507d..f8645b387 100644 --- a/fre/tests/test_fre_list_cli.py +++ b/fre/tests/test_fre_list_cli.py @@ -62,3 +62,43 @@ def test_cli_fre_list_exps_opt_dne(): ''' fre list platforms optionDNE ''' result = runner.invoke(fre.fre, args=["list", "platforms", "optionDNE"]) assert result.exit_code == 2 + +## fre list pp-components +def test_cli_fre_list_ppcomponents(): + ''' fre list pp-components ''' + result = runner.invoke(fre.fre, args=["list", "pp-components"]) + assert result.exit_code == 2 + +def test_cli_fre_list_ppcomponents_help(): + ''' fre list pp-components --help ''' + result = runner.invoke(fre.fre, args=["list", "pp-components", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_list_ppcomponents_opt_dne(): + ''' fre list pp-components optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "pp-components", "optionDNE"]) + assert result.exit_code == 2 + +def test_cli_fre_list_ppcomponents_app_opt(caplog): + ''' fre list pp-components -a postprocess optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "pp-components", "-y", "fre/list_/tests/yamls/model.yaml", + "-e", "experiment1", "-a", "postprocess"]) + assert result.exit_code == 0 + for c in ["atmos_cmip", "atmos", "atmos_diurnal", "atmos_scalar", "atmos_cmip-TEST"]: + assert c in caplog.text + +## fre list yamls +def test_cli_fre_list_yamls(): + ''' fre list yamls ''' + result = runner.invoke(fre.fre, args=["list", "yamls"]) + assert result.exit_code == 2 + +def test_cli_fre_list_yamls_help(): + ''' fre list yamls --help ''' + result = runner.invoke(fre.fre, args=["list", "yamls", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_list_yamls_opt_dne(): + ''' fre list yamls optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "yamls", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_yamltools_cli.py b/fre/tests/test_fre_yamltools_cli.py index 30855ced5..4586fccc6 100644 --- a/fre/tests/test_fre_yamltools_cli.py +++ b/fre/tests/test_fre_yamltools_cli.py @@ -40,16 +40,30 @@ def test_cli_fre_yamltools_opt_dne(): def test_cli_fre_yamltools_combine_help(): ''' fre yamltools ''' - result = runner.invoke(fre.fre, args=["yamltools", "combine-yamls"]) + result = runner.invoke(fre.fre, args=["yamltools", "combine"]) assert result.exit_code == 0 def test_cli_fre_yamltools_combine_help(): ''' fre yamltools ''' - result = runner.invoke(fre.fre, args=["yamltools", "combine-yamls", "--help"]) + result = runner.invoke(fre.fre, args=["yamltools", "combine", "--help"]) assert result.exit_code == 0 def test_cli_fre_yamltools_combine_opt_dne(): ''' fre yamltools ''' - result = runner.invoke(fre.fre, args=["yamltools", "combine-yamls", "optionDNE"]) + result = runner.invoke(fre.fre, args=["yamltools", "combine", "optionDNE"]) assert result.exit_code == 2 +def test_cli_fre_yamltools_piped_from_list(caplog): + ''' fre list yamls -y [model.yaml] | fre yamtools combine ''' + frelist_out = runner.invoke(fre.fre, args=["list", "yamls", "-y", "fre/yamltools/tests/yamls/model.yaml"]) + assert frelist_out.exit_code == 0 + + freyamltools_pipe = runner.invoke(fre.fre, args=["-v", "yamltools", "combine"], input=frelist_out.output) + assert freyamltools_pipe.exit_code == 0 + assert "PIPED" in caplog.text + +def test_cli_fre_yamltools_NOT_piped_from_list(caplog): + ''' fre yamtools combine usage without fre list ''' + result = runner.invoke(fre.fre, args=["-v", "yamltools", "combine", "-y", "fre/yamltools/tests/yamls/model.yaml,fre/yamltools/tests/yamls/settings.yaml"]) + assert result.exit_code == 0 + assert "NOT PIPED" in caplog.text diff --git a/fre/yamltools/README.md b/fre/yamltools/README.md index f6d896f35..0397ceccf 100644 --- a/fre/yamltools/README.md +++ b/fre/yamltools/README.md @@ -1,6 +1,22 @@ ## FRE yamltools `fre yamltools` provides subtools that help to manage and perform operations on yaml files. +## Quickstart + +To access YAML examples, clone the fre-cli repository: + + git clone --recursive https://github.com/NOAA-GFDL/fre-cli.git + + cd fre/yamltools/tests + +### Independent subtool use + + fre yamltools combine -y "yamls/model.yaml,yamls/compile_yamls/platforms.yaml,yamls/compile_yamls/compile.yaml" -o out.yaml + +### Piped from `fre list` + + fre list yamls -y yamls/model.yaml | fre -v yamltools combine -o out.yaml + ## Subtools - `fre yamltools combine-yamls [options]` - Purpose: @@ -13,12 +29,12 @@ - `-e, --experiment [experiment name]` - `--use [compile|pp] (required)` -### **Tests** - -To run `fre yamltools` test scripts, return to root directory of the fre-cli repo and call those tests with - - python -m pytest fre/yamltools/tests/[test script.py] - -Or run all tests with - - python -m pytest fre/yamltools/tests +- `fre yamltools combine [options]` + - Purpose: + - Creates combined and resolved YAML file using the YAML configurations passed. + - Options: + - `-y, --yamlfile [experiment yaml] (required)` + - `-e, --experiment [experiment name]` + - `-p, --platform [platform]` + - `-t, --target [target]` + - `-o, --output [path to output file]` diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py new file mode 100644 index 000000000..213162fe6 --- /dev/null +++ b/fre/yamltools/combine_yamls_script_new.py @@ -0,0 +1,170 @@ +""" +YAML Combination Utilities for FRE +---------------------------------- + +This module provides utility functions for combining model, experiment, compile, platform, and analysis +YAML files into unified configurations for the Flexible Runtime Environment (FRE) workflow. It offers routines +to consolidate YAMLs for CMORization, compilation, and post-processing, supporting both command-line tools +and internal workflow automation. + + +- can use fre list output to pipe to this tool +- can use this tool with just model yaml (will use fre list tool) +- can use this tool to pass in multiple yamls either comma separated string + (-y y1,y2) OR multiple options (-y y1 -y y2 ...) +- checks fre-version +- uses uw config compose (to combine), resolve (to check unrendered values) +""" + +import os +import logging +import contextlib +from pathlib import Path +from typing import Optional + +from uwtools.api import config +from uwtools.api.logging import use_uwtools_logger +from fre.yamltools.helpers import output_yaml, clean_yaml#, check_fre_version + +fre_logger = logging.getLogger(__name__) + +class MergeYamls(): + """ + :ivar str yamls: is the list of YAML configuration files to combine + :ivar str experiment: is the experiment name (relates to the run and postprocessing) + :ivar str platform: is the FRE platform defined + :ivar str target: is the predefined FRE targets; options include [prod/debug/repro]-openmp + :ivar str output: is the file path to a file that will incude the final combined, resolved YAML + configuration file + """ + def __init__(self, yamls, experiment, platform, target, output, no_clean): + self.y = yamls.split(",") + self.e = str(experiment) + self.p = str(platform) + self.t = str(target) + self.o = output + self.no_clean = no_clean + + def list_check_yamls(self, init_file): + """ + :param init_file: + :type init_file: + """ + yaml_list=[] + # create intermediate yaml to pass to uw config compose + # - needs yamls files; cannot pass dictionaries + # - intermediate yaml file will be removed later + init = {"name": self.e, "platform": self.p, "target": self.t} + output_yaml(init, init_file) + + ## append init file (needed for schema and other variables?) + yaml_list.append(init_file) + + ## self.y should be a list + ## case where it might not be? + yaml_list.extend(self.y) + + for y in yaml_list: + if not Path(y).exists(): + fre_logger.error(" *** YAML FILES DNE: %s ***", y) + raise ValueError("Yaml files could be not found!") + + return yaml_list + + def use_uwtools(self, yaml_list): + """ + :param self: + :param yaml_list: + :type yaml_list: + + :raises: + """ + ## COMBINE YAMLS AND RESOLVE WHERE WE CAN ## + ## uw config compose: pass yaml list to compose final yaml + + # config.compose returns a base class specifying methods to read, manipulate, + # and write several configuration-file formats. (use as_dict to return dictionary) + # use realize=True to resolve what we can here; if any unresolved, it does not error out + # yet (use config.realize) + + # Mainly just want to pass dictionary to config.resolve but output from config.compose is a printed dictionary + # we don't need all this output but maybe it can be wrapped up in fre_logger.debug if needed? + with open(os.devnull, 'w', encoding="utf-8") as f, contextlib.redirect_stdout(f): + print("This will not be displayed") + combined_yaml_dict = config.compose(configs=yaml_list, realize=True).as_dict() + + if self.no_clean: + final_yaml_dict = combined_yaml_dict + else: + # CLEAN SERIALIZED YAML ## + final_yaml_dict = clean_yaml(combined_yaml_dict) +# if not final_yaml_dict: +# raise ValueError("YAML configuration could not be cleaned (experiments)") + + ### SHOLD BE RESOLVED BUT THIS IS TO CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## + ## uw config realize: resolve final yaml + # get nice uw tools cli output + + # Save current root logger handlers and level + root_logger = logging.getLogger() + original_handlers = list(root_logger.handlers) + original_level = root_logger.level + + + uwlogger = use_uwtools_logger() + + config.realize(input_config = final_yaml_dict, + values_needed = True, + total = True) + + # 3. Restore root logger state back to original + root_logger.handlers = original_handlers + root_logger.setLevel(original_level) + + if self.o: + out_path = Path.cwd()/self.o + fre_logger.info("Writing resolved YAML file: %s", out_path) + output_yaml(final_yaml_dict, out_path) + # Is this helpful? (if config.realize fails, it will not be written to output file, + # but error will hopefully come up; this is just more output if the user wants to + # see what the combined yaml dictonary would look like + #fre_logger.debug(pformat(final_yaml_dict)) + else: + fre_logger.info("Resolved YAML saved as dictionary. To display dictionary, pass fre -v ...") + fre_logger.info(final_yaml_dict) + + return final_yaml_dict + +def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None, no_clean: Optional[bool]=False) -> dict: + """ + :param yamls: is the list of YAML configuration files to combine + :type yamls: str + :param experiment: is the experiment name (relates to the run and postprocessing) + :type experiment: str + :param platform: is the FRE platform defined + :type platform: str + :param target: is the predefined FRE targets; options include [prod/debug/repro]-openmp + :type target: str + :param output: is the file path to a file that will incude the final combined, resolved YAML + configuration file + :type output: str + :param no_clean: is a True/False value to determine whether or not to remove any keys/sections from the YAML. Default is False. + :type no_clean: boolean + """ +# fre_logger.info('checking fre_cli_version compatibility...') +# check_fre_version(combined) + init_obj = MergeYamls(yamls, experiment, platform, target, output, no_clean) + + init_file = f"{Path.cwd()}/init.yaml" + ymls = init_obj.list_check_yamls(init_file) + combined = init_obj.use_uwtools(ymls) + + # clean init_file (not needed anymore) + Path(init_file).unlink() + + return combined + +##### NOTES: +##### - report of unrendered keys only shown when total not used with values_needed +##### - report shows keys and vals; values is a list in a list and shows fre_properties AND FRE_STEM +##### - uw config compose WILL output yaml dictionary unless --output [file] is specified diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index d97c36c66..853679ec8 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -1,7 +1,11 @@ ''' click entry-point to 'fre yamltools' calls''' - +import sys +import logging +#import ast import click from fre.yamltools import combine_yamls_script +from fre.yamltools import combine_yamls_script_new +fre_logger = logging.getLogger(__name__) @click.group(help=click.style(" - yamltools subcommands", fg=(202,177,95))) def yamltools_cli(): @@ -16,8 +20,8 @@ def yamltools_cli(): help="Platform name", required=True) @click.option("-t", "--target", type=str, help="Target name", required=True) -@click.option("--use", type=click.Choice(['compile','pp']), - help="Process user is combining yamls for. Can pass 'compile' or 'pp'", required=True) +@click.option("--use", type=click.Choice(['compile','pp', 'cmor']), + help="Process user is combining yamls for. Can pass 'compile', 'pp', or 'cmor'", required=True) @click.option("-o", "--output", type=str, help="Output") def combine_yamls(yamlfile, @@ -30,3 +34,53 @@ def combine_yamls(yamlfile, combine_yamls_script.consolidate_yamls(yamlfile, experiment, platform, target, use, output) + +@yamltools_cli.command() +@click.option("-y", "--yamls", + type=str, + help="String of comma separated list of YAML files to be combined") +@click.option("-e", "--experiment", + type=str, + help="Experiment name") +@click.option("-p", "--platform", + type=str, + help="Platform name") +@click.option("-t", "--target", + type=str, + help="Target name") +@click.option("-o", "--output", type=str, + help="Output") +@click.option("--no-clean", type=bool, + help="Do not remove anything from the combinedm resolved YAML file.") +def combine(yamls, experiment, platform, target, output, no_clean): + """ + - Combine the model yaml with the compile, platform, + experiment, and analysis yamls + """ + # if not piped from fre list yamls + yaml_paths = [] + if yamls: + fre_logger.info("NOT PIPED") + yaml_paths = yamls +# print(yamls.split(",")) + else: + fre_logger.info("PIPED") + # stdout from last tool adds a newline for some reason + yaml_paths = sys.stdin.read().strip() + + combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output, no_clean) + +#@yamltools_cli.command() +#@click.option("-y", "--yaml", +# type=str, +# help="Combined, resolved YAML file to be used for parsing", +# required=True) +#@click.option("-js", "--json-schema", +# type=str, +# help="JSON schema file used to validate the YAML", +# required=True) +#def validate(yaml, schema): +# """ +# - Validate the combined, resolved YAML file against a JSON schema +# """ +# validate_script.validate_subtool(yaml, schema) diff --git a/fre/yamltools/helpers.py b/fre/yamltools/helpers.py index 74c0930f6..eb69253dc 100644 --- a/fre/yamltools/helpers.py +++ b/fre/yamltools/helpers.py @@ -2,6 +2,7 @@ # this boots yaml with !join- see __init__ import json +import yaml import logging import os from pathlib import Path @@ -172,9 +173,12 @@ def clean_yaml(yml_dict): # Clean the yaml # If keys exists, delete: keys_clean=["experiments", "fre_cli_version"] - for kc in keys_clean: - if kc in yml_dict.keys(): - del yml_dict[kc] + try: + for kc in keys_clean: + if kc in yml_dict.keys(): + del yml_dict[kc] + except: + raise ValueError("YAML configuration could not be cleaned (experiments, fre_cli_version)") # Dump cleaned dictionary back into combined yaml file #cleaned_yaml = yaml.safe_dump(yml_dict,default_flow_style=False,sort_keys=False) diff --git a/fre/yamltools/tests/AM5_example/am5.yaml b/fre/yamltools/tests/AM5_example/am5.yaml deleted file mode 100644 index c34be59f3..000000000 --- a/fre/yamltools/tests/AM5_example/am5.yaml +++ /dev/null @@ -1,61 +0,0 @@ -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "1979" - - &EXP_AMIP_END "2020" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - -# - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # cmor - - &CMOR_START - - &CMOR_STOP "2000" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -build: - # compile information - compileYaml: "compile_yamls/compile.yaml" - platformYaml: "compile_yamls/platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - settings: "settings.yaml" - pp: - - "pp_yamls/pp.c96_amip.yaml" - - "pp_yamls/pp-TEST.c96_amip.yaml" - analysis: - - "analysis_yamls/clouds.yaml" - - "analysis_yamls/land.yaml" - cmor: # cmor-tool development, not part of official am5.yaml (yet) - - "cmor_yamls/cmor.am5.yaml" - grid_yaml: - - "grid_yamls/TEST_grids.yaml" diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml deleted file mode 100644 index d9e244dba..000000000 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "1979" - - &EXP_AMIP_END "2020" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -build: - # compile information - compileYaml: "compile.yaml" - platformYaml: "wrong_platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - pp: - - diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml deleted file mode 100644 index 2bbb39004..000000000 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "1979" - - &EXP_AMIP_END "2020" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -build: - # compile information - compileYaml: "wrong_compile.yaml" - platformYaml: "wrong_platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - pp: - - diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml deleted file mode 100644 index 819b9c3f4..000000000 --- a/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml +++ /dev/null @@ -1,28 +0,0 @@ -platforms: - - name: ncrc5.intel - compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", - "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2022.2.1"], - "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" - modelRoot: ${HOME}/fremake_canopy/test - - name: ncrc5.intel23 - compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", - "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2023.1.0"], - "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" - modelRoot: ${HOME}/fremake_canopy/test - - name: hpcme.2023 - compiler: intel - RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] - modelRoot: /apps - container: True - containerBuild: "podman" - containerRun: "apptainer" diff --git a/fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml b/fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml deleted file mode 100644 index 57ad65b85..000000000 --- a/fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# local reusable variable overrides -fre_properties: - - &custom_interp "180,360" - -#c96_amip_postprocess: -postprocess: - # main pp instructions - components: - - type: "atmos_cmip" - sources: - - history_file: "atmos_month_cmip" - - history_file: "atmos_8xdaily_cmip" - - history_file: "atmos_daily_cmip" - sourceGrid: "cubedsphere" - xyInterp: *custom_interp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos" - sources: - - history_file: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: True - - type: "atmos_level_cmip" - sources: - - history_file: "atmos_level_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_level" - sources: - - history_file: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_month_aer" - sources: - - history_file: "atmos_month_aer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_diurnal" - sources: - - history_file: "atmos_diurnal" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_scalar" - sources: - - history_file: "atmos_scalar" - postprocess_on: True - - type: "aerosol_cmip" - xyInterp: *PP_XYINTERP96 - sources: - - history_file: "aerosol_month_cmip" - sourceGrid: "cubedsphere" - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False - - type: "land" - sources: - - history_file: "land_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "land_cmip" - sources: - - history_file: "land_month_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "tracer_level" - sources: - - history_file: "atmos_tracer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False diff --git a/fre/yamltools/tests/AM5_example/settings.yaml b/fre/yamltools/tests/AM5_example/settings.yaml deleted file mode 100644 index 749dcfba5..000000000 --- a/fre/yamltools/tests/AM5_example/settings.yaml +++ /dev/null @@ -1,24 +0,0 @@ -#to-do: work on being able to overwrite these if necessary? -fre_properties: - - &PP_AMIP_CHUNK96 "P1Y" - -directories: - history_dir: !join [fre/tests/test_files/ascii_files/mock_archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [fre/tests/test_files/ascii_files/mock_archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [fre/tests/test_files/ascii_files/mock_nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "fre/tests/test_files/ascii_files/mock_xtmp/$USER/ptmp" -# fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" - -# shared pp settings -postprocess: - settings: - history_segment: "P1Y" - site: "ppan" - pp_start: *ANA_AMIP_START - pp_stop: *ANA_AMIP_END - pp_chunks: - - *PP_AMIP_CHUNK96 - pp_grid_spec: *GRID_SPEC96 - switches: - clean_work: True - do_atmos_plevel_masking: True diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 845eefca9..0b0def0b3 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -11,14 +11,14 @@ import yaml from jsonschema import validate -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy ## SET-UP # Set example yaml paths, input directory, output directory #CWD = Path.cwd() TEST_DIR = Path("fre/yamltools/tests") -IN_DIR = Path(f"{TEST_DIR}/AM5_example") +IN_DIR = Path(f"{TEST_DIR}/yamls") SCHEMA_DIR = Path("fre/gfdl_msd_schemas/FRE") # Create output directories @@ -35,7 +35,7 @@ ## Set what would be click options # Compile -COMP_EXPERIMENT = "am5" +COMP_EXPERIMENT = "model_exp" COMP_PLATFORM = "ncrc5.intel23" COMP_TARGET = "prod" @@ -48,7 +48,7 @@ def test_modelyaml_exists(): """ Make sure main yaml file exists """ - assert Path(f"{IN_DIR}/am5.yaml").exists() + assert Path(f"{IN_DIR}/model.yaml").exists() def test_compileyaml_exists(): """ @@ -67,32 +67,46 @@ def test_merged_compile_yamls(): Check for the creation of the combined-[experiment] yaml Check that the model yaml was merged into the combined yaml """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/am5.yaml")) - use = "compile" - + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/compile_yamls/compile.yaml," + f"{IN_DIR}/compile_yamls/platforms.yaml") # Merge the yamls try: - cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: assert False +def test_merged_compile_yamls_noclean(): + """ + Check for the creation of the combined-[experiment] yaml + Check that the model yaml was merged into the combined yaml + """ + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/compile_yamls/compile.yaml," + f"{IN_DIR}/compile_yamls/platforms.yaml") + # Merge the yamls + result = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = True) + assert (["fre_properties" in result, + "experiments" in result]) + def test_combined_compileyaml_validation(): """ Validate the combined compile yaml """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/am5.yaml")) - use = "compile" - + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/compile_yamls/compile.yaml," + f"{IN_DIR}/compile_yamls/platforms.yaml") # Merge the yamls try: - out = cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: assert False schema_file = os.path.join(SCHEMA_DIR, "fre_make.json") - with open(schema_file,'r') as f: + with open(schema_file, 'r', encoding="utf-8") as f: s = f.read() schema = json.loads(s) @@ -108,14 +122,15 @@ def test_combined_compileyaml_combinefail(): Check to test if compile yaml is incorrect/does not exist, the combine fails. (compile yaml path misspelled) """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_compilefile.yaml")) - use = "compile" + yamls = (f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_compilefile.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/compile.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml") # Merge the yamls - should fail since there is no compile yaml specified in the model yaml try: #out = - cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: print("EXPECTED FAILURE") assert True @@ -125,13 +140,14 @@ def test_combined_compileyaml_validatefail(): Check if the schema is validating correctly Branch should be string """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml")) - use = "compile" + yamls = (f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/wrong_compile.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml") # Merge the yamls try: - out = cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: assert False @@ -151,20 +167,12 @@ def test_combined_compileyaml_validatefail(): def test_check_expected_platformyamlcontent(): ''' Test that expected yaml information (platform info) is included in dictionary content ''' - TEST_DIR = Path("fre/make/tests") - NM_EXAMPLE = Path("null_example") - YAMLFILE = "null_model.yaml" - EXP_NAME = YAMLFILE.split(".")[0] - - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}", - experiment = EXP_NAME, - platform = None, - target = None, - use = "compile", - output = None) + yamls = ("fre/yamltools/tests/yamls/model.yaml," + "fre/yamltools/tests/yamls/compile_yamls/compile.yaml," + "fre/yamltools/tests/yamls/compile_yamls/platforms.yaml") + yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None, + no_clean = False) # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', @@ -191,7 +199,6 @@ def test_check_expected_platformyamlcontent(): 'containerBase': 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25', 'mkTemplate': '/apps/mkmf/templates/hpcme-intel21.mk', 'volume': '/gpfs/f5'} - for key,value in yml_dict.items(): if key == "platforms": assert expected_platform_info_1 in value @@ -216,13 +223,14 @@ def test_merged_pp_yamls(): Check for the creation of the combined-[experiment] yaml Check that the model yaml was merged into the combined yaml """ - # Model yaml path - modelyaml = Path(f"{IN_DIR}/am5.yaml") - use = "pp" + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/pp_yamls/pp.c96_amip.yaml," + f"{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml") # Merge the yamls try: - cy.consolidate_yamls(modelyaml, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, use, output=None) + cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, + output = None, no_clean = False) except: assert False @@ -230,17 +238,19 @@ def test_combined_ppyaml_validation(): """ Validate the combined compile yaml """ - modelyaml = Path(f"{IN_DIR}/am5.yaml") - use = 'pp' - + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/settings.yaml," + f"{IN_DIR}/pp_yamls/pp.c96_amip.yaml," + f"{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml") # Merge the yamls try: - out = cy.consolidate_yamls(modelyaml, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, use, output=None) + out = cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, + output = None, no_clean = False) except: assert False schema_file = os.path.join(SCHEMA_DIR, "fre_pp.json") - with open(schema_file,'r') as f: + with open(schema_file, 'r', encoding="utf-8") as f: s = f.read() schema = json.loads(s) @@ -253,16 +263,16 @@ def test_combine_pp_yamls(tmp_path): """ model = { - 'experiments' : [ + 'experiments' : { - 'name' : 'expname', - 'settings': 'settings.yaml', - 'pp' : [ - 'pp1.yaml', - 'pp2.yaml' - ] + 'expname': { + 'settings': 'settings.yaml', + 'pp' : [ + 'pp1.yaml', + 'pp2.yaml' + ] + } } - ] } settings = { @@ -280,27 +290,19 @@ def test_combine_pp_yamls(tmp_path): pp1 = { 'postprocess' : { - 'components' : [ - { - 'type' : 'atmos_cmip', - 'sources' : "foo bar" }, - { - 'type' : 'land', - 'sources' : "land_month"} - ] + 'components' : { + 'atmos_cmip': {'sources' : "foo bar" }, + 'land': {'sources' : "land_month"} + } } } pp2 = { 'postprocess' : { - 'components' : [ - { - 'type' : 'ocean', - 'sources' : "a b c" }, - { - 'type' : 'ice', - 'sources' : "ice_month"} - ] + 'components' : { + 'ocean': {'sources' : "a b c" }, + 'ice': {'sources' : "ice_month"} + } } } @@ -317,20 +319,12 @@ def test_combine_pp_yamls(tmp_path): 'history_segment' : 'three', 'pp_start' : 'four' }, - 'components' : [ - { - 'type' : 'atmos_cmip', - 'sources' : "foo bar" }, - { - 'type' : 'land', - 'sources' : "land_month"}, - { - 'type' : 'ocean', - 'sources' : "a b c" }, - { - 'type' : 'ice', - 'sources' : "ice_month"} - ] + 'components' : { + 'atmos_cmip': {'sources' : "foo bar" }, + 'land': {'sources' : "land_month"}, + 'ocean': {'sources' : "a b c" }, + 'ice': {'sources' : "ice_month"} + } } } @@ -338,24 +332,29 @@ def test_combine_pp_yamls(tmp_path): tmp_path.mkdir(exist_ok=True) # create model and pp yamls - file_model = open(tmp_path / 'model.yaml', 'w') - file_settings = open(tmp_path / 'settings.yaml', 'w') - file_pp1 = open(tmp_path / 'pp1.yaml', 'w') - file_pp2 = open(tmp_path / 'pp2.yaml', 'w') + with open(tmp_path / 'model.yaml', 'w', encoding="utf-8") as model_yf: + yaml.dump(model, model_yf, default_flow_style=False, sort_keys=False) + + with open(tmp_path / 'settings.yaml', 'w', encoding="utf-8") as settings_yf: + yaml.dump(settings, settings_yf, default_flow_style=False, sort_keys=False) + + with open(tmp_path / 'pp1.yaml', 'w', encoding="utf-8") as pp1_yf: + yaml.dump(pp1, pp1_yf, default_flow_style=False, sort_keys=False) - # write to/ dump info into created model and pp yamls - yaml.dump(model, file_model, default_flow_style=False, sort_keys=False) - yaml.dump(settings, file_settings, default_flow_style=False, sort_keys=False) - yaml.dump(pp1, file_pp1, default_flow_style=False, sort_keys=False) - yaml.dump(pp2, file_pp2, default_flow_style=False, sort_keys=False) + with open(tmp_path / 'pp2.yaml', 'w', encoding="utf-8") as pp2_yf: + yaml.dump(pp2, pp2_yf, default_flow_style=False, sort_keys=False) # combine the yamls # output is a combined dictionary of necessary yaml info - output = cy.consolidate_yamls(tmp_path / 'model.yaml', 'expname', 'platform', 'target', 'pp', output=None) + yamls = (f"{tmp_path}/model.yaml," + f"{tmp_path}/settings.yaml," + f"{tmp_path}/pp1.yaml,{tmp_path}/pp2.yaml") + + output = cy.yamltools_combine_subtool(yamls, 'expname', 'platform', 'target', output = None, + no_clean = False) pp = pprint.PrettyPrinter(indent=4) pp.pprint(output) pp.pprint(combined) # compare dictionaries assert output == combined - diff --git a/fre/yamltools/tests/test_helpers.py b/fre/yamltools/tests/test_helpers.py index 84e43c5bc..be5b53c9f 100644 --- a/fre/yamltools/tests/test_helpers.py +++ b/fre/yamltools/tests/test_helpers.py @@ -1,11 +1,12 @@ import os +import re import tempfile import pytest import yaml import fre -from fre.yamltools.helpers import yaml_load, check_fre_version +from fre.yamltools.helpers import yaml_load, check_fre_version, clean_yaml @pytest.fixture @@ -69,3 +70,8 @@ def test_check_fre_version_missing(yaml_without_version, caplog): with caplog.at_level(logging.WARNING): check_fre_version(yaml_without_version) assert "fre_cli_version not specified" in caplog.text, f"i'd suspect the 'import fre' in fre/yamltools/helpers" + +def test_clean_fail(): + """clean_yaml should fail if nto given a dictionary""" + with pytest.raises(ValueError, match=re.escape("YAML configuration could not be cleaned (experiments, fre_cli_version)")): + clean_yaml("fre/yamltools/tests/yamls/model.yaml") diff --git a/fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml b/fre/yamltools/tests/yamls/COMPARE_TEST_OUTPUT_cmor.yaml similarity index 100% rename from fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml rename to fre/yamltools/tests/yamls/COMPARE_TEST_OUTPUT_cmor.yaml diff --git a/fre/yamltools/tests/AM5_example/analysis_yamls/clouds.yaml b/fre/yamltools/tests/yamls/analysis_yamls/clouds.yaml similarity index 55% rename from fre/yamltools/tests/AM5_example/analysis_yamls/clouds.yaml rename to fre/yamltools/tests/yamls/analysis_yamls/clouds.yaml index 09a1ac7c4..4c84d9a35 100644 --- a/fre/yamltools/tests/AM5_example/analysis_yamls/clouds.yaml +++ b/fre/yamltools/tests/yamls/analysis_yamls/clouds.yaml @@ -2,10 +2,10 @@ analysis: clouds-test: required: data_frequency: 'mon' - date_range: [*ANA_AMIP_START, *ANA_AMIP_END] + date_range: ["{{ fre_properties.ANA_AMIP_START }}", "{{ fre_properties.ANA_AMIP_END }}"] workflow: components: ["atmos-test"] script_type: 'independent' product: 'ts' - chunk_size: *PP_AMIP_CHUNK96 + chunk_size: "{{ fre_properties.PP_AMIP_CHUNK96 }}" analysis_on: True diff --git a/fre/yamltools/tests/AM5_example/analysis_yamls/land.yaml b/fre/yamltools/tests/yamls/analysis_yamls/land.yaml similarity index 54% rename from fre/yamltools/tests/AM5_example/analysis_yamls/land.yaml rename to fre/yamltools/tests/yamls/analysis_yamls/land.yaml index 440e02313..4e797103a 100644 --- a/fre/yamltools/tests/AM5_example/analysis_yamls/land.yaml +++ b/fre/yamltools/tests/yamls/analysis_yamls/land.yaml @@ -2,10 +2,10 @@ analysis: land-test: required: data_frequency: 'mon' - date_range: [*ANA_AMIP_START, *ANA_AMIP_END] + date_range: ["{{ fre_properties.ANA_AMIP_START }}", "{{ fre_properties.ANA_AMIP_END }}"] workflow: components: ["land-test"] script_type: 'one-shot' product: 'ts' - chunk_size: *PP_AMIP_CHUNK96 + chunk_size: "{{ fre_properties.PP_AMIP_CHUNK96 }}" analysis_on: True diff --git a/fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml b/fre/yamltools/tests/yamls/cmor_yamls/cmor.yaml similarity index 91% rename from fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml rename to fre/yamltools/tests/yamls/cmor_yamls/cmor.yaml index 263507182..e6fa1ec66 100644 --- a/fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml +++ b/fre/yamltools/tests/yamls/cmor_yamls/cmor.yaml @@ -4,8 +4,7 @@ cmor: # ------------------------------------------------------------------------------------- start: # *CMOR_START - stop: - *CMOR_STOP + stop: "{{ fre_properties.CMOR_STOP }}" mip_era: 'CMIP6' @@ -16,7 +15,7 @@ cmor: # note: cmor yamls are self-contained and do NOT import settings.yaml # ------------------------------------------------------------------------------------- directories: - pp_dir: !join [fre/tests/test_files/ascii_files/mock_archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + pp_dir: "fre/tests/test_files/ascii_files/mock_archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" table_dir: &table_dir "fre/tests/test_files/cmip6-cmor-tables/Tables" outdir: diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile.yaml similarity index 81% rename from fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml rename to fre/yamltools/tests/yamls/compile_yamls/compile.yaml index 5f9a361bf..564361914 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/compile.yaml @@ -8,12 +8,12 @@ compile: cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF" branch: "2022.01" cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "am5_phys" requires: ["FMS"] repo: "https://gitlab.gfdl.noaa.gov/FMS/am5_phys.git" branch: "2022.01" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "GFDL_atmos_cubed_sphere" requires: ["FMS", "am5_phys"] repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" @@ -25,25 +25,25 @@ compile: "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90", "GFDL_atmos_cubed_sphere/tools", "GFDL_atmos_cubed_sphere/GFDL_tools"] - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "atmos_drivers" requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere"] repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" cppdefs: "-DSPMD -DCLIMATE_NUDGE" branch: "2022.01" paths: ["atmos_drivers/coupled"] - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "ice_sis" requires: ["FMS", "ice_param", "mom6"] repo: "https://gitlab.gfdl.noaa.gov/FMS/ice_sis.git" branch: "2021.02" - otherFlags: !join [*FMSincludes, " ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" - component: "ice_param" repo: "https://github.com/NOAA-GFDL/ice_param.git" cppdefs: "-Duse_yaml -Duse_libMPI -Duse_netCDF" branch: "2021.02" requires: ["FMS", "mom6"] - otherFlags: !join [*FMSincludes," ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" - component: "land_lad2" requires: ["FMS"] repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git" @@ -51,17 +51,17 @@ compile: branch: "land_lad2_2021.02" doF90Cpp: True cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "mom6" requires: ["FMS"] paths: ["mom6/MOM6-examples/src/MOM6/config_src/dynamic", "mom6/MOM6-examples/src/MOM6/config_src/coupled_driver", "mom6/MOM6-examples/src/MOM6/src/*/", "mom6/MOM6-examples/src/MOM6/src/*/*/", "mom6/ocean_BGC/generic_tracers", "mom6/ocean_BGC/mocsy/src"] branch: ["2021.02","dev/gfdl/2018.04.06"] repo: ["https://github.com/NOAA-GFDL/ocean_BGC.git","https://github.com/NOAA-GFDL/MOM6-examples.git"] makeOverrides: 'OPENMP=""' - otherFlags: !join [*FMSincludes, " ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" - component: "FMScoupler" paths: ["FMScoupler/full", "FMScoupler/shared"] repo: "https://github.com/NOAA-GFDL/FMScoupler.git" branch: "2022.01" requires: ["FMS", "atmos_drivers", "am5_phys", "land_lad2", "ice_sis", "ice_param", "mom6"] - otherFlags: !join [*FMSincludes, " ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" diff --git a/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_compilefile.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_compilefile.yaml new file mode 100644 index 000000000..49ecdea00 --- /dev/null +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_compilefile.yaml @@ -0,0 +1,20 @@ +# reusable variables +fre_properties: + AM5_VERSION: "am5f7b12r1" + FRE_STEM: "am5/{{ fre_properties.AM5_VERSION }}" + + # compile information + release: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +build: + # compile information + compileYaml: "compile.yaml" + platformYaml: "wrong_platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip" + pp: + - diff --git a/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_datatype.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_datatype.yaml new file mode 100644 index 000000000..e98048635 --- /dev/null +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_datatype.yaml @@ -0,0 +1,20 @@ +# reusable variables +fre_properties: + AM5_VERSION: "am5f7b12r1" + FRE_STEM: "am5/{{ fre_properties.AM5_VERSION }}" + + # compile information + release: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +build: + # compile information + compileYaml: "wrong_compile.yaml" + platformYaml: "wrong_platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip" + pp: + - diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_compile.yaml similarity index 85% rename from fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml rename to fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_compile.yaml index c122764a3..4ca821f85 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_compile.yaml @@ -8,4 +8,4 @@ compile: cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF" branch: 2022.01 cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_platforms.yaml similarity index 89% rename from fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml rename to fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_platforms.yaml index 896568582..2d2117661 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_platforms.yaml @@ -3,7 +3,7 @@ platforms: compiler: intel envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2022.2.1"], + "module load {{ fre_properties.INTEL}}/2022.2.1", "module load fre/bronx-23", "module load cray-hdf5/1.12.2.3", "module load cray-netcdf/4.9.0.3"] @@ -13,7 +13,7 @@ platforms: compiler: intel envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2023.1.0"], + "module load {{ fre_properties.INTEL}}/2023.1.0", "module load fre/bronx-23", "module load cray-hdf5/1.12.2.3", "module load cray-netcdf/4.9.0.3"] diff --git a/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml new file mode 100644 index 000000000..639f54c33 --- /dev/null +++ b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml @@ -0,0 +1,21 @@ +platforms: + - name: ncrc5.intel23 + compiler: intel + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", + "source $MODULESHOME/init/sh", + "module load {{ fre_properties.INTEL }}/2023.2.0", + "module load fre/bronx-23", + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" + modelRoot: ${HOME}/fremake_canopy/test + - name: hpcme.2023 + compiler: intel + RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25' + mkTemplate: '/apps/mkmf/templates/hpcme-intel21.mk' + volume: '/gpfs/f5' diff --git a/fre/yamltools/tests/AM5_example/grid_yamls/TEST_grids.yaml b/fre/yamltools/tests/yamls/grid_yamls/TEST_grids.yaml similarity index 100% rename from fre/yamltools/tests/AM5_example/grid_yamls/TEST_grids.yaml rename to fre/yamltools/tests/yamls/grid_yamls/TEST_grids.yaml diff --git a/fre/yamltools/tests/yamls/model.yaml b/fre/yamltools/tests/yamls/model.yaml new file mode 100644 index 000000000..6ed487d95 --- /dev/null +++ b/fre/yamltools/tests/yamls/model.yaml @@ -0,0 +1,48 @@ +fre_properties: + VERSION: "foo" + FRE_STEM: "am5/{{ fre_properties.VERSION }}" + + # amip + EXP_AMIP_START: "1979" + EXP_AMIP_END: "2020" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" + + PP_AMIP_CHUNK96: "P1Y" + PP_AMIP_CHUNK384: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" + + # cmor + CMOR_START: + CMOR_STOP: "2000" + + # grids + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + release: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +fre_cli_version: "2026.01.alpha2" + +build: + # compile information + compileYaml: "compile_yamls/compile.yaml" + platformYaml: "compile_yamls/platforms.yaml" + +experiments: + experiment1: + settings: "settings.yaml" + pp: + - "pp_yamls/pp.c96_amip.yaml" + - "pp_yamls/pp-TEST.c96_amip.yaml" + analysis: + - "analysis_yamls/clouds.yaml" + - "analysis_yamls/land.yaml" + cmor: # cmor-tool development, not part of official am5.yaml (yet) + - "cmor_yamls/cmor.yaml" + grid_yaml: + - "grid_yamls/TEST_grids.yaml" diff --git a/fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml b/fre/yamltools/tests/yamls/pp_yamls/pp-TEST.c96_amip.yaml similarity index 76% rename from fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml rename to fre/yamltools/tests/yamls/pp_yamls/pp-TEST.c96_amip.yaml index db43e9c42..6a447a930 100644 --- a/fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml +++ b/fre/yamltools/tests/yamls/pp_yamls/pp-TEST.c96_amip.yaml @@ -1,25 +1,25 @@ # local reusable variable overrides fre_properties: - - &custom_interp "180,360" + custom_interp: "180,360" postprocess: # main pp instructions components: - - type: "atmos_cmip-TEST" + atmos_cmip-TEST: sources: - history_file: "atmos_month_cmip" - history_file: "atmos_8xdaily_cmip" - history_file: "atmos_daily_cmip" sourceGrid: "cubedsphere" - xyInterp: *custom_interp + xyInterp: "{{ fre_properties.custom_interp }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos-TEST" + atmos-TEST: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False diff --git a/fre/yamltools/tests/yamls/pp_yamls/pp.c96_amip.yaml b/fre/yamltools/tests/yamls/pp_yamls/pp.c96_amip.yaml new file mode 100644 index 000000000..5d8966a7b --- /dev/null +++ b/fre/yamltools/tests/yamls/pp_yamls/pp.c96_amip.yaml @@ -0,0 +1,94 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + atmos_cmip: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.custom_interp }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos_level_cmip: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False +# - type: "atmos_month_aer" +# sources: +# - history_file: "atmos_month_aer" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'atmos' +# postprocess_on: False +# - type: "atmos_diurnal" +# sources: +# - history_file: "atmos_diurnal" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order2" +# inputRealm: 'atmos' +# postprocess_on: False +# - type: "atmos_scalar" +# sources: +# - history_file: "atmos_scalar" +# postprocess_on: True +# - type: "aerosol_cmip" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# sources: +# - history_file: "aerosol_month_cmip" +# sourceGrid: "cubedsphere" +# interpMethod: "conserve_order1" +# inputRealm: 'atmos' +# postprocess_on: False +# - type: "land" +# sources: +# - history_file: "land_month" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'land' +# postprocess_on: False +# - type: "land_cmip" +# sources: +# - history_file: "land_month_cmip" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'land' +# postprocess_on: False +# - type: "tracer_level" +# sources: +# - history_file: "atmos_tracer" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'atmos' +# postprocess_on: False diff --git a/fre/yamltools/tests/yamls/settings.yaml b/fre/yamltools/tests/yamls/settings.yaml new file mode 100644 index 000000000..74d463001 --- /dev/null +++ b/fre/yamltools/tests/yamls/settings.yaml @@ -0,0 +1,24 @@ +#to-do: work on being able to overwrite these if necessary? +fre_properties: + PP_AMIP_CHUNK96: "P1Y" + +directories: + history_dir: "fre/tests/test_files/ascii_files/mock_archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + pp_dir: "fre/tests/test_files/ascii_files/mock_archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "fre/tests/test_files/ascii_files/mock_nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" + ptmp_dir: "fre/tests/test_files/ascii_files/mock_xtmp/$USER/ptmp" +# fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" + +# shared pp settings +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" + pp_chunks: + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: + clean_work: True + do_atmos_plevel_masking: True