From 5d815969de25e6459eaa9156cb266dfe3ce3470c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 17:25:11 -0400 Subject: [PATCH 01/14] update fre list subtools to use tool with uwtools --- fre/list_/frelist.py | 9 +++- fre/list_/list_experiments_script.py | 51 ++++++----------------- fre/list_/list_platforms_script.py | 38 ++++++++++++----- fre/list_/list_pp_components_script.py | 45 +++++++++++++------- fre/yamltools/combine_yamls_script_new.py | 30 +++++++------ fre/yamltools/freyamltools.py | 6 ++- 6 files changed, 98 insertions(+), 81 deletions(-) diff --git a/fre/list_/frelist.py b/fre/list_/frelist.py index afad49a07..bc1f65756 100644 --- a/fre/list_/frelist.py +++ b/fre/list_/frelist.py @@ -44,9 +44,14 @@ def platforms(yamlfile): type=str, help="Experiment to be post-processed", required=True) -def pp_components(yamlfile, experiment): +@click.option("-a", + "--application", + type=str, + help="Post-processing application name.", + required=True) +def pp_components(yamlfile, experiment, application): """ - List components to be post-processed for a defined experiment""" - list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment) + list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment, application) @list_cli.command() @click.option("-y", diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index ecf205868..237c6cd75 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -5,15 +5,7 @@ """ import logging -import yaml - -from fre.yamltools.info_parsers import pp_info_parser as ppip - -## to-do: figure out validation -# Imports associated with commented block -# from pathlib import Path -# from fre.yamltools import helpers -# from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy fre_logger = logging.getLogger(__name__) @@ -29,34 +21,15 @@ def list_experiments_subtool(yamlfile: str): platform = None target = None - # Combine model - # Create pp yaml instance - yamldict = ppip.InitPPYaml(yamlfile, exp, platform, target) - yaml_str = yamldict.combine_model() - yaml_dict = yaml.load(yaml_str, Loader = yaml.Loader) + yaml_dict = cy.yamltools_combine_subtool(yamls = yamlfile, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = True) -## COULD HAVE been one way to validate but section we'd want to parse was -## cleaned in final/"combined" yaml information -## Currently not a way to validate model yaml information because we only -## have schemas for the final "combined" compile or pp information (both -## of which remove the "experiments" section I believe -# exp = yamlfile.split("/")[-1].split(".")[0] -# platform = "None" -# target = "None" -# -# # Combine model / experiment -# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, -# experiment = exp, -# platform = platform, -# target = target, -# use = "compile", -# output = None) -# -# # Validate combined yaml information -# frelist_dir = Path(__file__).resolve().parents[2] -# schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json" -# # from fre.yamltools -# helpers.validate_yaml(yml_dict, schema_path) +#### TO-DO: CURRENTLY NO YAML PIECE VALIDATION +### No way to validate just the model yaml # set logger level to INFO former_log_level = fre_logger.level @@ -64,9 +37,9 @@ def list_experiments_subtool(yamlfile: str): # log the experiment names, which should show up on screen for sure fre_logger.info("Experiments found:") - for i in yaml_dict.get("experiments"): - fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") + + for i in yaml_dict["experiments"].keys(): + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index 2ebc0a0e5..7d36d3703 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -6,7 +6,8 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) @@ -22,17 +23,33 @@ def list_platforms_subtool(yamlfile: str): former_log_level = fre_logger.level fre_logger.setLevel(logging.INFO) - exp = yamlfile.split("/")[-1].split(".")[0] + exp = None platform = None target = None - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "compile", - output = None) +# model_yf_path = Path(yamlfile).resolve().parent +# with open(yamlfile, 'r') as yf: +# yml = yaml.safe_load(yf) +# +# platform_yf_path = f"{model_yf_path}/{yml['build']['platformYaml']}" + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = None) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "compile", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) # Validate the yaml fre_pkg_dir = Path(__file__).resolve().parents[1] @@ -41,8 +58,7 @@ def list_platforms_subtool(yamlfile: str): helpers.validate_yaml(yml_dict, schema_path) fre_logger.info("Platforms available:") - for i in yml_dict.get("platforms"): + for i in yml_dict["platforms"]: fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_pp_components_script.py b/fre/list_/list_pp_components_script.py index 0be6128c7..5f475c5fb 100644 --- a/fre/list_/list_pp_components_script.py +++ b/fre/list_/list_pp_components_script.py @@ -19,12 +19,13 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) -def list_ppcomps_subtool(yamlfile: str, experiment: str): +def list_ppcomps_subtool(yamlfile: str, experiment: str, application: str): """ List_ppcomps_subtool lists the components to be post-processed. @@ -32,6 +33,8 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): :type yamlfile: str :param experiment: is the experiment name defined in the model.yaml :type experiment: str + :param application: + :type applicatin: str """ # set logger level to INFO former_log_level = fre_logger.level @@ -40,14 +43,26 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): exp = experiment platform = None target = None + app = application + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = app) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "pp", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "pp", - output = None) # Validate combined yaml information frelist_dir = Path(__file__).resolve().parents[2] @@ -55,15 +70,15 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): # from fre.yamltools helpers.validate_yaml(yml_dict, schema_path) + comp_info = yml_dict["postprocess"]["components"] # log the experiment names, which should show up on screen for sure fre_logger.info("Components to be post-processed:") - for i in yml_dict["postprocess"]["components"]: - if "postprocess_on" in i: - if i.get("postprocess_on") is True: - fre_logger.info(' - %s', i.get("type")) + for i in comp_info.keys(): + if "postprocess_on" in comp_info[i]: + if comp_info[i].get("postprocess_on") is True: + fre_logger.info(' - %s', i) else: - fre_logger.info(' - %s', i.get("type")) - fre_logger.info("\n") + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index 82410cc07..7ce5f13a9 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -37,12 +37,13 @@ class LetsGo(): :ivar str output: is the file path to a file that will incude the final combined, resolved YAML configuration file """ - def __init__(self, yamls, experiment, platform, target, output): + def __init__(self, yamls, experiment, platform, target, output, no_clean): self.y = yamls.split(",") self.e = str(experiment) self.p = str(platform) self.t = str(target) self.o = output + self.no_clean = no_clean def list_check_yamls(self, init_file): """ @@ -93,10 +94,13 @@ def use_uwtools(self, yaml_list): print("This will not be displayed") combined_yaml_dict = config.compose(configs=yaml_list, realize=True).as_dict() - # CLEAN SERIALIZED YAML ## - cleaned_yaml_dict = clean_yaml(combined_yaml_dict) - if not cleaned_yaml_dict: - raise ValueError("YAML configuration could not be cleaned (experiments)") + if self.no_clean: + final_yaml_dict = combined_yaml_dict + else: + # CLEAN SERIALIZED YAML ## + final_yaml_dict = clean_yaml(combined_yaml_dict) + if not final_yaml_dict: + raise ValueError("YAML configuration could not be cleaned (experiments)") ### SHOLD BE RESOLVED BUT THIS IS TO CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## ## uw config realize: resolve final yaml @@ -110,7 +114,7 @@ def use_uwtools(self, yaml_list): uwlogger = use_uwtools_logger() - config.realize(input_config = cleaned_yaml_dict, + config.realize(input_config = final_yaml_dict, values_needed = True, total = True) @@ -121,16 +125,16 @@ def use_uwtools(self, yaml_list): if self.o: out_path = Path.cwd()/self.o fre_logger.info("Writing resolved YAML file: %s", out_path) - output_yaml(cleaned_yaml_dict, out_path) + output_yaml(final_yaml_dict, out_path) # Is this helpful? (if config.realize fails, it will not be written to output file, # but error will hopefully come up; this is just more output if the user wants to # see what the combined yaml dictonary would look like - #fre_logger.debug(pformat(cleaned_yaml_dict)) + #fre_logger.debug(pformat(final_yaml_dict)) else: fre_logger.info("Resolved YAML saved as dictionary. To display dictionary, pass fre -v ...") - fre_logger.info(cleaned_yaml_dict) + fre_logger.info(final_yaml_dict) - return cleaned_yaml_dict + return final_yaml_dict # def validate(self, final_dict) # """ @@ -155,7 +159,7 @@ def use_uwtools(self, yaml_list): #with open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json") as j1, open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_pp.json") as j2, open("j3.json", 'w') as j3: # dump([load(j1), load(j2)], j3) -def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None) -> dict: +def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None, no_clean: Optional[bool]=False) -> dict: """ :param yamls: is the list of YAML configuration files to combine :type yamls: str @@ -168,10 +172,12 @@ def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:st :param output: is the file path to a file that will incude the final combined, resolved YAML configuration file :type output: str + :param no_clean: is a True/False value to determine whether or not to remove any keys/sections from the YAML. Default is False. + :type no_clean: boolean """ # fre_logger.info('checking fre_cli_version compatibility...') # check_fre_version(combined) - init_obj = LetsGo(yamls, experiment, platform, target, output) + init_obj = LetsGo(yamls, experiment, platform, target, output, no_clean) init_file = f"{Path.cwd()}/init.yaml" ymls = init_obj.list_check_yamls(init_file) diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index aab992354..853679ec8 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -50,7 +50,9 @@ def combine_yamls(yamlfile, help="Target name") @click.option("-o", "--output", type=str, help="Output") -def combine(yamls, experiment, platform, target, output): +@click.option("--no-clean", type=bool, + help="Do not remove anything from the combinedm resolved YAML file.") +def combine(yamls, experiment, platform, target, output, no_clean): """ - Combine the model yaml with the compile, platform, experiment, and analysis yamls @@ -66,7 +68,7 @@ def combine(yamls, experiment, platform, target, output): # stdout from last tool adds a newline for some reason yaml_paths = sys.stdin.read().strip() - combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output) + combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output, no_clean) #@yamltools_cli.command() #@click.option("-y", "--yaml", From b432561be553df09ddda55671edc61e100a7d8c2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 17:26:15 -0400 Subject: [PATCH 02/14] fix tests and add test yamls --- .../tests/test_list_experiments_script.py | 21 +++---- fre/list_/tests/test_list_platforms_script.py | 33 ++++------- .../tests/test_list_pp_components_script.py | 34 +++++------ fre/list_/tests/test_list_yamls_script.py | 30 +++++----- fre/list_/tests/yamls/compile.yaml | 49 +++++++++++++++ fre/list_/tests/yamls/empty_platforms.yaml | 1 + fre/list_/tests/yamls/model.yaml | 28 +++++---- fre/list_/tests/yamls/platforms.yaml | 34 +++++++++++ fre/list_/tests/yamls/wrong_null_model.yaml | 15 +++++ .../yamls/yaml_include/pp-test.c96_amip.yaml | 33 +++++++++++ .../tests/yamls/yaml_include/pp.c96_amip.yaml | 59 +++++++++++++++++++ .../tests/yamls/yaml_include/settings.yaml | 37 ++++++++++++ 12 files changed, 297 insertions(+), 77 deletions(-) create mode 100644 fre/list_/tests/yamls/compile.yaml create mode 100644 fre/list_/tests/yamls/empty_platforms.yaml create mode 100644 fre/list_/tests/yamls/platforms.yaml create mode 100644 fre/list_/tests/yamls/wrong_null_model.yaml create mode 100644 fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml create mode 100644 fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml create mode 100644 fre/list_/tests/yamls/yaml_include/settings.yaml diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index f78bf1f93..4a7af0c29 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -4,34 +4,29 @@ from pathlib import Path import pytest -import yaml from fre.list_ import list_experiments_script from fre.yamltools import helpers - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") -YAMLFILE = "null_model.yaml" +TEST_DIR = Path("fre/list_/tests/yamls") +YAMLFILE = "model.yaml" EXP_NAME = "None" # yaml file checks def test_modelyaml_exists(): ''' Test that model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() # Test whole tool def test_exp_list(caplog): ''' Test fre list exps subtool ''' - list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = [ 'Experiments found:', - ' - null_model_full', - ' - null_model_0', - ' - null_model_1', - ' - null_model_2' ] + ' - experiment1', + ' - experiment2' ] for i in check_out: assert i in caplog.text @@ -48,10 +43,10 @@ def test_exp_list(caplog): ) def test_yamlvalidate(): ''' Test yaml is being validated ''' - yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yamlfilepath = Path(f"{TEST_DIR}/{YAMLFILE}") # Combine model / experiment - yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # Validate and capture output assert helpers.validate_yaml(yml_dict, VAL_SCHEMA) diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index 8dd7eba56..1e0af98cb 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -4,63 +4,56 @@ from pathlib import Path import pytest -import yaml - from fre.list_ import list_platforms_script -from fre.yamltools import combine_yamls_script as cy - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") +TEST_DIR = Path("fre/list_/tests/yamls") PLATFORM = None TARGET = None -YAMLFILE = "null_model.yaml" -EXP_NAME = YAMLFILE.split(".")[0] +YAMLFILE = "model.yaml" +EXP_NAME = YAMLFILE.split(".", maxsplit=1)[0] VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_make.json") # Bad yaml example -BADYAMLFILE_PATH = f"{TEST_DIR}/{NM_EXAMPLE}/wrong_model/wrong_null_model.yaml" +BADYAMLFILE_PATH = f"{TEST_DIR}/wrong_null_model.yaml" # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() def test_compileyaml_exists(): ''' Test compile yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() + assert Path(f"{TEST_DIR}/compile.yaml").exists() def test_platformyaml_exists(): ''' Test platforms yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() + assert Path(f"{TEST_DIR}/platforms.yaml").exists() # Test whole tool def test_platforms_list_correct(caplog): ''' Test fre list platforms subtool ''' - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = ["Platforms available:", " - ncrc5.intel23", - " - hpcme.2023", + " - hpcmini.2025", " - ci.gnu", - " - con.twostep" ] + " - hpcme.intel25" ] for i in check_out: assert i in caplog.text # make sure level is INFO for record in caplog.records: - record.levelname == "INFO" + assert record.levelname == "INFO" # Test validation def test_yamlvalidate(caplog): ''' Test yaml is being validated and is actually valid''' - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - # Combine model / experiment - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -69,7 +62,7 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - record.levelname == "INFO" + assert record.levelname == "INFO" def test_not_valid_yaml(): ''' Test the correct output matches the ValueError raised when yaml is invalid ''' diff --git a/fre/list_/tests/test_list_pp_components_script.py b/fre/list_/tests/test_list_pp_components_script.py index f273d955c..ff024537c 100644 --- a/fre/list_/tests/test_list_pp_components_script.py +++ b/fre/list_/tests/test_list_pp_components_script.py @@ -6,12 +6,13 @@ # SET-UP -TEST_DIR = Path("fre/pp/tests") -AM5_EXAMPLE = Path("AM5_example") -MODEL_YAMLFILE = "am5.yaml" +#TEST_DIR = Path("fre/pp/tests") +TEST_DIR = Path("fre/list_/tests/yamls") +#AM5_EXAMPLE = Path("AM5_example") +MODEL_YAMLFILE = "model.yaml" SETTINGS_YAMLFILE = "yaml_include/settings.yaml" -PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml", "yaml_include/settings.yaml"] -EXP_NAME = "c96L65_am5f7b12r1_amip" +PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml"] #, "yaml_include/settings.yaml"] +EXP_NAME = "experiment1" VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_pp.json") PLATFORM = "FOO" TARGET = "BAR" @@ -19,35 +20,33 @@ # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{MODEL_YAMLFILE}").exists() def test_settingsyaml_exist(): ''' Test settings yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{SETTINGS_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{SETTINGS_YAMLFILE}").exists() def test_ppyamls_exist(): ''' Test post-processing yamls exist ''' for pp_yaml in PP_YAMLFILES: - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{pp_yaml}").exists() + assert Path(f"{TEST_DIR}/{pp_yaml}").exists() # Test whole tool def test_pp_comp_list(caplog): ''' Test fre list pp-components subtool ''' - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") # check the logging output check_out = [ 'Components to be post-processed:', ' - atmos_cmip', ' - atmos', - ' - atmos_level_cmip', - ' - atmos_level', - ' - atmos_month_aer', ' - atmos_diurnal', - ' - atmos_scalar'] - + ' - atmos_scalar', + ' - atmos_cmip-TEST' ] for i in check_out: assert i in caplog.text - # make sure the level is WARNING for version mismatch and INFO otherwise for record in caplog.records: assert record.levelname in ["WARNING", "INFO"] @@ -56,7 +55,9 @@ def test_pp_comp_list(caplog): def test_yamlvalidate(caplog): ''' Test yaml is being validated ''' # Combine model / experiment - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -66,4 +67,3 @@ def test_yamlvalidate(caplog): for record in caplog.records: assert record.levelname in ["WARNING", "INFO"] - diff --git a/fre/list_/tests/test_list_yamls_script.py b/fre/list_/tests/test_list_yamls_script.py index e89ca0cbe..094e1b7f5 100644 --- a/fre/list_/tests/test_list_yamls_script.py +++ b/fre/list_/tests/test_list_yamls_script.py @@ -85,7 +85,7 @@ def test_list_yamls_postprocess_only(self, model_yaml_path): # Should NOT contain compile, platform, run, or analysis YAMLs assert "compile.yaml" not in yaml_names - assert "platforms.yaml" not in yaml_names +# assert "platforms.yaml" not in yaml_names assert "run1.yaml" not in yaml_names assert "analysis1.yaml" not in yaml_names @@ -138,23 +138,23 @@ def test_list_yamls_postprocess_and_analysis(self, model_yaml_path): def test_list_yamls_no_experiment(self, model_yaml_path): """Test with no experiment name provided (default compile behavior)""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="", - application=None - ) +# with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="", + application=None + ) - yaml_names = [Path(y).name for y in result.split(",")] + yaml_names = [Path(y).name for y in result.split(",")] - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain compile and platform YAMLs - assert "compile.yaml" in yaml_names - assert "platforms.yaml" in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain compile and platform YAMLs + assert "compile.yaml" in yaml_names + assert "platforms.yaml" in yaml_names - # Should NOT contain experiment-specific YAMLs - assert "run1.yaml" not in yaml_names + # Should NOT contain experiment-specific YAMLs + assert "run1.yaml" not in yaml_names def test_list_yamls_returns_full_paths(self, model_yaml_path): """Test that returned YAMLs have full paths""" diff --git a/fre/list_/tests/yamls/compile.yaml b/fre/list_/tests/yamls/compile.yaml new file mode 100644 index 000000000..51a72b73c --- /dev/null +++ b/fre/list_/tests/yamls/compile.yaml @@ -0,0 +1,49 @@ +compile: + experiment: "null_model_full" + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + additionalInstructions: "echo \"Testing additionalInstructions\"" + cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" + otherFlags: "-fallow-argument-mismatch" # only needed for gcc + branch: "2025.04" + - component: "atmos_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/atmos_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "land_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/land_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_param" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ocean_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_null.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_null" + requires: ["FMS", "ice_param", "ocean_null"] + repo: "https://github.com/NOAA-GFDL/ice_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "coupler" + requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML" + paths: [ "coupler/full", + "coupler/shared" ] + diff --git a/fre/list_/tests/yamls/empty_platforms.yaml b/fre/list_/tests/yamls/empty_platforms.yaml new file mode 100644 index 000000000..9bceccc06 --- /dev/null +++ b/fre/list_/tests/yamls/empty_platforms.yaml @@ -0,0 +1 @@ +platforms: diff --git a/fre/list_/tests/yamls/model.yaml b/fre/list_/tests/yamls/model.yaml index 43cc46edb..035dbdcf3 100644 --- a/fre/list_/tests/yamls/model.yaml +++ b/fre/list_/tests/yamls/model.yaml @@ -1,22 +1,25 @@ # reusable variables fre_properties: - VERSION: "foo" - FRE_STEM: "test_list/{{ fre_properties.VERSION }}" + VERSION: "foo" + FRE_STEM: "test_list/{{ fre_properties.VERSION }}" # amip - EXP_AMIP_START: "19790101T0000Z" - EXP_AMIP_END: "20200101T0000Z" - ANA_AMIP_START: "1980" - ANA_AMIP_END: "2020" + EXP_AMIP_START: "19790101T0000Z" + EXP_AMIP_END: "20200101T0000Z" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" - PP_AMIP_CHUNK96: "P1Y" - PP_XYINTERP96: "180,288" - PP_XYINTERP384: "720,1152" + PP_AMIP_CHUNK96: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" # compile information - INTEL: "intel-classic" - FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + branch: "test" + + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" # compile information build: @@ -33,3 +36,4 @@ experiments: - "yaml_include/pp-test.c96_amip.yaml" analysis: - "analysis1.yaml" + experiment2: diff --git a/fre/list_/tests/yamls/platforms.yaml b/fre/list_/tests/yamls/platforms.yaml new file mode 100644 index 000000000..00d1c74fc --- /dev/null +++ b/fre/list_/tests/yamls/platforms.yaml @@ -0,0 +1,34 @@ +platforms: + - name: ncrc5.intel23 + compiler: intel + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", + "source $MODULESHOME/init/sh", + "module load {{ fre_properties.INTEL }}/2023.2.0", + "module load fre/bronx-23", + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" + modelRoot: ${HOME}/fremake_canopy/test + - name: ci.gnu + compiler: gnu + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test + - name: hpcmini.2025 + compiler: gnu + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "ghcr.io/rem1776/hpc-mini:alpine3gcc14.2" + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + - name: hpcme.intel25 + compiler: intel + RUNenv: "" + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3" + mkTemplate: "/apps/mkmf/templates/hpcme-intel2025.03-oneapi.mk" + container2step: True + container2base: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3rte" diff --git a/fre/list_/tests/yamls/wrong_null_model.yaml b/fre/list_/tests/yamls/wrong_null_model.yaml new file mode 100644 index 000000000..ccec12827 --- /dev/null +++ b/fre/list_/tests/yamls/wrong_null_model.yaml @@ -0,0 +1,15 @@ +# reusable variables +fre_properties: + VERSION: "full" + FRE_STEM: "null_model/{{ fre_properties.VERSION }}" + # compile information + branch: "main" + INTEL: "intel-classic" + FMSincludes: "-IFMS/include" + +build: + compileYaml: "compile.yaml" + platformYaml: "empty_platforms.yaml" + +experiments: + experiment1: diff --git a/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..b54f0fc77 --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "200,200" + +#c96_amip_postprocess: +postprocess: + components: + atmos_cmip-TEST: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.custom_interp}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos-TEST: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level_cmip-TEST: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..a06f8162e --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,59 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + atmos_cmip: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.custom_interp}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos_level_cmip: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_month_aer: + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + atmos_diurnal: + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_scalar: + sources: + - history_file: "atmos_scalar" diff --git a/fre/list_/tests/yamls/yaml_include/settings.yaml b/fre/list_/tests/yamls/yaml_include/settings.yaml new file mode 100644 index 000000000..b790137a0 --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/settings.yaml @@ -0,0 +1,37 @@ +#c96_amip_directories: +directories: + history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + refined_history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history_dir_refined" + pp_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "/nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" + pp_chunks: + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: + clean_work: True + do_atmos_plevel_masking: True + preanalysis: + vitals: + script: "/path/to/vitals-script" + inputs: ['atmos_month', 'aerosol_month'] + do_preanalysis: True + refinediag: + ocean: + script: "/path/to/ocean-script" + inputs: ['ocean_monthly', 'ocean_bgc'] + outputs: ['ocean_refined'] + do_refinediag: True + aerosol: + script: "/path/to/aerosol-script" + inputs: ['aerosol_month'] + outputs: ['aerosol_refined'] + do_refinediag: True From 559ede12e368573e51dde7e68acf7164c77437e2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 16:16:15 -0400 Subject: [PATCH 03/14] update fre yamltools compile tests - add yamls for this tool for now as I update yaml formats for uwtools --- .../tests/test_combine_yamls_script.py | 12 ++++++---- .../tests/yamls/compile_yamls/platforms.yaml | 23 +++++++------------ 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 16af8e6ed..f9c4ce790 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -155,10 +155,13 @@ def test_check_expected_platformyamlcontent(): # EXP_NAME = YAMLFILE.split(".")[0] # Combine model / experiment - yamls = ("fre/make/tests/null_example/null_model.yaml," - "fre/make/tests/null_example/compile.yaml," - "fre/make/tests/null_example/platforms.yaml") - yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None) +# yamls = ("fre/make/tests/null_example/null_model.yaml," +# "fre/make/tests/null_example/compile.yaml," +# "fre/make/tests/null_example/platforms.yaml") + yamls = ("fre/yamltools/tests/yamls/model.yaml," + "fre/yamltools/tests/yamls/compile_yamls/compile.yaml," + "fre/yamltools/tests/yamls/compile_yamls/platforms.yaml") + yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = "/home/Dana.Singh/fre/singh/cli-dev/generalize-yaml-serialization/HERE.yml") # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', @@ -185,7 +188,6 @@ def test_check_expected_platformyamlcontent(): 'containerBase': 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25', 'mkTemplate': '/apps/mkmf/templates/hpcme-intel21.mk', 'volume': '/gpfs/f5'} - for key,value in yml_dict.items(): if key == "platforms": assert expected_platform_info_1 in value diff --git a/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml index 144261ce1..639f54c33 100644 --- a/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml @@ -1,23 +1,13 @@ platforms: - - name: ncrc5.intel - compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", - "source $MODULESHOME/init/sh", - "module load {{ fre_properties.INTEL }}/2022.2.1", - "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" - modelRoot: ${HOME}/fremake_canopy/test - name: ncrc5.intel23 compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - "module load {{ fre_properties.INTEL }}/2023.1.0", + "module load {{ fre_properties.INTEL }}/2023.2.0", "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" modelRoot: ${HOME}/fremake_canopy/test - name: hpcme.2023 compiler: intel @@ -26,3 +16,6 @@ platforms: container: True containerBuild: "podman" containerRun: "apptainer" + containerBase: 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25' + mkTemplate: '/apps/mkmf/templates/hpcme-intel21.mk' + volume: '/gpfs/f5' From d3546ead6683c328796d3bb4b100eaa95b3a61d1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 16:42:14 -0400 Subject: [PATCH 04/14] whoops remove this --- fre/yamltools/tests/test_combine_yamls_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index f9c4ce790..fe349bc2e 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -161,7 +161,7 @@ def test_check_expected_platformyamlcontent(): yamls = ("fre/yamltools/tests/yamls/model.yaml," "fre/yamltools/tests/yamls/compile_yamls/compile.yaml," "fre/yamltools/tests/yamls/compile_yamls/platforms.yaml") - yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = "/home/Dana.Singh/fre/singh/cli-dev/generalize-yaml-serialization/HERE.yml") + yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None) # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', From 66b5f6b2b0876c0550828b84588af7234fe25444 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 17:33:42 -0400 Subject: [PATCH 05/14] only target fre package NOT uwtools logger --- fre/list_/tests/test_list_experiments_script.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index 4a7af0c29..2b9c96f2d 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -32,7 +32,8 @@ def test_exp_list(caplog): # make sure the level is INFO for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation @pytest.mark.skip( From 6a75cfc9b4197c298ff85dc55b20f5077db6a189 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 17:42:45 -0400 Subject: [PATCH 06/14] do the same for other tests --- fre/list_/tests/test_list_platforms_script.py | 6 ++++-- fre/list_/tests/test_list_pp_components_script.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index 1e0af98cb..16e6ecd2f 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -47,7 +47,8 @@ def test_platforms_list_correct(caplog): # make sure level is INFO for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation def test_yamlvalidate(caplog): @@ -62,7 +63,8 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" def test_not_valid_yaml(): ''' Test the correct output matches the ValueError raised when yaml is invalid ''' diff --git a/fre/list_/tests/test_list_pp_components_script.py b/fre/list_/tests/test_list_pp_components_script.py index ff024537c..fb0b30388 100644 --- a/fre/list_/tests/test_list_pp_components_script.py +++ b/fre/list_/tests/test_list_pp_components_script.py @@ -49,7 +49,8 @@ def test_pp_comp_list(caplog): assert i in caplog.text # make sure the level is WARNING for version mismatch and INFO otherwise for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] # Test validation def test_yamlvalidate(caplog): @@ -66,4 +67,5 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] From f12f10318f50ae434af86c62eaabd0bdfd76ccc8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 18:03:49 -0400 Subject: [PATCH 07/14] fix test for yaml additions --- fre/list_/tests/test_list_yamls_script.py | 35 +++++++++++------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/fre/list_/tests/test_list_yamls_script.py b/fre/list_/tests/test_list_yamls_script.py index 094e1b7f5..d909d355e 100644 --- a/fre/list_/tests/test_list_yamls_script.py +++ b/fre/list_/tests/test_list_yamls_script.py @@ -67,27 +67,24 @@ def test_list_yamls_runtime_only(self, model_yaml_path): def test_list_yamls_postprocess_only(self, model_yaml_path): """Test postprocess_only flag (without analysis) returns model YAML plus postprocessing YAMLs""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="experiment1", - application="postprocess" - ) - - yaml_names = [Path(y).name for y in result.split(",")] + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="experiment1", + application="postprocess" + ) - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain settings and postprocessing YAMLs - assert "settings.yaml" in yaml_names - assert "pp.c96_amip.yaml" in yaml_names - assert "pp-test.c96_amip.yaml" in yaml_names + yaml_names = [Path(y).name for y in result.split(",")] - # Should NOT contain compile, platform, run, or analysis YAMLs - assert "compile.yaml" not in yaml_names -# assert "platforms.yaml" not in yaml_names - assert "run1.yaml" not in yaml_names - assert "analysis1.yaml" not in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain settings and postprocessing YAMLs + assert "settings.yaml" in yaml_names + assert "pp.c96_amip.yaml" in yaml_names + assert "pp-test.c96_amip.yaml" in yaml_names + # Should NOT contain compile, platform, run, or analysis YAMLs + assert "compile.yaml" not in yaml_names + assert "run1.yaml" not in yaml_names + assert "analysis1.yaml" not in yaml_names def test_list_yamls_analysis_only(self, model_yaml_path): """Test analysis_only flag (without postprocess) returns model YAML plus analysis YAMLs""" From 361eac9c4ed371f1270d3ca2bfcfd0490601cd1c Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 5 Oct 2026 09:59:45 -0400 Subject: [PATCH 08/14] Fix conda install command in build_conda.yml Updated conda install command to include the '-y' flag for automatic confirmation. --- .github/workflows/build_conda.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index c7a630df5..4afcba2b4 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,7 +1,11 @@ name: build_conda +# conda-build v26.9.1 caused an error in the build related to "AttributeError" +# when looking for certain conda plugins +# Don't think plugins are needed here env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + CONDA_NO_PLUGINS: true on: pull_request: @@ -43,7 +47,7 @@ jobs: - name: Conda install conda-build run: | echo "conda install conda-build" - conda install conda-forge::conda-build + conda install -y conda-forge::conda-build - name: Check conda-build version and supported package formats run: | From f0803f30b31fdba44bfaab9b8a2b124a5ae83282 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:10:44 -0400 Subject: [PATCH 09/14] Add conda solver configuration to build workflow --- .github/workflows/build_conda.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 4afcba2b4..938fce79c 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -44,6 +44,13 @@ jobs: echo "printing conda config just in case" conda config --show + # Container being used may rely on "classic" solver but conda + # versions >=23.10.0 use a "libmamba solver as default + - name: Set conda solver + run: | + conda install --solver=classic conda-forge::conda-libmamba-solver + conda config --set solver libmamba + - name: Conda install conda-build run: | echo "conda install conda-build" From fc0c4fc1014364903691bde531890e2580ecf92e Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:27:28 -0400 Subject: [PATCH 10/14] Actually set the solver as classic Updated comments regarding conda-build and solver settings. --- .github/workflows/build_conda.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 938fce79c..b23a9f13b 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,11 +1,14 @@ name: build_conda -# conda-build v26.9.1 caused an error in the build related to "AttributeError" -# when looking for certain conda plugins -# Don't think plugins are needed here +# 1. conda-build v26.9.1 caused an error in the build related to +# "AttributeError" when looking for certain conda plugins +# Don't think plugins are needed here +# 2. Container being used may rely on "classic" solver but conda +# versions >=23.10.0 use a "libmamba solver as default env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true CONDA_NO_PLUGINS: true + CONDA_SOLVER: classic on: pull_request: @@ -44,13 +47,6 @@ jobs: echo "printing conda config just in case" conda config --show - # Container being used may rely on "classic" solver but conda - # versions >=23.10.0 use a "libmamba solver as default - - name: Set conda solver - run: | - conda install --solver=classic conda-forge::conda-libmamba-solver - conda config --set solver libmamba - - name: Conda install conda-build run: | echo "conda install conda-build" From e9440fb4c1cb64c9c29d4a5c305aa6e755678158 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 5 Oct 2026 11:22:35 -0400 Subject: [PATCH 11/14] test version that worked before --- .github/workflows/build_conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index b23a9f13b..491438eb5 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -8,7 +8,7 @@ name: build_conda env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true CONDA_NO_PLUGINS: true - CONDA_SOLVER: classic +# CONDA_SOLVER: classic on: pull_request: @@ -50,7 +50,7 @@ jobs: - name: Conda install conda-build run: | echo "conda install conda-build" - conda install -y conda-forge::conda-build + conda install -y conda-forge::conda-build=26.7.3 - name: Check conda-build version and supported package formats run: | From 6931715dfc31138b35dba687a737ade72dd9dc09 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 5 Oct 2026 17:05:34 -0400 Subject: [PATCH 12/14] install libmamba --- .github/workflows/build_conda.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 491438eb5..4c93c1542 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,14 +1,10 @@ name: build_conda -# 1. conda-build v26.9.1 caused an error in the build related to -# "AttributeError" when looking for certain conda plugins -# Don't think plugins are needed here -# 2. Container being used may rely on "classic" solver but conda -# versions >=23.10.0 use a "libmamba solver as default +# Container being used may rely on "classic" solver but conda +# versions >=23.10.0 use a "libmamba solver as default env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - CONDA_NO_PLUGINS: true -# CONDA_SOLVER: classic + CONDA_SOLVER: libmamba on: pull_request: @@ -47,7 +43,13 @@ jobs: echo "printing conda config just in case" conda config --show + - name: Conda install libmamba conda solver + run: | + conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda config --set solver libmamba + - name: Conda install conda-build + timeout-minutes: 10 run: | echo "conda install conda-build" conda install -y conda-forge::conda-build=26.7.3 From 76fb85a5ad4e1c8a9c304820828fcbc7270bd32e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 5 Oct 2026 17:19:32 -0400 Subject: [PATCH 13/14] remove version specification --- .github/workflows/build_conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 4c93c1542..fb21d5323 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -52,7 +52,7 @@ jobs: timeout-minutes: 10 run: | echo "conda install conda-build" - conda install -y conda-forge::conda-build=26.7.3 + conda install conda-forge::conda-build - name: Check conda-build version and supported package formats run: | From 79bfe5197609ccb22ad46984adbcdf3005eb584e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 6 Oct 2026 14:00:12 -0400 Subject: [PATCH 14/14] hopefully fix build --- .github/workflows/build_conda.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index fb21d5323..935efac81 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,10 +1,7 @@ name: build_conda -# Container being used may rely on "classic" solver but conda -# versions >=23.10.0 use a "libmamba solver as default env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - CONDA_SOLVER: libmamba on: pull_request: @@ -43,7 +40,7 @@ jobs: echo "printing conda config just in case" conda config --show - - name: Conda install libmamba conda solver + - name: Conda install and set libmamba conda solver run: | conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba conda config --set solver libmamba @@ -97,11 +94,22 @@ jobs: echo "printing conda config just in case" conda config --show + - name: Conda install and set libmamba conda solver + run: | + conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda config --set solver libmamba + - name: Conda install conda-build run: | echo "conda install conda-build" conda install conda-forge::conda-build + - name: Check conda-build version and supported package formats + run: | + conda --version + conda build --version + conda build --help + - name: Download tarball uses: actions/download-artifact@v5 with: