From 83d1a594b86fa33d1fd68a628b93fd161e94abda Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 27 Apr 2026 10:01:10 -0400 Subject: [PATCH 01/47] #874 refactoring fre yamltools combine-yamls to use uw tools --- fre/yamltools/__init__.py | 16 +-- fre/yamltools/combine_yamls_script_NEW.py | 118 ++++++++++++++++++++++ fre/yamltools/freyamltools.py | 21 ++++ fre/yamltools/helpers.py | 1 + 4 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 fre/yamltools/combine_yamls_script_NEW.py diff --git a/fre/yamltools/__init__.py b/fre/yamltools/__init__.py index c2e7ac2d8..706c48cf4 100644 --- a/fre/yamltools/__init__.py +++ b/fre/yamltools/__init__.py @@ -1,8 +1,8 @@ -""" -other bits in fre.yamltools use this with a `from . import *`. This way, the -yaml module understands what to do with the `!join` references in our yaml files -when they appear. -""" -import yaml -from .constructors import _join_constructor -yaml.add_constructor('!join', _join_constructor) +#""" +#other bits in fre.yamltools use this with a `from . import *`. This way, the +#yaml module understands what to do with the `!join` references in our yaml files +#when they appear. +#""" +#import yaml +#from .constructors import _join_constructor +#yaml.add_constructor('!join', _join_constructor) diff --git a/fre/yamltools/combine_yamls_script_NEW.py b/fre/yamltools/combine_yamls_script_NEW.py new file mode 100644 index 000000000..18c36f802 --- /dev/null +++ b/fre/yamltools/combine_yamls_script_NEW.py @@ -0,0 +1,118 @@ +""" +YAML Combination Utilities for FRE +---------------------------------- + +This module provides utility functions for combining model, experiment, compile, platform, and analysis +YAML files into unified configurations for the Flexible Runtime Environment (FRE) workflow. It offers routines +to consolidate YAMLs for CMORization, compilation, and post-processing, supporting both command-line tools +and internal workflow automation. +""" + +import os +import yaml +import logging +from pathlib import Path +from pprint import pformat +import pprint +from typing import Optional, Union, Any, Dict + +from fre.yamltools.helpers import output_yaml, check_fre_version, clean_yaml + +from uwtools.api import config +from uwtools.api.logging import use_uwtools_logger +#>>> use_uwtools_logger() +import yaml +from fre.list_.list_yamls_script import list_yamls_subtool + +fre_logger = logging.getLogger(__name__) + +def yamltools_combine_subtool(yamlfile:str, experiment:str, platform:str, + target:str, output: Optional[str]=None) -> dict: + """ + Depending on `use` argument passed, either create the final + combined yaml for compilation or post-processing + + :param yamlfile: Path to the model YAML configuration + :type yamlfile: str + :param experiment: Post-processing experiment name + :type experiment: str + :param platform: Platform name to be used + :type platform: str + :param output: Output file name + :type output: str + :raise ValueError: if 'use' value is not a valid entry (compile, pp or cmor) + :return: yaml dictionary containing combined information from multiple yaml configurations + :rtype: dict + + ..note:: The output file name should include a .yaml extension to indicate + it is a YAML configuration file + """ + # Check fre_cli_version compatibility before any YAML combining + fre_logger.info('checking fre_cli_version compatibility...') + check_fre_version(yamlfile) + + YAMS = [] + init = {"name": experiment, "platform": platform, "target": target} + + # create intermediate yaml to pass to uw config compose + # - needs yamls files; cannot pass dictionaries + # - will be removed later + init_file = f"{Path.cwd()}/{output}" + output_yaml(init, init_file) + +## LIST YAMLS ## + # list yamls to combine + ytc = list_yamls_subtool(yamlfile, experiment) + + ## append init file (needed for schema and other variables?) + YAMS.append(init_file) + + ## append list of yamls + YAMS.extend(ytc) + +## COMBINE YAMLS AND RESOLVE WHERE WE CAN ## + ## uw config compose: pass yaml list to compose final yaml + # CAN THIS BE DICTIONARIES?? + # config.compose returns a base class specifying methods to read, manipulate, + # and write several configuration-file formats. (use as_dict to return dictionary) + combined_yaml_dict = config.compose(configs=YAMS, realize=True).as_dict() + +## CLEAN SERIALIZED YAML ## + cleaned_yaml_dict = clean_yaml(combined_yaml_dict) + if not cleaned_yaml_dict: + raise ValueError("YAML configuration could not be cleaned (experiments)") + +## CATCH UNRESOLVED AND OUTPUT ## + ## uw config realize: resolve final yaml + ## cant add values_needed=True or else the variables won't render??? + use_uwtools_logger() + try: + if output: + fre_logger.info("Writing resolved YAML file: %s", init_file) + final_dict = config.realize(input_config = cleaned_yaml_dict, + total = True, + output_file = init_file) + else: + fre_logger.info("Resolved YAML saved as dictionary") + final_dict = config.realize(input_config = cleaned_yaml_dict, + total = True) + # clean init_file + Path(init_file).unlink() + except: +# fre_logger.info("COULD NOT WRITE FILE; COULD NOT RESOLVE, DO NOT PASS GO") + config.realize(input_config = cleaned_yaml_dict, + values_needed = True) + raise ValueError("COULD NOT WRITE FILE; COULD NOT RESOLVE, DO NOT PASS GO") + + +# VALIDATE SERIALIZED YAML ## + ## uw config validate + # output from validate is True or False + validate_out = config.validate(schema_file="/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json", + config_data=cleaned_yaml_dict) + + if validate_out: + fre_logger.info("VALID") + else: + fre_logger.error("INVALID") + raise ValueError("INVALID. CHECK YA YAMLS") diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index 0f06f8ad2..0cbf64b6e 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -2,6 +2,7 @@ import click from fre.yamltools import combine_yamls_script +from fre.yamltools import combine_yamls_script_NEW @click.group(help=click.style(" - yamltools subcommands", fg=(202,177,95))) def yamltools_cli(): @@ -30,3 +31,23 @@ def combine_yamls(yamlfile, combine_yamls_script.consolidate_yamls(yamlfile, experiment, platform, target, use, output) + +@yamltools_cli.command() +@click.option("-y", "--yamlfile", type=str, + help="YAML file to be used for parsing", required=True) +@click.option("-e", "--experiment", type=str, + help="Experiment name") +@click.option("-p", "--platform", type=str, + help="Platform name") +@click.option("-t", "--target", type=str, + help="Target name") +@click.option("-o", "--output", type=str, + help="Output") +def combine_yamls_new(yamlfile, experiment, + platform, target, + output): + """ + - Combine the model yaml with the compile, platform, + experiment, and analysis yamls + """ + combine_yamls_script_NEW.yamltools_combine_subtool(yamlfile, experiment, platform, target, output) diff --git a/fre/yamltools/helpers.py b/fre/yamltools/helpers.py index 613e514c4..0e6850016 100644 --- a/fre/yamltools/helpers.py +++ b/fre/yamltools/helpers.py @@ -2,6 +2,7 @@ # this boots yaml with !join- see __init__ import json +import yaml import logging import os from pathlib import Path From 75fb2ab675bacd0e8af5ecbfbfe0e8a5b0d426d6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 17 Jul 2026 18:47:45 -0400 Subject: [PATCH 02/47] #874 add uw tools and refactor combine tool --- environment.yml | 2 + fre/yamltools/combine_yamls_script_NEW.py | 216 +++++++++++++--------- fre/yamltools/freyamltools.py | 97 ++++++---- meta.yaml | 2 + 4 files changed, 195 insertions(+), 122 deletions(-) diff --git a/environment.yml b/environment.yml index 05801b268..713947eee 100644 --- a/environment.yml +++ b/environment.yml @@ -2,6 +2,7 @@ name: fre-cli channels: - conda-forge - noaa-gfdl + - ufs-community dependencies: - python>=3.11 - noaa-gfdl::analysis_scripts==0.0.1 @@ -26,3 +27,4 @@ dependencies: - conda-forge::pyyaml - conda-forge::xarray>=2024.* - conda-forge::netcdf4>=1.7.* + - ufs-community::uwtools==2.18.0 diff --git a/fre/yamltools/combine_yamls_script_NEW.py b/fre/yamltools/combine_yamls_script_NEW.py index 18c36f802..7ec5dfa8c 100644 --- a/fre/yamltools/combine_yamls_script_NEW.py +++ b/fre/yamltools/combine_yamls_script_NEW.py @@ -6,113 +6,151 @@ YAML files into unified configurations for the Flexible Runtime Environment (FRE) workflow. It offers routines to consolidate YAMLs for CMORization, compilation, and post-processing, supporting both command-line tools and internal workflow automation. + + +- can use fre list output to pipe to this tool +- can use this tool with just model yaml (will use fre list tool) +- can use this tool to pass in multiple yamls either comma separated string (-y y1,y2) OR multiple options (-y y1 -y y2 ...) +- checks fre-version +- uses uw config compose (to combine), resolve (to check unrendered values), and validate (to validate yaml) + """ import os import yaml import logging from pathlib import Path -from pprint import pformat -import pprint from typing import Optional, Union, Any, Dict - from fre.yamltools.helpers import output_yaml, check_fre_version, clean_yaml from uwtools.api import config from uwtools.api.logging import use_uwtools_logger -#>>> use_uwtools_logger() import yaml from fre.list_.list_yamls_script import list_yamls_subtool +import contextlib + fre_logger = logging.getLogger(__name__) -def yamltools_combine_subtool(yamlfile:str, experiment:str, platform:str, - target:str, output: Optional[str]=None) -> dict: +class LetsGo(): + def __init__(self, yamls, experiment, platform, target, output): + self.y = yamls.split(",") + self.e = experiment + self.p = platform + self.t = target + self.o = output + + def list_check_yamls(self, init_file): + """ + """ + YAMS=[] + # create intermediate yaml to pass to uw config compose + # - needs yamls files; cannot pass dictionaries + # - will be removed later + init = {"name": self.e, "platform": self.p, "target": self.t} + output_yaml(init, init_file) + + ## append init file (needed for schema and other variables?) + YAMS.append(init_file) + + #### yamls is a comma separated string, we have to make it a list + YAMS.extend(self.y) + + for y in YAMS: + if not Path(y).exists(): + fre_logger.error("ITS NOT GONNA WORK WITHOUT THEM DAWG --> DNE: %s", y) + + return YAMS + + def use_uwtools(self, YAMS): + """ + """ + ## COMBINE YAMLS AND RESOLVE WHERE WE CAN ## + ## uw config compose: pass yaml list to compose final yaml + + # CAN THIS BE DICTIONARIES?? + # config.compose returns a base class specifying methods to read, manipulate, + # and write several configuration-file formats. (use as_dict to return dictionary) + # use realize=True to resolve what we can here; if any unresolved, it does not error out yet (use config.realize) + + # Mainly just want to pass dictionary to config.resolve but output from config.compose is a printed dictionary + # we don't need all this output but maybe it can be wrapped up in fre_logger.debug if needed? + with open(os.devnull, 'w') as f, contextlib.redirect_stdout(f): + print("This will not be displayed") + combined_yaml_dict = config.compose(configs=YAMS, realize=True).as_dict() + + # CLEAN SERIALIZED YAML ## + cleaned_yaml_dict = clean_yaml(combined_yaml_dict) + if not cleaned_yaml_dict: + raise ValueError("YAML configuration could not be cleaned (experiments)") + +### CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## + ## uw config realize: resolve final yaml + # get nice uw tools cli output + use_uwtools_logger() + + final_dict = config.realize(input_config = cleaned_yaml_dict, + values_needed = True, + total = True) + + if self.o: + out_path = Path.cwd()/self.o + fre_logger.info("Writing resolved YAML file: %s", out_path) + output_yaml(cleaned_yaml_dict, out_path) + # Is this helpful? (if config.realize fails, it will not be written to output file, + # but error will hopefully come up; this is just more output if the user wants to + # see what the combined yaml dictonary would look like + #fre_logger.debug(pformat(cleaned_yaml_dict)) + else: + fre_logger.info("Resolved YAML saved as dictionary. To display dictionary, pass fre -v ...") + fre_logger.info(cleaned_yaml_dict) + + return cleaned_yaml_dict + +# def validate(self, final_dict) +# """ +# """ + +# VALIDATE SERIALIZED YAML ## +# ## uw config validate +# # output from validate is True or False +# validate_out = config.validate(schema_file="/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json", +# config_data=cleaned_yaml_dict) +# +# # If the YAML is not valid, exit with error instead of continuing +# if validate_out: +# fre_logger.info("VALID") +# else: +# fre_logger.error("INVALID") +# raise ValueError("INVALID. CHECK YA YAMLS") +# +## Combine schemas? +#from json import load, dump +# +#with open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json") as j1, open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_pp.json") as j2, open("j3.json", 'w') as j3: +# dump([load(j1), load(j2)], j3) + +def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None) -> dict: """ - Depending on `use` argument passed, either create the final - combined yaml for compilation or post-processing - - :param yamlfile: Path to the model YAML configuration - :type yamlfile: str - :param experiment: Post-processing experiment name - :type experiment: str - :param platform: Platform name to be used - :type platform: str - :param output: Output file name - :type output: str - :raise ValueError: if 'use' value is not a valid entry (compile, pp or cmor) - :return: yaml dictionary containing combined information from multiple yaml configurations - :rtype: dict - - ..note:: The output file name should include a .yaml extension to indicate - it is a YAML configuration file """ - # Check fre_cli_version compatibility before any YAML combining + init_obj = LetsGo(yamls, experiment, platform, target, output) + + init_file = f"{Path.cwd()}/init.yaml" + ymls = init_obj.list_check_yamls(init_file) + combined = init_obj.use_uwtools(ymls) + fre_logger.info('checking fre_cli_version compatibility...') - check_fre_version(yamlfile) - - YAMS = [] - init = {"name": experiment, "platform": platform, "target": target} - - # create intermediate yaml to pass to uw config compose - # - needs yamls files; cannot pass dictionaries - # - will be removed later - init_file = f"{Path.cwd()}/{output}" - output_yaml(init, init_file) - -## LIST YAMLS ## - # list yamls to combine - ytc = list_yamls_subtool(yamlfile, experiment) - - ## append init file (needed for schema and other variables?) - YAMS.append(init_file) - - ## append list of yamls - YAMS.extend(ytc) - -## COMBINE YAMLS AND RESOLVE WHERE WE CAN ## - ## uw config compose: pass yaml list to compose final yaml - # CAN THIS BE DICTIONARIES?? - # config.compose returns a base class specifying methods to read, manipulate, - # and write several configuration-file formats. (use as_dict to return dictionary) - combined_yaml_dict = config.compose(configs=YAMS, realize=True).as_dict() - -## CLEAN SERIALIZED YAML ## - cleaned_yaml_dict = clean_yaml(combined_yaml_dict) - if not cleaned_yaml_dict: - raise ValueError("YAML configuration could not be cleaned (experiments)") - -## CATCH UNRESOLVED AND OUTPUT ## - ## uw config realize: resolve final yaml - ## cant add values_needed=True or else the variables won't render??? - use_uwtools_logger() - try: - if output: - fre_logger.info("Writing resolved YAML file: %s", init_file) - final_dict = config.realize(input_config = cleaned_yaml_dict, - total = True, - output_file = init_file) - else: - fre_logger.info("Resolved YAML saved as dictionary") - final_dict = config.realize(input_config = cleaned_yaml_dict, - total = True) - # clean init_file - Path(init_file).unlink() - except: -# fre_logger.info("COULD NOT WRITE FILE; COULD NOT RESOLVE, DO NOT PASS GO") - config.realize(input_config = cleaned_yaml_dict, - values_needed = True) - raise ValueError("COULD NOT WRITE FILE; COULD NOT RESOLVE, DO NOT PASS GO") + check_fre_version(combined) +# #validate +# init_obj.validate(combined) -# VALIDATE SERIALIZED YAML ## - ## uw config validate - # output from validate is True or False - validate_out = config.validate(schema_file="/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json", - config_data=cleaned_yaml_dict) - - if validate_out: - fre_logger.info("VALID") - else: - fre_logger.error("INVALID") - raise ValueError("INVALID. CHECK YA YAMLS") + # clean init_file (not needed anymore) + Path(init_file).unlink() + + return combined + +##### NOTES: +##### - report of unrendered keys only shown when total not used with values_needed +##### - report shows keys and vals; values is a list in a list and shows fre_properties AND FRE_STEM +##### - uw config compose WILL output yaml dictionary unless --output [file] is specified diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index 0cbf64b6e..2a93cf8d0 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -1,5 +1,6 @@ ''' click entry-point to 'fre yamltools' calls''' - +import sys +import ast import click from fre.yamltools import combine_yamls_script from fre.yamltools import combine_yamls_script_NEW @@ -8,46 +9,76 @@ def yamltools_cli(): ''' entry point to fre yamltools click commands ''' -@yamltools_cli.command() -@click.option("-y", "--yamlfile", type=str, - help="YAML file to be used for parsing", required=True) -@click.option("-e", "--experiment", type=str, - help="Experiment name") -@click.option("-p", "--platform", type=str, - help="Platform name", required=True) -@click.option("-t", "--target", type=str, - help="Target name", required=True) -@click.option("--use", type=click.Choice(['compile','pp', 'cmor']), - help="Process user is combining yamls for. Can pass 'compile', 'pp', or 'cmor'", required=True) -@click.option("-o", "--output", type=str, - help="Output") -def combine_yamls(yamlfile, - experiment, platform, target, - use, output): - """ - - Combine the model yaml with the compile, platform, - experiment, and analysis yamls - """ - combine_yamls_script.consolidate_yamls(yamlfile, - experiment, platform, target, - use, output) +#@yamltools_cli.command() +#@click.option("-y", "--yamlfile", type=str, +# help="YAML file to be used for parsing", required=True) +#@click.option("-e", "--experiment", type=str, +# help="Experiment name") +#@click.option("-p", "--platform", type=str, +# help="Platform name", required=True) +#@click.option("-t", "--target", type=str, +# help="Target name", required=True) +#@click.option("--use", type=click.Choice(['compile','pp', 'cmor']), +# help="Process user is combining yamls for. Can pass 'compile', 'pp', or 'cmor'", required=True) +#@click.option("-o", "--output", type=str, +# help="Output") +#def combine_yamls(yamlfile, +# experiment, platform, target, +# use, output): +# """ +# - Combine the model yaml with the compile, platform, +# experiment, and analysis yamls +# """ +# combine_yamls_script.consolidate_yamls(yamlfile, +# experiment, platform, target, +# use, output) @yamltools_cli.command() -@click.option("-y", "--yamlfile", type=str, - help="YAML file to be used for parsing", required=True) -@click.option("-e", "--experiment", type=str, +@click.option("-y", "--yamls", + type=str, + help="YAML file to be used for parsing") +@click.option("-e", "--experiment", + type=str, help="Experiment name") -@click.option("-p", "--platform", type=str, +@click.option("-p", "--platform", + type=str, help="Platform name") -@click.option("-t", "--target", type=str, +@click.option("-t", "--target", + type=str, help="Target name") @click.option("-o", "--output", type=str, help="Output") -def combine_yamls_new(yamlfile, experiment, - platform, target, - output): +def combine(yamls, experiment, platform, target, output): """ - Combine the model yaml with the compile, platform, experiment, and analysis yamls """ - combine_yamls_script_NEW.yamltools_combine_subtool(yamlfile, experiment, platform, target, output) + # if not piped from fre list yamls + yaml_paths = [] + if yamls: + print("not piped") + yaml_paths = yamls +# print(yamls.split(",")) + else: + print("piped") + # stdout from last tool adds a newline for some reason + yaml_paths = sys.stdin.read().strip() + + combine_yamls_script_NEW.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output) +# compile_only, runtime_only, postprocess_only, +# analysis_only, output) + +#@yamltools_cli.command() +#@click.option("-y", "--yaml", +# type=str, +# help="Combined, resolved YAML file to be used for parsing", +# required=True) +#@click.option("-js", "--json-schema", +# type=str, +# help="JSON schema file used to validate the YAML", +# required=True) +#def validate(yaml, schema): +# """ +# - Validate the combined, resolved YAML file against a JSON schema +# """ +# validate_script.validate_subtool(yaml, schema) diff --git a/meta.yaml b/meta.yaml index 56b556b42..7ce3014cc 100644 --- a/meta.yaml +++ b/meta.yaml @@ -16,6 +16,7 @@ build: channels: - conda-forge - noaa-gfdl + - ufs-community requirements: build: @@ -47,6 +48,7 @@ requirements: - conda-forge::pyyaml - conda-forge::xarray>=2024.* - conda-forge::netcdf4>=1.7.* + - ufs-community::uwtools==2.18.0 test: requires: From b1e337a2f210b148f88c4e246aa9f231ec49b6ec Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 4 Sep 2026 17:07:05 -0400 Subject: [PATCH 03/47] #874 address pylint messages and add ability to pipe subtools --- fre/list_/list_yamls_script.py | 18 ++-- ...ipt_NEW.py => combine_yamls_script_new.py} | 92 +++++++++++++------ fre/yamltools/freyamltools.py | 4 +- 3 files changed, 75 insertions(+), 39 deletions(-) rename fre/yamltools/{combine_yamls_script_NEW.py => combine_yamls_script_new.py} (70%) diff --git a/fre/list_/list_yamls_script.py b/fre/list_/list_yamls_script.py index 1820fe3d0..3126634c0 100644 --- a/fre/list_/list_yamls_script.py +++ b/fre/list_/list_yamls_script.py @@ -12,6 +12,7 @@ import logging from pathlib import Path import yaml +import click fre_logger = logging.getLogger(__name__) @@ -25,6 +26,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): :type experiment: str :param application: is the application name :type application: str + :return: is a comma separated string of yaml files (absolute paths) + :rtype: str :raise ValueError: if the experiment, application passed does not exist and if yaml files do not exist @@ -94,13 +97,13 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): fre_logger.info(" - %s", y) ### Might add this in when fre yamltools combine-yamls is refactored -# fre_logger.info("") -# fre_logger.info('If combining these yamls, there are 2 options:') -# fre_logger.info(' 1. Pipe this tool to "fre yamltools combine"') -# fre_logger.info(' 2. Copy and paste this string (including quotes) -# as the -y option in "fre yamltools combine -y :') -# fre_logger.info(' "%s"', yamls_full_path) -# fre_logger.info("") + fre_logger.info("") + fre_logger.info('If combining these yamls, there are 2 options:') + fre_logger.info(' 1. Pipe this tool to "fre yamltools combine"') + fre_logger.info(' 2. Copy and paste this string (including quotes) ' + 'as the -y option in "fre yamltools combine -y :') + fre_logger.info(' "%s"', yamls_full_path) + fre_logger.info("") fre_logger.setLevel(former_log_level) # Check if the paths exist; give warning @@ -113,4 +116,5 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): if "True" in fail: raise ValueError(" *** PROVIDE THE MISSING YAML CONFIGURATIONS ***") + click.echo(yamls_full_path) return yamls_full_path diff --git a/fre/yamltools/combine_yamls_script_NEW.py b/fre/yamltools/combine_yamls_script_new.py similarity index 70% rename from fre/yamltools/combine_yamls_script_NEW.py rename to fre/yamltools/combine_yamls_script_new.py index 7ec5dfa8c..79acb3b8c 100644 --- a/fre/yamltools/combine_yamls_script_NEW.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -10,30 +10,42 @@ - can use fre list output to pipe to this tool - can use this tool with just model yaml (will use fre list tool) -- can use this tool to pass in multiple yamls either comma separated string (-y y1,y2) OR multiple options (-y y1 -y y2 ...) +- can use this tool to pass in multiple yamls either comma separated string + (-y y1,y2) OR multiple options (-y y1 -y y2 ...) - checks fre-version - uses uw config compose (to combine), resolve (to check unrendered values), and validate (to validate yaml) - """ import os -import yaml import logging +import contextlib from pathlib import Path -from typing import Optional, Union, Any, Dict -from fre.yamltools.helpers import output_yaml, check_fre_version, clean_yaml +from typing import Optional from uwtools.api import config from uwtools.api.logging import use_uwtools_logger -import yaml -from fre.list_.list_yamls_script import list_yamls_subtool - -import contextlib +#from fre.list_.list_yamls_script import list_yamls_subtool +from fre.yamltools.helpers import output_yaml, clean_yaml#, check_fre_version fre_logger = logging.getLogger(__name__) class LetsGo(): + """ + :ivar: ?? + """ def __init__(self, yamls, experiment, platform, target, output): + """ + :param yamls: + :type yamls: + :param experiment: + :type experiment: + :param platform: + :type platform: + :param target: + :type target: + :param output: + :type output: + """ self.y = yamls.split(",") self.e = experiment self.p = platform @@ -42,56 +54,66 @@ def __init__(self, yamls, experiment, platform, target, output): def list_check_yamls(self, init_file): """ + :param init_file: + :type init_file: """ - YAMS=[] + yaml_list=[] # create intermediate yaml to pass to uw config compose # - needs yamls files; cannot pass dictionaries - # - will be removed later + # - intermediate yaml file will be removed later init = {"name": self.e, "platform": self.p, "target": self.t} output_yaml(init, init_file) ## append init file (needed for schema and other variables?) - YAMS.append(init_file) + yaml_list.append(init_file) - #### yamls is a comma separated string, we have to make it a list - YAMS.extend(self.y) + ## self.y should be a list + ## case where it might not be? + yaml_list.extend(self.y) - for y in YAMS: + for y in yaml_list: if not Path(y).exists(): - fre_logger.error("ITS NOT GONNA WORK WITHOUT THEM DAWG --> DNE: %s", y) + fre_logger.error(" *** YAML FILES DNE: %s ***", y) + raise ValueError("Yaml files could be not found!") - return YAMS + return yaml_list - def use_uwtools(self, YAMS): + def use_uwtools(self, yaml_list): """ + :param self: + :param yaml_list: + :type yaml_list: + + :raises: """ - ## COMBINE YAMLS AND RESOLVE WHERE WE CAN ## + ## COMBINE YAMLS AND RESOLVE WHERE WE CAN ## ## uw config compose: pass yaml list to compose final yaml # CAN THIS BE DICTIONARIES?? # config.compose returns a base class specifying methods to read, manipulate, # and write several configuration-file formats. (use as_dict to return dictionary) - # use realize=True to resolve what we can here; if any unresolved, it does not error out yet (use config.realize) + # use realize=True to resolve what we can here; if any unresolved, it does not error out + # yet (use config.realize) # Mainly just want to pass dictionary to config.resolve but output from config.compose is a printed dictionary # we don't need all this output but maybe it can be wrapped up in fre_logger.debug if needed? - with open(os.devnull, 'w') as f, contextlib.redirect_stdout(f): + with open(os.devnull, 'w', encoding="utf-8") as f, contextlib.redirect_stdout(f): print("This will not be displayed") - combined_yaml_dict = config.compose(configs=YAMS, realize=True).as_dict() + combined_yaml_dict = config.compose(configs=yaml_list, realize=True).as_dict() - # CLEAN SERIALIZED YAML ## + # CLEAN SERIALIZED YAML ## cleaned_yaml_dict = clean_yaml(combined_yaml_dict) if not cleaned_yaml_dict: raise ValueError("YAML configuration could not be cleaned (experiments)") -### CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## + ### SHOLD BE RESOLVED BUT THIS IS TO CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## ## uw config realize: resolve final yaml # get nice uw tools cli output use_uwtools_logger() - final_dict = config.realize(input_config = cleaned_yaml_dict, - values_needed = True, - total = True) + config.realize(input_config = cleaned_yaml_dict, + values_needed = True, + total = True) if self.o: out_path = Path.cwd()/self.o @@ -132,16 +154,26 @@ def use_uwtools(self, YAMS): def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None) -> dict: """ + :param yamls: + :type yamls: str + :param experiment: + :type experiment: str + :param platform: + :type platform: str + :param target: + :type target: str + :param output: + :type output: str """ +# fre_logger.info('checking fre_cli_version compatibility...') +# check_fre_version(combined) + init_obj = LetsGo(yamls, experiment, platform, target, output) init_file = f"{Path.cwd()}/init.yaml" ymls = init_obj.list_check_yamls(init_file) combined = init_obj.use_uwtools(ymls) - fre_logger.info('checking fre_cli_version compatibility...') - check_fre_version(combined) - # #validate # init_obj.validate(combined) diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index 2e7f817b5..75727bf5e 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -3,7 +3,7 @@ import ast import click from fre.yamltools import combine_yamls_script -from fre.yamltools import combine_yamls_script_NEW +from fre.yamltools import combine_yamls_script_new @click.group(help=click.style(" - yamltools subcommands", fg=(202,177,95))) def yamltools_cli(): @@ -64,7 +64,7 @@ def combine(yamls, experiment, platform, target, output): # stdout from last tool adds a newline for some reason yaml_paths = sys.stdin.read().strip() - combine_yamls_script_NEW.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output) + combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output) #@yamltools_cli.command() #@click.option("-y", "--yaml", From db6c29bf2efbdcd738ef42641c2aacf49e1f82c7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 14 Sep 2026 15:55:18 -0400 Subject: [PATCH 04/47] #874 update uwtools version --- environment.yml | 2 +- meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 93e24cda1..b63dfa7d8 100644 --- a/environment.yml +++ b/environment.yml @@ -28,4 +28,4 @@ dependencies: - conda-forge::pyyaml - conda-forge::xarray>=2024.* - conda-forge::netcdf4>=1.7.* - - ufs-community::uwtools==2.18.0 + - ufs-community::uwtools==2.20.0 diff --git a/meta.yaml b/meta.yaml index 068e2b750..cb44110e0 100644 --- a/meta.yaml +++ b/meta.yaml @@ -41,7 +41,7 @@ requirements: - conda-forge::pyyaml - conda-forge::xarray>=2024.* - conda-forge::netcdf4>=1.7.* - - ufs-community::uwtools==2.18.0 + - ufs-community::uwtools==2.20.0 test: requires: From d637e597386ecd5c70afc0ee3dc3df045aa2f454 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 21 Sep 2026 14:42:31 -0400 Subject: [PATCH 05/47] #874 remove am5 specifics --- fre/yamltools/tests/AM5_example/am5.yaml | 61 ------------ .../compile_fail/am5-wrong_compilefile.yaml | 49 ---------- .../compile_fail/am5-wrong_datatype.yaml | 49 ---------- .../AM5_example/pp_yamls/pp.c96_amip.yaml | 94 ------------------- fre/yamltools/tests/AM5_example/settings.yaml | 24 ----- .../COMPARE_TEST_OUTPUT_cmor.yaml | 0 .../analysis_yamls/clouds.yaml | 4 +- .../analysis_yamls/land.yaml | 4 +- .../cmor_yamls/cmor.yaml} | 5 +- .../compile_yamls/compile.yaml | 18 ++-- .../compile_fail/am5-wrong_compilefile.yaml | 20 ++++ .../compile_fail/am5-wrong_datatype.yaml | 20 ++++ .../compile_fail/wrong_compile.yaml | 2 +- .../compile_fail/wrong_platforms.yaml | 4 +- .../compile_yamls/platforms.yaml | 4 +- .../grid_yamls/TEST_grids.yaml | 0 fre/yamltools/tests/yamls/model.yaml | 48 ++++++++++ .../pp_yamls/pp-TEST.c96_amip.yaml | 10 +- .../tests/yamls/pp_yamls/pp.c96_amip.yaml | 94 +++++++++++++++++++ fre/yamltools/tests/yamls/settings.yaml | 24 +++++ 20 files changed, 231 insertions(+), 303 deletions(-) delete mode 100644 fre/yamltools/tests/AM5_example/am5.yaml delete mode 100644 fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml delete mode 100644 fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml delete mode 100644 fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml delete mode 100644 fre/yamltools/tests/AM5_example/settings.yaml rename fre/yamltools/tests/{AM5_example => yamls}/COMPARE_TEST_OUTPUT_cmor.yaml (100%) rename fre/yamltools/tests/{AM5_example => yamls}/analysis_yamls/clouds.yaml (55%) rename fre/yamltools/tests/{AM5_example => yamls}/analysis_yamls/land.yaml (54%) rename fre/yamltools/tests/{AM5_example/cmor_yamls/cmor.am5.yaml => yamls/cmor_yamls/cmor.yaml} (91%) rename fre/yamltools/tests/{AM5_example => yamls}/compile_yamls/compile.yaml (81%) create mode 100644 fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_compilefile.yaml create mode 100644 fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_datatype.yaml rename fre/yamltools/tests/{AM5_example => yamls}/compile_yamls/compile_fail/wrong_compile.yaml (85%) rename fre/yamltools/tests/{AM5_example => yamls}/compile_yamls/compile_fail/wrong_platforms.yaml (89%) rename fre/yamltools/tests/{AM5_example => yamls}/compile_yamls/platforms.yaml (89%) rename fre/yamltools/tests/{AM5_example => yamls}/grid_yamls/TEST_grids.yaml (100%) create mode 100644 fre/yamltools/tests/yamls/model.yaml rename fre/yamltools/tests/{AM5_example => yamls}/pp_yamls/pp-TEST.c96_amip.yaml (76%) create mode 100644 fre/yamltools/tests/yamls/pp_yamls/pp.c96_amip.yaml create mode 100644 fre/yamltools/tests/yamls/settings.yaml diff --git a/fre/yamltools/tests/AM5_example/am5.yaml b/fre/yamltools/tests/AM5_example/am5.yaml deleted file mode 100644 index c34be59f3..000000000 --- a/fre/yamltools/tests/AM5_example/am5.yaml +++ /dev/null @@ -1,61 +0,0 @@ -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "1979" - - &EXP_AMIP_END "2020" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - -# - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # cmor - - &CMOR_START - - &CMOR_STOP "2000" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -build: - # compile information - compileYaml: "compile_yamls/compile.yaml" - platformYaml: "compile_yamls/platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - settings: "settings.yaml" - pp: - - "pp_yamls/pp.c96_amip.yaml" - - "pp_yamls/pp-TEST.c96_amip.yaml" - analysis: - - "analysis_yamls/clouds.yaml" - - "analysis_yamls/land.yaml" - cmor: # cmor-tool development, not part of official am5.yaml (yet) - - "cmor_yamls/cmor.am5.yaml" - grid_yaml: - - "grid_yamls/TEST_grids.yaml" diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml deleted file mode 100644 index d9e244dba..000000000 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "1979" - - &EXP_AMIP_END "2020" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -build: - # compile information - compileYaml: "compile.yaml" - platformYaml: "wrong_platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - pp: - - diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml deleted file mode 100644 index 2bbb39004..000000000 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "1979" - - &EXP_AMIP_END "2020" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -build: - # compile information - compileYaml: "wrong_compile.yaml" - platformYaml: "wrong_platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - pp: - - diff --git a/fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml b/fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml deleted file mode 100644 index 57ad65b85..000000000 --- a/fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# local reusable variable overrides -fre_properties: - - &custom_interp "180,360" - -#c96_amip_postprocess: -postprocess: - # main pp instructions - components: - - type: "atmos_cmip" - sources: - - history_file: "atmos_month_cmip" - - history_file: "atmos_8xdaily_cmip" - - history_file: "atmos_daily_cmip" - sourceGrid: "cubedsphere" - xyInterp: *custom_interp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos" - sources: - - history_file: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: True - - type: "atmos_level_cmip" - sources: - - history_file: "atmos_level_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_level" - sources: - - history_file: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_month_aer" - sources: - - history_file: "atmos_month_aer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_diurnal" - sources: - - history_file: "atmos_diurnal" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - postprocess_on: False - - type: "atmos_scalar" - sources: - - history_file: "atmos_scalar" - postprocess_on: True - - type: "aerosol_cmip" - xyInterp: *PP_XYINTERP96 - sources: - - history_file: "aerosol_month_cmip" - sourceGrid: "cubedsphere" - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False - - type: "land" - sources: - - history_file: "land_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "land_cmip" - sources: - - history_file: "land_month_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "tracer_level" - sources: - - history_file: "atmos_tracer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False diff --git a/fre/yamltools/tests/AM5_example/settings.yaml b/fre/yamltools/tests/AM5_example/settings.yaml deleted file mode 100644 index 749dcfba5..000000000 --- a/fre/yamltools/tests/AM5_example/settings.yaml +++ /dev/null @@ -1,24 +0,0 @@ -#to-do: work on being able to overwrite these if necessary? -fre_properties: - - &PP_AMIP_CHUNK96 "P1Y" - -directories: - history_dir: !join [fre/tests/test_files/ascii_files/mock_archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [fre/tests/test_files/ascii_files/mock_archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [fre/tests/test_files/ascii_files/mock_nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "fre/tests/test_files/ascii_files/mock_xtmp/$USER/ptmp" -# fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" - -# shared pp settings -postprocess: - settings: - history_segment: "P1Y" - site: "ppan" - pp_start: *ANA_AMIP_START - pp_stop: *ANA_AMIP_END - pp_chunks: - - *PP_AMIP_CHUNK96 - pp_grid_spec: *GRID_SPEC96 - switches: - clean_work: True - do_atmos_plevel_masking: True diff --git a/fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml b/fre/yamltools/tests/yamls/COMPARE_TEST_OUTPUT_cmor.yaml similarity index 100% rename from fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml rename to fre/yamltools/tests/yamls/COMPARE_TEST_OUTPUT_cmor.yaml diff --git a/fre/yamltools/tests/AM5_example/analysis_yamls/clouds.yaml b/fre/yamltools/tests/yamls/analysis_yamls/clouds.yaml similarity index 55% rename from fre/yamltools/tests/AM5_example/analysis_yamls/clouds.yaml rename to fre/yamltools/tests/yamls/analysis_yamls/clouds.yaml index 09a1ac7c4..4c84d9a35 100644 --- a/fre/yamltools/tests/AM5_example/analysis_yamls/clouds.yaml +++ b/fre/yamltools/tests/yamls/analysis_yamls/clouds.yaml @@ -2,10 +2,10 @@ analysis: clouds-test: required: data_frequency: 'mon' - date_range: [*ANA_AMIP_START, *ANA_AMIP_END] + date_range: ["{{ fre_properties.ANA_AMIP_START }}", "{{ fre_properties.ANA_AMIP_END }}"] workflow: components: ["atmos-test"] script_type: 'independent' product: 'ts' - chunk_size: *PP_AMIP_CHUNK96 + chunk_size: "{{ fre_properties.PP_AMIP_CHUNK96 }}" analysis_on: True diff --git a/fre/yamltools/tests/AM5_example/analysis_yamls/land.yaml b/fre/yamltools/tests/yamls/analysis_yamls/land.yaml similarity index 54% rename from fre/yamltools/tests/AM5_example/analysis_yamls/land.yaml rename to fre/yamltools/tests/yamls/analysis_yamls/land.yaml index 440e02313..4e797103a 100644 --- a/fre/yamltools/tests/AM5_example/analysis_yamls/land.yaml +++ b/fre/yamltools/tests/yamls/analysis_yamls/land.yaml @@ -2,10 +2,10 @@ analysis: land-test: required: data_frequency: 'mon' - date_range: [*ANA_AMIP_START, *ANA_AMIP_END] + date_range: ["{{ fre_properties.ANA_AMIP_START }}", "{{ fre_properties.ANA_AMIP_END }}"] workflow: components: ["land-test"] script_type: 'one-shot' product: 'ts' - chunk_size: *PP_AMIP_CHUNK96 + chunk_size: "{{ fre_properties.PP_AMIP_CHUNK96 }}" analysis_on: True diff --git a/fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml b/fre/yamltools/tests/yamls/cmor_yamls/cmor.yaml similarity index 91% rename from fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml rename to fre/yamltools/tests/yamls/cmor_yamls/cmor.yaml index 263507182..e6fa1ec66 100644 --- a/fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml +++ b/fre/yamltools/tests/yamls/cmor_yamls/cmor.yaml @@ -4,8 +4,7 @@ cmor: # ------------------------------------------------------------------------------------- start: # *CMOR_START - stop: - *CMOR_STOP + stop: "{{ fre_properties.CMOR_STOP }}" mip_era: 'CMIP6' @@ -16,7 +15,7 @@ cmor: # note: cmor yamls are self-contained and do NOT import settings.yaml # ------------------------------------------------------------------------------------- directories: - pp_dir: !join [fre/tests/test_files/ascii_files/mock_archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + pp_dir: "fre/tests/test_files/ascii_files/mock_archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" table_dir: &table_dir "fre/tests/test_files/cmip6-cmor-tables/Tables" outdir: diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile.yaml similarity index 81% rename from fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml rename to fre/yamltools/tests/yamls/compile_yamls/compile.yaml index 5f9a361bf..68f958226 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/compile.yaml @@ -8,12 +8,12 @@ compile: cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF" branch: "2022.01" cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "am5_phys" requires: ["FMS"] repo: "https://gitlab.gfdl.noaa.gov/FMS/am5_phys.git" branch: "2022.01" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "GFDL_atmos_cubed_sphere" requires: ["FMS", "am5_phys"] repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" @@ -25,25 +25,25 @@ compile: "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90", "GFDL_atmos_cubed_sphere/tools", "GFDL_atmos_cubed_sphere/GFDL_tools"] - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "atmos_drivers" requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere"] repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" cppdefs: "-DSPMD -DCLIMATE_NUDGE" branch: "2022.01" paths: ["atmos_drivers/coupled"] - otherFlags: *FMSincludes + OtherFlags: "{{ fre_properties.FMSincludes }}" - component: "ice_sis" requires: ["FMS", "ice_param", "mom6"] repo: "https://gitlab.gfdl.noaa.gov/FMS/ice_sis.git" branch: "2021.02" - otherFlags: !join [*FMSincludes, " ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" - component: "ice_param" repo: "https://github.com/NOAA-GFDL/ice_param.git" cppdefs: "-Duse_yaml -Duse_libMPI -Duse_netCDF" branch: "2021.02" requires: ["FMS", "mom6"] - otherFlags: !join [*FMSincludes," ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" - component: "land_lad2" requires: ["FMS"] repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git" @@ -51,17 +51,17 @@ compile: branch: "land_lad2_2021.02" doF90Cpp: True cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "mom6" requires: ["FMS"] paths: ["mom6/MOM6-examples/src/MOM6/config_src/dynamic", "mom6/MOM6-examples/src/MOM6/config_src/coupled_driver", "mom6/MOM6-examples/src/MOM6/src/*/", "mom6/MOM6-examples/src/MOM6/src/*/*/", "mom6/ocean_BGC/generic_tracers", "mom6/ocean_BGC/mocsy/src"] branch: ["2021.02","dev/gfdl/2018.04.06"] repo: ["https://github.com/NOAA-GFDL/ocean_BGC.git","https://github.com/NOAA-GFDL/MOM6-examples.git"] makeOverrides: 'OPENMP=""' - otherFlags: !join [*FMSincludes, " ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" - component: "FMScoupler" paths: ["FMScoupler/full", "FMScoupler/shared"] repo: "https://github.com/NOAA-GFDL/FMScoupler.git" branch: "2022.01" requires: ["FMS", "atmos_drivers", "am5_phys", "land_lad2", "ice_sis", "ice_param", "mom6"] - otherFlags: !join [*FMSincludes, " ", *momIncludes] + otherFlags: "{{ fre_properties.FMSincludes }} {{ fre_properties.momIncludes}}" diff --git a/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_compilefile.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_compilefile.yaml new file mode 100644 index 000000000..49ecdea00 --- /dev/null +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_compilefile.yaml @@ -0,0 +1,20 @@ +# reusable variables +fre_properties: + AM5_VERSION: "am5f7b12r1" + FRE_STEM: "am5/{{ fre_properties.AM5_VERSION }}" + + # compile information + release: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +build: + # compile information + compileYaml: "compile.yaml" + platformYaml: "wrong_platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip" + pp: + - diff --git a/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_datatype.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_datatype.yaml new file mode 100644 index 000000000..e98048635 --- /dev/null +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/am5-wrong_datatype.yaml @@ -0,0 +1,20 @@ +# reusable variables +fre_properties: + AM5_VERSION: "am5f7b12r1" + FRE_STEM: "am5/{{ fre_properties.AM5_VERSION }}" + + # compile information + release: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +build: + # compile information + compileYaml: "wrong_compile.yaml" + platformYaml: "wrong_platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip" + pp: + - diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_compile.yaml similarity index 85% rename from fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml rename to fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_compile.yaml index c122764a3..4ca821f85 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_compile.yaml @@ -8,4 +8,4 @@ compile: cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF" branch: 2022.01 cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF" - otherFlags: *FMSincludes + otherFlags: "{{ fre_properties.FMSincludes }}" diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_platforms.yaml similarity index 89% rename from fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml rename to fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_platforms.yaml index 896568582..2d2117661 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/compile_fail/wrong_platforms.yaml @@ -3,7 +3,7 @@ platforms: compiler: intel envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2022.2.1"], + "module load {{ fre_properties.INTEL}}/2022.2.1", "module load fre/bronx-23", "module load cray-hdf5/1.12.2.3", "module load cray-netcdf/4.9.0.3"] @@ -13,7 +13,7 @@ platforms: compiler: intel envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2023.1.0"], + "module load {{ fre_properties.INTEL}}/2023.1.0", "module load fre/bronx-23", "module load cray-hdf5/1.12.2.3", "module load cray-netcdf/4.9.0.3"] diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml similarity index 89% rename from fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml rename to fre/yamltools/tests/yamls/compile_yamls/platforms.yaml index 819b9c3f4..144261ce1 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml @@ -3,7 +3,7 @@ platforms: compiler: intel envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2022.2.1"], + "module load {{ fre_properties.INTEL }}/2022.2.1", "module load fre/bronx-23", "module load cray-hdf5/1.12.2.3", "module load cray-netcdf/4.9.0.3"] @@ -13,7 +13,7 @@ platforms: compiler: intel envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - !join ["module load ", *INTEL, "/2023.1.0"], + "module load {{ fre_properties.INTEL }}/2023.1.0", "module load fre/bronx-23", "module load cray-hdf5/1.12.2.3", "module load cray-netcdf/4.9.0.3"] diff --git a/fre/yamltools/tests/AM5_example/grid_yamls/TEST_grids.yaml b/fre/yamltools/tests/yamls/grid_yamls/TEST_grids.yaml similarity index 100% rename from fre/yamltools/tests/AM5_example/grid_yamls/TEST_grids.yaml rename to fre/yamltools/tests/yamls/grid_yamls/TEST_grids.yaml diff --git a/fre/yamltools/tests/yamls/model.yaml b/fre/yamltools/tests/yamls/model.yaml new file mode 100644 index 000000000..6ed487d95 --- /dev/null +++ b/fre/yamltools/tests/yamls/model.yaml @@ -0,0 +1,48 @@ +fre_properties: + VERSION: "foo" + FRE_STEM: "am5/{{ fre_properties.VERSION }}" + + # amip + EXP_AMIP_START: "1979" + EXP_AMIP_END: "2020" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" + + PP_AMIP_CHUNK96: "P1Y" + PP_AMIP_CHUNK384: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" + + # cmor + CMOR_START: + CMOR_STOP: "2000" + + # grids + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + release: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +fre_cli_version: "2026.01.alpha2" + +build: + # compile information + compileYaml: "compile_yamls/compile.yaml" + platformYaml: "compile_yamls/platforms.yaml" + +experiments: + experiment1: + settings: "settings.yaml" + pp: + - "pp_yamls/pp.c96_amip.yaml" + - "pp_yamls/pp-TEST.c96_amip.yaml" + analysis: + - "analysis_yamls/clouds.yaml" + - "analysis_yamls/land.yaml" + cmor: # cmor-tool development, not part of official am5.yaml (yet) + - "cmor_yamls/cmor.yaml" + grid_yaml: + - "grid_yamls/TEST_grids.yaml" diff --git a/fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml b/fre/yamltools/tests/yamls/pp_yamls/pp-TEST.c96_amip.yaml similarity index 76% rename from fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml rename to fre/yamltools/tests/yamls/pp_yamls/pp-TEST.c96_amip.yaml index db43e9c42..6a447a930 100644 --- a/fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml +++ b/fre/yamltools/tests/yamls/pp_yamls/pp-TEST.c96_amip.yaml @@ -1,25 +1,25 @@ # local reusable variable overrides fre_properties: - - &custom_interp "180,360" + custom_interp: "180,360" postprocess: # main pp instructions components: - - type: "atmos_cmip-TEST" + atmos_cmip-TEST: sources: - history_file: "atmos_month_cmip" - history_file: "atmos_8xdaily_cmip" - history_file: "atmos_daily_cmip" sourceGrid: "cubedsphere" - xyInterp: *custom_interp + xyInterp: "{{ fre_properties.custom_interp }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos-TEST" + atmos-TEST: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False diff --git a/fre/yamltools/tests/yamls/pp_yamls/pp.c96_amip.yaml b/fre/yamltools/tests/yamls/pp_yamls/pp.c96_amip.yaml new file mode 100644 index 000000000..5d8966a7b --- /dev/null +++ b/fre/yamltools/tests/yamls/pp_yamls/pp.c96_amip.yaml @@ -0,0 +1,94 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + atmos_cmip: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.custom_interp }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos_level_cmip: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False +# - type: "atmos_month_aer" +# sources: +# - history_file: "atmos_month_aer" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'atmos' +# postprocess_on: False +# - type: "atmos_diurnal" +# sources: +# - history_file: "atmos_diurnal" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order2" +# inputRealm: 'atmos' +# postprocess_on: False +# - type: "atmos_scalar" +# sources: +# - history_file: "atmos_scalar" +# postprocess_on: True +# - type: "aerosol_cmip" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# sources: +# - history_file: "aerosol_month_cmip" +# sourceGrid: "cubedsphere" +# interpMethod: "conserve_order1" +# inputRealm: 'atmos' +# postprocess_on: False +# - type: "land" +# sources: +# - history_file: "land_month" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'land' +# postprocess_on: False +# - type: "land_cmip" +# sources: +# - history_file: "land_month_cmip" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'land' +# postprocess_on: False +# - type: "tracer_level" +# sources: +# - history_file: "atmos_tracer" +# sourceGrid: "cubedsphere" +# xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" +# interpMethod: "conserve_order1" +# inputRealm: 'atmos' +# postprocess_on: False diff --git a/fre/yamltools/tests/yamls/settings.yaml b/fre/yamltools/tests/yamls/settings.yaml new file mode 100644 index 000000000..74d463001 --- /dev/null +++ b/fre/yamltools/tests/yamls/settings.yaml @@ -0,0 +1,24 @@ +#to-do: work on being able to overwrite these if necessary? +fre_properties: + PP_AMIP_CHUNK96: "P1Y" + +directories: + history_dir: "fre/tests/test_files/ascii_files/mock_archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + pp_dir: "fre/tests/test_files/ascii_files/mock_archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "fre/tests/test_files/ascii_files/mock_nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" + ptmp_dir: "fre/tests/test_files/ascii_files/mock_xtmp/$USER/ptmp" +# fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" + +# shared pp settings +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" + pp_chunks: + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: + clean_work: True + do_atmos_plevel_masking: True From 787547ccfc74b0864b1cbdfae9909c87f727458d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 21 Sep 2026 14:43:43 -0400 Subject: [PATCH 06/47] #874 update scripts and tests --- fre/yamltools/combine_yamls_script_new.py | 8 +- fre/yamltools/freyamltools.py | 46 +++---- .../tests/test_combine_yamls_script.py | 118 ++++++++---------- 3 files changed, 80 insertions(+), 92 deletions(-) diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index 79acb3b8c..e5592ea63 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -47,10 +47,10 @@ def __init__(self, yamls, experiment, platform, target, output): :type output: """ self.y = yamls.split(",") - self.e = experiment - self.p = platform - self.t = target - self.o = output + self.e = str(experiment) + self.p = str(platform) + self.t = str(target) + self.o = str(output) def list_check_yamls(self, init_file): """ diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index 75727bf5e..9179c0099 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -9,29 +9,29 @@ def yamltools_cli(): ''' entry point to fre yamltools click commands ''' -#@yamltools_cli.command() -#@click.option("-y", "--yamlfile", type=str, -# help="YAML file to be used for parsing", required=True) -#@click.option("-e", "--experiment", type=str, -# help="Experiment name") -#@click.option("-p", "--platform", type=str, -# help="Platform name", required=True) -#@click.option("-t", "--target", type=str, -# help="Target name", required=True) -#@click.option("--use", type=click.Choice(['compile','pp', 'cmor']), -# help="Process user is combining yamls for. Can pass 'compile', 'pp', or 'cmor'", required=True) -#@click.option("-o", "--output", type=str, -# help="Output") -#def combine_yamls(yamlfile, -# experiment, platform, target, -# use, output): -# """ -# - Combine the model yaml with the compile, platform, -# experiment, and analysis yamls -# """ -# combine_yamls_script.consolidate_yamls(yamlfile, -# experiment, platform, target, -# use, output) +@yamltools_cli.command() +@click.option("-y", "--yamlfile", type=str, + help="YAML file to be used for parsing", required=True) +@click.option("-e", "--experiment", type=str, + help="Experiment name") +@click.option("-p", "--platform", type=str, + help="Platform name", required=True) +@click.option("-t", "--target", type=str, + help="Target name", required=True) +@click.option("--use", type=click.Choice(['compile','pp', 'cmor']), + help="Process user is combining yamls for. Can pass 'compile', 'pp', or 'cmor'", required=True) +@click.option("-o", "--output", type=str, + help="Output") +def combine_yamls(yamlfile, + experiment, platform, target, + use, output): + """ + - Combine the model yaml with the compile, platform, + experiment, and analysis yamls + """ + combine_yamls_script.consolidate_yamls(yamlfile, + experiment, platform, target, + use, output) @yamltools_cli.command() @click.option("-y", "--yamls", diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 845eefca9..7af940e0c 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -11,7 +11,7 @@ import yaml from jsonschema import validate -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy #import combine_yamls_script_new as cy ## SET-UP @@ -34,6 +34,7 @@ Path(outdir).mkdir(parents=True,exist_ok=True) ## Set what would be click options +#YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml" # Compile COMP_EXPERIMENT = "am5" COMP_PLATFORM = "ncrc5.intel23" @@ -69,11 +70,10 @@ def test_merged_compile_yamls(): """ # Model yaml path modelyaml = str(Path(f"{IN_DIR}/am5.yaml")) - use = "compile" - + YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml" # Merge the yamls try: - cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: assert False @@ -83,11 +83,10 @@ def test_combined_compileyaml_validation(): """ # Model yaml path modelyaml = str(Path(f"{IN_DIR}/am5.yaml")) - use = "compile" - + YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml" # Merge the yamls try: - out = cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + out = cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: assert False @@ -108,14 +107,12 @@ def test_combined_compileyaml_combinefail(): Check to test if compile yaml is incorrect/does not exist, the combine fails. (compile yaml path misspelled) """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_compilefile.yaml")) - use = "compile" + YAMLS = "fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml" # Merge the yamls - should fail since there is no compile yaml specified in the model yaml try: #out = - cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: print("EXPECTED FAILURE") assert True @@ -127,11 +124,12 @@ def test_combined_compileyaml_validatefail(): """ # Model yaml path modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml")) - use = "compile" + + YAMLS = "fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml" # Merge the yamls try: - out = cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use, output = None) + out = cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: assert False @@ -156,15 +154,9 @@ def test_check_expected_platformyamlcontent(): YAMLFILE = "null_model.yaml" EXP_NAME = YAMLFILE.split(".")[0] - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}", - experiment = EXP_NAME, - platform = None, - target = None, - use = "compile", - output = None) + YAMLS = "fre/make/tests/null_example/null_model.yaml,fre/make/tests/null_example/compile.yaml,fre/make/tests/null_example/platforms.yaml" + yml_dict = cy.yamltools_combine_subtool(YAMLS, None, None, None, output = None) # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', @@ -218,11 +210,20 @@ def test_merged_pp_yamls(): """ # Model yaml path modelyaml = Path(f"{IN_DIR}/am5.yaml") - use = "pp" + YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml" + +# analysis: +# - "analysis_yamls/clouds.yaml" +# - "analysis_yamls/land.yaml" +# cmor: # cmor-tool development, not part of official am5.yaml (yet) +# - "cmor_yamls/cmor.am5.yaml" +# grid_yaml: +# - "grid_yamls/TEST_grids.yaml" + # Merge the yamls try: - cy.consolidate_yamls(modelyaml, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, use, output=None) + cy.yamltools_combine_subtool(YAMLS, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) except: assert False @@ -231,11 +232,11 @@ def test_combined_ppyaml_validation(): Validate the combined compile yaml """ modelyaml = Path(f"{IN_DIR}/am5.yaml") - use = 'pp' + YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml" # Merge the yamls try: - out = cy.consolidate_yamls(modelyaml, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, use, output=None) + out = cy.yamltools_combine_subtool(YAMLS, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) except: assert False @@ -253,16 +254,16 @@ def test_combine_pp_yamls(tmp_path): """ model = { - 'experiments' : [ + 'experiments' : { - 'name' : 'expname', - 'settings': 'settings.yaml', - 'pp' : [ - 'pp1.yaml', - 'pp2.yaml' - ] + 'expname': { + 'settings': 'settings.yaml', + 'pp' : [ + 'pp1.yaml', + 'pp2.yaml' + ] + } } - ] } settings = { @@ -280,29 +281,21 @@ def test_combine_pp_yamls(tmp_path): pp1 = { 'postprocess' : { - 'components' : [ - { - 'type' : 'atmos_cmip', - 'sources' : "foo bar" }, - { - 'type' : 'land', - 'sources' : "land_month"} - ] + 'components' : { + 'atmos_cmip': {'sources' : "foo bar" }, + 'land': {'sources' : "land_month"} + } } } pp2 = { 'postprocess' : { - 'components' : [ - { - 'type' : 'ocean', - 'sources' : "a b c" }, - { - 'type' : 'ice', - 'sources' : "ice_month"} - ] + 'components' : { + 'ocean': {'sources' : "a b c" }, + 'ice': {'sources' : "ice_month"} + } } - } + } combined = { 'name' : 'expname', @@ -317,20 +310,12 @@ def test_combine_pp_yamls(tmp_path): 'history_segment' : 'three', 'pp_start' : 'four' }, - 'components' : [ - { - 'type' : 'atmos_cmip', - 'sources' : "foo bar" }, - { - 'type' : 'land', - 'sources' : "land_month"}, - { - 'type' : 'ocean', - 'sources' : "a b c" }, - { - 'type' : 'ice', - 'sources' : "ice_month"} - ] + 'components' : { + 'atmos_cmip': {'sources' : "foo bar" }, + 'land': {'sources' : "land_month"}, + 'ocean': {'sources' : "a b c" }, + 'ice': {'sources' : "ice_month"} + } } } @@ -351,7 +336,10 @@ def test_combine_pp_yamls(tmp_path): # combine the yamls # output is a combined dictionary of necessary yaml info - output = cy.consolidate_yamls(tmp_path / 'model.yaml', 'expname', 'platform', 'target', 'pp', output=None) +# output = cy.consolidate_yamls(tmp_path / 'model.yaml', 'expname', 'platform', 'target', 'pp', output=None) + YAMLS = f"{tmp_path}/model.yaml,{tmp_path}/settings.yaml,{tmp_path}/pp1.yaml,{tmp_path}/pp2.yaml" + + output = cy.yamltools_combine_subtool(YAMLS, 'expname', 'platform', 'target', output = None) pp = pprint.PrettyPrinter(indent=4) pp.pprint(output) pp.pprint(combined) From 837357446a5d9ddc221a72a365b2b4f79a4aac65 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 21 Sep 2026 15:50:17 -0400 Subject: [PATCH 07/47] #874 update tests --- .../tests/test_combine_yamls_script.py | 27 +++++++++---------- .../tests/yamls/compile_yamls/compile.yaml | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 7af940e0c..49a8e52a6 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -18,7 +18,7 @@ # Set example yaml paths, input directory, output directory #CWD = Path.cwd() TEST_DIR = Path("fre/yamltools/tests") -IN_DIR = Path(f"{TEST_DIR}/AM5_example") +IN_DIR = Path(f"{TEST_DIR}/yamls") SCHEMA_DIR = Path("fre/gfdl_msd_schemas/FRE") # Create output directories @@ -36,7 +36,7 @@ ## Set what would be click options #YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml" # Compile -COMP_EXPERIMENT = "am5" +COMP_EXPERIMENT = "model_exp" COMP_PLATFORM = "ncrc5.intel23" COMP_TARGET = "prod" @@ -49,7 +49,7 @@ def test_modelyaml_exists(): """ Make sure main yaml file exists """ - assert Path(f"{IN_DIR}/am5.yaml").exists() + assert Path(f"{IN_DIR}/model.yaml").exists() def test_compileyaml_exists(): """ @@ -69,8 +69,8 @@ def test_merged_compile_yamls(): Check that the model yaml was merged into the combined yaml """ # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/am5.yaml")) - YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml" + modelyaml = str(Path(f"{IN_DIR}/model.yaml")) + YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/compile_yamls/compile.yaml,{IN_DIR}/compile_yamls/platforms.yaml" # Merge the yamls try: cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) @@ -82,8 +82,8 @@ def test_combined_compileyaml_validation(): Validate the combined compile yaml """ # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/am5.yaml")) - YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml" + modelyaml = str(Path(f"{IN_DIR}/model.yaml")) + YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/compile_yamls/compile.yaml,{IN_DIR}/compile_yamls/platforms.yaml" # Merge the yamls try: out = cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) @@ -107,7 +107,7 @@ def test_combined_compileyaml_combinefail(): Check to test if compile yaml is incorrect/does not exist, the combine fails. (compile yaml path misspelled) """ - YAMLS = "fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml" + YAMLS = f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_compilefile.yaml,{IN_DIR}/compile_yamls/compile_fail/compile.yaml,{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml" # Merge the yamls - should fail since there is no compile yaml specified in the model yaml try: @@ -125,7 +125,7 @@ def test_combined_compileyaml_validatefail(): # Model yaml path modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml")) - YAMLS = "fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/wrong_platforms.yaml" + YAMLS = f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml,{IN_DIR}/compile_yamls/compile_fail/wrong_compile.yaml,{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml" # Merge the yamls try: @@ -209,8 +209,8 @@ def test_merged_pp_yamls(): Check that the model yaml was merged into the combined yaml """ # Model yaml path - modelyaml = Path(f"{IN_DIR}/am5.yaml") - YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml" + modelyaml = Path(f"{IN_DIR}/model.yaml") + YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/pp_yamls/pp.c96_amip.yaml,{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml" # analysis: # - "analysis_yamls/clouds.yaml" @@ -231,9 +231,8 @@ def test_combined_ppyaml_validation(): """ Validate the combined compile yaml """ - modelyaml = Path(f"{IN_DIR}/am5.yaml") - YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp.c96_amip.yaml,fre/yamltools/tests/AM5_example/pp_yamls/pp-TEST.c96_amip.yaml" - + modelyaml = Path(f"{IN_DIR}/model.yaml") + YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/settings.yaml,{IN_DIR}/pp_yamls/pp.c96_amip.yaml,{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml" # Merge the yamls try: out = cy.yamltools_combine_subtool(YAMLS, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) diff --git a/fre/yamltools/tests/yamls/compile_yamls/compile.yaml b/fre/yamltools/tests/yamls/compile_yamls/compile.yaml index 68f958226..564361914 100644 --- a/fre/yamltools/tests/yamls/compile_yamls/compile.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/compile.yaml @@ -32,7 +32,7 @@ compile: cppdefs: "-DSPMD -DCLIMATE_NUDGE" branch: "2022.01" paths: ["atmos_drivers/coupled"] - OtherFlags: "{{ fre_properties.FMSincludes }}" + otherFlags: "{{ fre_properties.FMSincludes }}" - component: "ice_sis" requires: ["FMS", "ice_param", "mom6"] repo: "https://gitlab.gfdl.noaa.gov/FMS/ice_sis.git" From b5cd8857fa756816b242ea1226bca47703b2a83d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 21 Sep 2026 17:06:06 -0400 Subject: [PATCH 08/47] #874 address some pylint messages --- fre/yamltools/combine_yamls_script_new.py | 1 - .../tests/test_combine_yamls_script.py | 106 +++++++++--------- 2 files changed, 51 insertions(+), 56 deletions(-) diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index e5592ea63..aaea2f3ae 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -167,7 +167,6 @@ def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:st """ # fre_logger.info('checking fre_cli_version compatibility...') # check_fre_version(combined) - init_obj = LetsGo(yamls, experiment, platform, target, output) init_file = f"{Path.cwd()}/init.yaml" diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 49a8e52a6..22910b5d6 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -34,7 +34,6 @@ Path(outdir).mkdir(parents=True,exist_ok=True) ## Set what would be click options -#YAMLS = "fre/yamltools/tests/AM5_example/am5.yaml,fre/yamltools/tests/AM5_example/compile_yamls/compile.yaml,fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml" # Compile COMP_EXPERIMENT = "model_exp" COMP_PLATFORM = "ncrc5.intel23" @@ -68,12 +67,12 @@ def test_merged_compile_yamls(): Check for the creation of the combined-[experiment] yaml Check that the model yaml was merged into the combined yaml """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/model.yaml")) - YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/compile_yamls/compile.yaml,{IN_DIR}/compile_yamls/platforms.yaml" + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/compile_yamls/compile.yaml," + f"{IN_DIR}/compile_yamls/platforms.yaml") # Merge the yamls try: - cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: assert False @@ -81,17 +80,17 @@ def test_combined_compileyaml_validation(): """ Validate the combined compile yaml """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/model.yaml")) - YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/compile_yamls/compile.yaml,{IN_DIR}/compile_yamls/platforms.yaml" + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/compile_yamls/compile.yaml," + f"{IN_DIR}/compile_yamls/platforms.yaml") # Merge the yamls try: - out = cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: assert False schema_file = os.path.join(SCHEMA_DIR, "fre_make.json") - with open(schema_file,'r') as f: + with open(schema_file, 'r', encoding="utf-8") as f: s = f.read() schema = json.loads(s) @@ -107,12 +106,14 @@ def test_combined_compileyaml_combinefail(): Check to test if compile yaml is incorrect/does not exist, the combine fails. (compile yaml path misspelled) """ - YAMLS = f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_compilefile.yaml,{IN_DIR}/compile_yamls/compile_fail/compile.yaml,{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml" + yamls = (f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_compilefile.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/compile.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml") # Merge the yamls - should fail since there is no compile yaml specified in the model yaml try: #out = - cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: print("EXPECTED FAILURE") assert True @@ -122,14 +123,13 @@ def test_combined_compileyaml_validatefail(): Check if the schema is validating correctly Branch should be string """ - # Model yaml path - modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml")) - - YAMLS = f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml,{IN_DIR}/compile_yamls/compile_fail/wrong_compile.yaml,{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml" + yamls = (f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/wrong_compile.yaml," + f"{IN_DIR}/compile_yamls/compile_fail/wrong_platforms.yaml") # Merge the yamls try: - out = cy.yamltools_combine_subtool(YAMLS, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) except: assert False @@ -149,14 +149,16 @@ def test_combined_compileyaml_validatefail(): def test_check_expected_platformyamlcontent(): ''' Test that expected yaml information (platform info) is included in dictionary content ''' - TEST_DIR = Path("fre/make/tests") - NM_EXAMPLE = Path("null_example") - YAMLFILE = "null_model.yaml" - EXP_NAME = YAMLFILE.split(".")[0] +# TEST_DIR = Path("fre/make/tests") +# NM_EXAMPLE = Path("null_example") +# YAMLFILE = "null_model.yaml" +# EXP_NAME = YAMLFILE.split(".")[0] # Combine model / experiment - YAMLS = "fre/make/tests/null_example/null_model.yaml,fre/make/tests/null_example/compile.yaml,fre/make/tests/null_example/platforms.yaml" - yml_dict = cy.yamltools_combine_subtool(YAMLS, None, None, None, output = None) + yamls = ("fre/make/tests/null_example/null_model.yaml," + "fre/make/tests/null_example/compile.yaml," + "fre/make/tests/null_example/platforms.yaml") + yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None) # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', @@ -208,22 +210,13 @@ def test_merged_pp_yamls(): Check for the creation of the combined-[experiment] yaml Check that the model yaml was merged into the combined yaml """ - # Model yaml path - modelyaml = Path(f"{IN_DIR}/model.yaml") - YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/pp_yamls/pp.c96_amip.yaml,{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml" - -# analysis: -# - "analysis_yamls/clouds.yaml" -# - "analysis_yamls/land.yaml" -# cmor: # cmor-tool development, not part of official am5.yaml (yet) -# - "cmor_yamls/cmor.am5.yaml" -# grid_yaml: -# - "grid_yamls/TEST_grids.yaml" - + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/pp_yamls/pp.c96_amip.yaml," + f"{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml") # Merge the yamls try: - cy.yamltools_combine_subtool(YAMLS, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) + cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) except: assert False @@ -231,16 +224,18 @@ def test_combined_ppyaml_validation(): """ Validate the combined compile yaml """ - modelyaml = Path(f"{IN_DIR}/model.yaml") - YAMLS = f"{IN_DIR}/model.yaml,{IN_DIR}/settings.yaml,{IN_DIR}/pp_yamls/pp.c96_amip.yaml,{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml" + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/settings.yaml," + f"{IN_DIR}/pp_yamls/pp.c96_amip.yaml," + f"{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml") # Merge the yamls try: - out = cy.yamltools_combine_subtool(YAMLS, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) + out = cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) except: assert False schema_file = os.path.join(SCHEMA_DIR, "fre_pp.json") - with open(schema_file,'r') as f: + with open(schema_file, 'r', encoding="utf-8") as f: s = f.read() schema = json.loads(s) @@ -292,9 +287,9 @@ def test_combine_pp_yamls(tmp_path): 'components' : { 'ocean': {'sources' : "a b c" }, 'ice': {'sources' : "ice_month"} - } + } } - } + } combined = { 'name' : 'expname', @@ -322,27 +317,28 @@ def test_combine_pp_yamls(tmp_path): tmp_path.mkdir(exist_ok=True) # create model and pp yamls - file_model = open(tmp_path / 'model.yaml', 'w') - file_settings = open(tmp_path / 'settings.yaml', 'w') - file_pp1 = open(tmp_path / 'pp1.yaml', 'w') - file_pp2 = open(tmp_path / 'pp2.yaml', 'w') + with open(tmp_path / 'model.yaml', 'w', encoding="utf-8") as model_yf: + yaml.dump(model, model_yf, default_flow_style=False, sort_keys=False) + + with open(tmp_path / 'settings.yaml', 'w', encoding="utf-8") as settings_yf: + yaml.dump(settings, settings_yf, default_flow_style=False, sort_keys=False) - # write to/ dump info into created model and pp yamls - yaml.dump(model, file_model, default_flow_style=False, sort_keys=False) - yaml.dump(settings, file_settings, default_flow_style=False, sort_keys=False) - yaml.dump(pp1, file_pp1, default_flow_style=False, sort_keys=False) - yaml.dump(pp2, file_pp2, default_flow_style=False, sort_keys=False) + with open(tmp_path / 'pp1.yaml', 'w', encoding="utf-8") as pp1_yf: + yaml.dump(pp1, pp1_yf, default_flow_style=False, sort_keys=False) + + with open(tmp_path / 'pp2.yaml', 'w', encoding="utf-8") as pp2_yf: + yaml.dump(pp2, pp2_yf, default_flow_style=False, sort_keys=False) # combine the yamls # output is a combined dictionary of necessary yaml info -# output = cy.consolidate_yamls(tmp_path / 'model.yaml', 'expname', 'platform', 'target', 'pp', output=None) - YAMLS = f"{tmp_path}/model.yaml,{tmp_path}/settings.yaml,{tmp_path}/pp1.yaml,{tmp_path}/pp2.yaml" + yamls = (f"{tmp_path}/model.yaml," + f"{tmp_path}/settings.yaml," + f"{tmp_path}/pp1.yaml,{tmp_path}/pp2.yaml") - output = cy.yamltools_combine_subtool(YAMLS, 'expname', 'platform', 'target', output = None) + output = cy.yamltools_combine_subtool(yamls, 'expname', 'platform', 'target', output = None) pp = pprint.PrettyPrinter(indent=4) pp.pprint(output) pp.pprint(combined) # compare dictionaries assert output == combined - From 01fa034c468f101a62f490c24e62b742dcfd2b43 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 21 Sep 2026 17:10:01 -0400 Subject: [PATCH 09/47] #874 more cleanup --- fre/yamltools/__init__.py | 8 -------- fre/yamltools/tests/test_combine_yamls_script.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/fre/yamltools/__init__.py b/fre/yamltools/__init__.py index 706c48cf4..e69de29bb 100644 --- a/fre/yamltools/__init__.py +++ b/fre/yamltools/__init__.py @@ -1,8 +0,0 @@ -#""" -#other bits in fre.yamltools use this with a `from . import *`. This way, the -#yaml module understands what to do with the `!join` references in our yaml files -#when they appear. -#""" -#import yaml -#from .constructors import _join_constructor -#yaml.add_constructor('!join', _join_constructor) diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 22910b5d6..16af8e6ed 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -11,7 +11,7 @@ import yaml from jsonschema import validate -from fre.yamltools import combine_yamls_script_new as cy #import combine_yamls_script_new as cy +from fre.yamltools import combine_yamls_script_new as cy ## SET-UP From 428cd6425bfee998745bf2168d09dcb234e38ab3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 22 Sep 2026 17:21:59 -0400 Subject: [PATCH 10/47] #874 slight cleanup for fre list yamls - suppress output for when fre list yamls is not piped to fre yamltools combine - switch combining order for platforms and compile (so users can define reusable variables in platforms) --- fre/list_/list_yamls_script.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fre/list_/list_yamls_script.py b/fre/list_/list_yamls_script.py index 3126634c0..ed2078572 100644 --- a/fre/list_/list_yamls_script.py +++ b/fre/list_/list_yamls_script.py @@ -11,6 +11,7 @@ """ import logging from pathlib import Path +import sys import yaml import click @@ -43,8 +44,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): with open(yamlfile, 'r', encoding="utf-8") as yf: yaml_dict = yaml.load(yf, Loader = yaml.Loader) - compile_data = yaml_dict["build"].get("compileYaml") platform_data = yaml_dict["build"].get("platformYaml") + compile_data = yaml_dict["build"].get("compileYaml") exp_data = yaml_dict["experiments"].get(experiment) yamls = [model_yaml] @@ -71,7 +72,7 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): else: yamls.append(exp_data[a]) else: - yamls.extend([compile_data, platform_data]) + yamls.extend([platform_data, compile_data]) for value in exp_data.values(): if isinstance(value, list): yamls.extend(value) @@ -79,7 +80,7 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): yamls.append(value) else: fre_logger.info("No experiment name passed. Will only provide YAMLs related to compilation.") - yamls.extend([compile_data, platform_data]) + yamls.extend([platform_data, compile_data]) yamls_full_path = "" # Add full path for yaml configurations @@ -116,5 +117,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str): if "True" in fail: raise ValueError(" *** PROVIDE THE MISSING YAML CONFIGURATIONS ***") - click.echo(yamls_full_path) + # if not piped to fre yamltools combine, no need to show all the yamls + if not sys.stdout.isatty(): + click.echo(yamls_full_path) + return yamls_full_path From a978de4bf60f9332cb4bca448bdfff5bdbbed698 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 22 Sep 2026 17:26:00 -0400 Subject: [PATCH 11/47] #874 update logging usage, comments, and documentation --- fre/yamltools/combine_yamls_script_new.py | 44 ++++++++++++----------- fre/yamltools/freyamltools.py | 8 +++-- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index aaea2f3ae..7238cda1e 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -24,28 +24,20 @@ from uwtools.api import config from uwtools.api.logging import use_uwtools_logger -#from fre.list_.list_yamls_script import list_yamls_subtool from fre.yamltools.helpers import output_yaml, clean_yaml#, check_fre_version fre_logger = logging.getLogger(__name__) class LetsGo(): """ - :ivar: ?? + :ivar str yamls: is the list of YAML configuration files to combine + :ivar str experiment: is the experiment name (relates to the run and postprocessing) + :ivar str platform: is the FRE platform defined + :ivar str target: is the predefined FRE targets; options include [prod/debug/repro]-openmp + :ivar str output: is the file path to a file that will incude the final combined, resolved YAML + configuration file """ def __init__(self, yamls, experiment, platform, target, output): - """ - :param yamls: - :type yamls: - :param experiment: - :type experiment: - :param platform: - :type platform: - :param target: - :type target: - :param output: - :type output: - """ self.y = yamls.split(",") self.e = str(experiment) self.p = str(platform) @@ -109,12 +101,23 @@ def use_uwtools(self, yaml_list): ### SHOLD BE RESOLVED BUT THIS IS TO CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## ## uw config realize: resolve final yaml # get nice uw tools cli output - use_uwtools_logger() + + # Save current root logger handlers and level + root_logger = logging.getLogger() + original_handlers = list(root_logger.handlers) + original_level = root_logger.level + + + uwlogger = use_uwtools_logger() config.realize(input_config = cleaned_yaml_dict, values_needed = True, total = True) + # 3. Restore root logger state back to original + root_logger.handlers = original_handlers + root_logger.setLevel(original_level) + if self.o: out_path = Path.cwd()/self.o fre_logger.info("Writing resolved YAML file: %s", out_path) @@ -154,15 +157,16 @@ def use_uwtools(self, yaml_list): def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None) -> dict: """ - :param yamls: + :param yamls: is the list of YAML configuration files to combine :type yamls: str - :param experiment: + :param experiment: is the experiment name (relates to the run and postprocessing) :type experiment: str - :param platform: + :param platform: is the FRE platform defined :type platform: str - :param target: + :param target: is the predefined FRE targets; options include [prod/debug/repro]-openmp :type target: str - :param output: + :param output: is the file path to a file that will incude the final combined, resolved YAML + configuration file :type output: str """ # fre_logger.info('checking fre_cli_version compatibility...') diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index 9179c0099..bdefbfa1a 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -1,9 +1,11 @@ ''' click entry-point to 'fre yamltools' calls''' import sys -import ast +import logging +#import ast import click from fre.yamltools import combine_yamls_script from fre.yamltools import combine_yamls_script_new +fre_logger = logging.getLogger(__name__) @click.group(help=click.style(" - yamltools subcommands", fg=(202,177,95))) def yamltools_cli(): @@ -56,11 +58,11 @@ def combine(yamls, experiment, platform, target, output): # if not piped from fre list yamls yaml_paths = [] if yamls: - print("not piped") + fre_logger.info("NOT PIPED") yaml_paths = yamls # print(yamls.split(",")) else: - print("piped") + fre_logger.info("PIPED") # stdout from last tool adds a newline for some reason yaml_paths = sys.stdin.read().strip() From 80a54a04f13170e06766ace5f3d335c43a1916b7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 23 Sep 2026 11:45:47 -0400 Subject: [PATCH 12/47] #874 add uwtools dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b3f0f617c..568f3106e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ 'numpy', 'pyyaml', 'xarray', + 'uwtools', ] classifiers = [ From c108b550eb15e72bdea9cc22df10b216926746f8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 23 Sep 2026 12:03:33 -0400 Subject: [PATCH 13/47] #874 update readme --- fre/yamltools/README.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/fre/yamltools/README.md b/fre/yamltools/README.md index f6d896f35..0397ceccf 100644 --- a/fre/yamltools/README.md +++ b/fre/yamltools/README.md @@ -1,6 +1,22 @@ ## FRE yamltools `fre yamltools` provides subtools that help to manage and perform operations on yaml files. +## Quickstart + +To access YAML examples, clone the fre-cli repository: + + git clone --recursive https://github.com/NOAA-GFDL/fre-cli.git + + cd fre/yamltools/tests + +### Independent subtool use + + fre yamltools combine -y "yamls/model.yaml,yamls/compile_yamls/platforms.yaml,yamls/compile_yamls/compile.yaml" -o out.yaml + +### Piped from `fre list` + + fre list yamls -y yamls/model.yaml | fre -v yamltools combine -o out.yaml + ## Subtools - `fre yamltools combine-yamls [options]` - Purpose: @@ -13,12 +29,12 @@ - `-e, --experiment [experiment name]` - `--use [compile|pp] (required)` -### **Tests** - -To run `fre yamltools` test scripts, return to root directory of the fre-cli repo and call those tests with - - python -m pytest fre/yamltools/tests/[test script.py] - -Or run all tests with - - python -m pytest fre/yamltools/tests +- `fre yamltools combine [options]` + - Purpose: + - Creates combined and resolved YAML file using the YAML configurations passed. + - Options: + - `-y, --yamlfile [experiment yaml] (required)` + - `-e, --experiment [experiment name]` + - `-p, --platform [platform]` + - `-t, --target [target]` + - `-o, --output [path to output file]` From 43c4f9abb7f6178dd6620b8896692e1b9871700f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 23 Sep 2026 12:17:35 -0400 Subject: [PATCH 14/47] #874 update tests and address a comment --- fre/tests/test_fre_yamltools_cli.py | 6 +++--- fre/yamltools/combine_yamls_script_new.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fre/tests/test_fre_yamltools_cli.py b/fre/tests/test_fre_yamltools_cli.py index 30855ced5..fe4120258 100644 --- a/fre/tests/test_fre_yamltools_cli.py +++ b/fre/tests/test_fre_yamltools_cli.py @@ -40,16 +40,16 @@ def test_cli_fre_yamltools_opt_dne(): def test_cli_fre_yamltools_combine_help(): ''' fre yamltools ''' - result = runner.invoke(fre.fre, args=["yamltools", "combine-yamls"]) + result = runner.invoke(fre.fre, args=["yamltools", "combine"]) assert result.exit_code == 0 def test_cli_fre_yamltools_combine_help(): ''' fre yamltools ''' - result = runner.invoke(fre.fre, args=["yamltools", "combine-yamls", "--help"]) + result = runner.invoke(fre.fre, args=["yamltools", "combine", "--help"]) assert result.exit_code == 0 def test_cli_fre_yamltools_combine_opt_dne(): ''' fre yamltools ''' - result = runner.invoke(fre.fre, args=["yamltools", "combine-yamls", "optionDNE"]) + result = runner.invoke(fre.fre, args=["yamltools", "combine", "optionDNE"]) assert result.exit_code == 2 diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index 7238cda1e..82410cc07 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -42,7 +42,7 @@ def __init__(self, yamls, experiment, platform, target, output): self.e = str(experiment) self.p = str(platform) self.t = str(target) - self.o = str(output) + self.o = output def list_check_yamls(self, init_file): """ From d9c3d93bae679cc0e36c206c0d7440c62eac3a75 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 23 Sep 2026 12:23:07 -0400 Subject: [PATCH 15/47] #874 TEMPORARY - add back in so all tests don't fail - updates to other tests will ideally be done in PRs after this is in --- fre/yamltools/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fre/yamltools/__init__.py b/fre/yamltools/__init__.py index e69de29bb..c2e7ac2d8 100644 --- a/fre/yamltools/__init__.py +++ b/fre/yamltools/__init__.py @@ -0,0 +1,8 @@ +""" +other bits in fre.yamltools use this with a `from . import *`. This way, the +yaml module understands what to do with the `!join` references in our yaml files +when they appear. +""" +import yaml +from .constructors import _join_constructor +yaml.add_constructor('!join', _join_constructor) From e15243e2460f3c1f911745b65a6686679f2f43f9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 23 Sep 2026 13:07:47 -0400 Subject: [PATCH 16/47] #874 add test for piping subtools --- fre/tests/test_fre_yamltools_cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fre/tests/test_fre_yamltools_cli.py b/fre/tests/test_fre_yamltools_cli.py index fe4120258..ae86b0b64 100644 --- a/fre/tests/test_fre_yamltools_cli.py +++ b/fre/tests/test_fre_yamltools_cli.py @@ -53,3 +53,11 @@ def test_cli_fre_yamltools_combine_opt_dne(): result = runner.invoke(fre.fre, args=["yamltools", "combine", "optionDNE"]) assert result.exit_code == 2 +def test_cli_fre_yamltools_piped_from_list(caplog): + ''' fre list yamls -y [model.yaml] | fre yamtools combine ''' + frelist_out = runner.invoke(fre.fre, args=["list", "yamls", "-y", "fre/yamltools/tests/yamls/model.yaml"]) + assert frelist_out.exit_code == 0 + + freyamltools_pipe = runner.invoke(fre.fre, args=["-v", "yamltools", "combine"], input=frelist_out.output) + assert freyamltools_pipe.exit_code == 0 + assert "PIPED" in caplog.text From 53be9e3a16fe966a42df688d8c239e5f5976802e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 23 Sep 2026 13:12:14 -0400 Subject: [PATCH 17/47] #874 update --- fre/yamltools/freyamltools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index bdefbfa1a..bbb867000 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -38,7 +38,7 @@ def combine_yamls(yamlfile, @yamltools_cli.command() @click.option("-y", "--yamls", type=str, - help="YAML file to be used for parsing") + help="Comma separated list of YAML files to be combined") @click.option("-e", "--experiment", type=str, help="Experiment name") From fe7c43379db4e1c3089bf6cf5935e97445229184 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 23 Sep 2026 13:13:45 -0400 Subject: [PATCH 18/47] #874 update --- fre/yamltools/freyamltools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index bbb867000..aab992354 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -38,7 +38,7 @@ def combine_yamls(yamlfile, @yamltools_cli.command() @click.option("-y", "--yamls", type=str, - help="Comma separated list of YAML files to be combined") + help="String of comma separated list of YAML files to be combined") @click.option("-e", "--experiment", type=str, help="Experiment name") From e0618726f342f1e63a69ec2ea3e0a4c6704061a6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 29 Sep 2026 11:37:01 -0400 Subject: [PATCH 19/47] #874 undo these - changes in a different PR --- environment.yml | 2 -- meta.yaml | 1 - pyproject.toml | 1 - 3 files changed, 4 deletions(-) diff --git a/environment.yml b/environment.yml index 1c51a94d3..d4e5d35bc 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,6 @@ name: fre-cli channels: - conda-forge - noaa-gfdl - - ufs-community dependencies: - python>=3.11 - pip @@ -28,4 +27,3 @@ dependencies: - conda-forge::pyyaml - conda-forge::xarray>=2024.* - conda-forge::netcdf4>=1.7.* - - ufs-community::uwtools==2.20.0 diff --git a/meta.yaml b/meta.yaml index 8406d79f8..f9f5f62be 100644 --- a/meta.yaml +++ b/meta.yaml @@ -38,7 +38,6 @@ requirements: - conda-forge::pyyaml - conda-forge::xarray>=2024.* - conda-forge::netcdf4>=1.7.* - - ufs-community::uwtools==2.20.0 test: source_files: diff --git a/pyproject.toml b/pyproject.toml index 568f3106e..b3f0f617c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ 'numpy', 'pyyaml', 'xarray', - 'uwtools', ] classifiers = [ From 4e47ba7070b51ce40e70b31a0ee2b0db0fbbef74 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 10:21:47 -0400 Subject: [PATCH 20/47] #874 update submodule --- fre/gfdl_msd_schemas | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/gfdl_msd_schemas b/fre/gfdl_msd_schemas index 33a5412b2..837e5e777 160000 --- a/fre/gfdl_msd_schemas +++ b/fre/gfdl_msd_schemas @@ -1 +1 @@ -Subproject commit 33a5412b2354adde4a7832eb81d4b8cbd965ffcb +Subproject commit 837e5e777319c6a497abf1012100688e0ceb480c From ccc5a729a8e9e6a9f832f39a9c7c28a5cdd2eebe Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 11:55:37 -0400 Subject: [PATCH 21/47] #874 update yaml format for updated schema --- fre/make/tests/null_example/null_model.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fre/make/tests/null_example/null_model.yaml b/fre/make/tests/null_example/null_model.yaml index a5a06d523..2834341c6 100644 --- a/fre/make/tests/null_example/null_model.yaml +++ b/fre/make/tests/null_example/null_model.yaml @@ -1,23 +1,23 @@ # reusable variables fre_properties: - - &VERSION "full" - - &FRE_STEM !join [null_model/, *VERSION] + VERSION: &VERSION "full" + FRE_STEM: &FRE_STEM !join [null_model/, *VERSION] # compile information - - &branch "main" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/include" - - &VOLUME "/gpfs/f5" + BRANCH: &branch "main" + INTEL: &INTEL "intel-classic" + FMSincludes: &FMSincludes "-IFMS/include" + VOLUME: &VOLUME "/gpfs/f5" build: compileYaml: "compile.yaml" platformYaml: "platforms.yaml" experiments: - - name: "null_model_full" + null_model_full: pp: - - name: "null_model_0" + null_model_0: pp: - - name: "null_model_1" + null_model_1: pp: - - name: "null_model_2" + null_model_2: pp: From 18984e06918c1ec2eac7b3d8ae127711b422cc07 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 13:19:31 -0400 Subject: [PATCH 22/47] update fre pp configure-yaml and tests for uwtools integration --- fre/pp/configure_script_yaml.py | 21 +++-- fre/pp/tests/AM5_example/am5.yaml | 54 ------------ .../AM5_example/yaml_include/pp.c96_amip.yaml | 87 ------------------- fre/pp/tests/test_configure_script_yaml.py | 61 ++++++++----- fre/pp/tests/yamls/model.yaml | 56 ++++++++++++ .../tests/yamls/yaml_include/platforms.yaml | 12 +++ .../yaml_include/pp-test.c96_amip.yaml | 15 ++-- .../tests/yamls/yaml_include/pp.c96_amip.yaml | 54 ++++++++++++ .../yaml_include/settings.yaml | 20 ++--- 9 files changed, 195 insertions(+), 185 deletions(-) delete mode 100644 fre/pp/tests/AM5_example/am5.yaml delete mode 100644 fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml create mode 100644 fre/pp/tests/yamls/model.yaml create mode 100644 fre/pp/tests/yamls/yaml_include/platforms.yaml rename fre/pp/tests/{AM5_example => yamls}/yaml_include/pp-test.c96_amip.yaml (73%) create mode 100644 fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml rename fre/pp/tests/{AM5_example => yamls}/yaml_include/settings.yaml (51%) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 0b1c77286..97327f69a 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -11,7 +11,9 @@ from jsonschema import validate, SchemaError, ValidationError import metomi.rose.config -import fre.yamltools.combine_yamls_script as cy +#import fre.yamltools.combine_yamls_script as cy +import fre.yamltools.combine_yamls_script_new as cy +import fre.list_.list_yamls_script as ly fre_logger = logging.getLogger(__name__) @@ -265,10 +267,19 @@ def yaml_info(yamlfile: str = None, experiment: str = None, platform: str = None # Combine model, experiment, and analysis yamls cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") outfile = os.path.join(cylc_dir, f"{e}.yaml") - full_yamldict = cy.consolidate_yamls(yamlfile = yml, - experiment = e, platform = p, target = t, - use = "pp", - output = outfile) + +# full_yamldict = cy.consolidate_yamls(yamlfile = yml, +# experiment = e, platform = p, target = t, +# use = "pp", +# output = outfile) + yamls = ly.list_yamls_subtool(yamlfile = yml, + experiment = e, + application= "postprocess") + full_yamldict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = e, + platform = p, + target = t, + output = outfile) # Validate yaml validate_yaml(full_yamldict) diff --git a/fre/pp/tests/AM5_example/am5.yaml b/fre/pp/tests/AM5_example/am5.yaml deleted file mode 100644 index 24d365b25..000000000 --- a/fre/pp/tests/AM5_example/am5.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "19790101T0000Z" - - &EXP_AMIP_END "20200101T0000Z" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "P1Y" - - &PP_AMIP_CHUNK384 "P1Y" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "P5Y" - - &PP_CPLD_CHUNK_B "P20Y" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -# compile information -build: - compileYaml: "compile.yaml" - platformYaml: "yaml_include/platforms.yaml" - -experiments: - - name: "c96L65_am5f7b12r1_amip" - settings: "yaml_include/settings.yaml" - pp: - - "yaml_include/pp.c96_amip.yaml" - - "yaml_include/pp-test.c96_amip.yaml" - - name: "c96L65_am5f7b12r1_pdclim1850F" - pp: - - "yaml_include/pp.c96_clim.yaml" diff --git a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml deleted file mode 100644 index 3859217bd..000000000 --- a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# local reusable variable overrides -fre_properties: - - &custom_interp "180,360" - -#c96_amip_postprocess: -postprocess: - # main pp instructions - components: - - type: "atmos_cmip" - sources: - - history_file: "atmos_month_cmip" - - history_file: "atmos_8xdaily_cmip" - - history_file: "atmos_daily_cmip" - sourceGrid: "cubedsphere" - xyInterp: *custom_interp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos" - sources: - - history_file: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_level_cmip" - sources: - - history_file: "atmos_level_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_level" - sources: - - history_file: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_month_aer" - sources: - - history_file: "atmos_month_aer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - - type: "atmos_diurnal" - sources: - - history_file: "atmos_diurnal" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_scalar" - sources: - - history_file: "atmos_scalar" - - type: "aerosol_cmip" - xyInterp: *PP_XYINTERP96 - sources: - - history_file: "aerosol_month_cmip" - sourceGrid: "cubedsphere" - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False - - type: "land" - sources: - - history_file: "land_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "land_cmip" - sources: - - history_file: "land_month_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - postprocess_on: False - - type: "tracer_level" - sources: - - history_file: "atmos_tracer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - postprocess_on: False diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index 870e727b2..01535280c 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -16,17 +16,18 @@ import metomi.rose.config from fre.pp import configure_script_yaml as csy -from fre.yamltools import combine_yamls_script as cy - +#from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +import fre.list_.list_yamls_script as ly # Set what would be click options -EXPERIMENT = "c96L65_am5f7b12r1_amip" +EXPERIMENT = "experiment_1" PLATFORM = "gfdl.ncrc5-intel22-classic" TARGET = "prod-openmp" # Set example yaml paths, input directory TEST_DIR = Path("fre/pp/tests") -TEST_YAML = Path("AM5_example/am5.yaml") +TEST_YAML = Path("yamls/model.yaml") def test_combinedyaml_exists(): """ @@ -34,15 +35,17 @@ def test_combinedyaml_exists(): """ assert Path(f"{TEST_DIR}/{TEST_YAML}").exists() -def test_configure_script(): +def test_configure_script(monkeypatch): """ Tests success of configure yaml script Creates rose-suite, regrid rose-app, remap rose-app TO-DO: will break this up for better tests """ # Set home for ~/cylc-src location in script - old_home = os.environ["HOME"] - os.environ["HOME"] = str(Path(f"{TEST_DIR}/configure_yaml_out")) + monkeypatch.setenv("HOME", str(Path(f"{TEST_DIR}/configure_yaml_out"))) + +# old_home = os.environ["HOME"] +# os.environ["HOME"] = str(Path(f"{TEST_DIR}/configure_yaml_out")) # Set output directory OUT_DIR = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}") @@ -54,7 +57,7 @@ def test_configure_script(): # Invoke configure_yaml_script.py csy.yaml_info(model_yaml, EXPERIMENT, PLATFORM, TARGET) - os.environ["HOME"] = old_home +# os.environ["HOME"] = old_home # Check for configuration creation and final combined yaml assert all([ Path(f"{OUT_DIR}/{EXPERIMENT}.yaml").exists(), @@ -64,12 +67,21 @@ def test_validate(): """ Test the success of validation. """ - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", - experiment = EXPERIMENT, - platform = PLATFORM, - target = TARGET, - use = "pp", - output = None) +# yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", +# experiment = EXPERIMENT, +# platform = PLATFORM, +# target = TARGET, +# use = "pp", +# output = None) + yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", + experiment = EXPERIMENT, + application= "postprocess") + yml_dict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = EXPERIMENT, + platform = PLATFORM, + target = TARGET, + output = None) + try: csy.validate_yaml(yml_dict) except: @@ -79,12 +91,21 @@ def test_validate_fail(): """ Test that validation fails when given the wrong yaml dictionary. """ - yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", - experiment = EXPERIMENT, - platform = PLATFORM, - target = TARGET, - use = "pp", - output = f"{Path(__file__).parent}/csy_out.yaml") +# yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", +# experiment = EXPERIMENT, +# platform = PLATFORM, +# target = TARGET, +# use = "pp", +# output = f"{Path(__file__).parent}/csy_out.yaml") + yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", + experiment = EXPERIMENT, + application= "postprocess") + yml_dict = cy.yamltools_combine_subtool(yamls = yamls, + experiment = EXPERIMENT, + platform = PLATFORM, + target = TARGET, + output = f"{Path(__file__).parent}/csy_out.yaml") + # Missing history_dir, ptmp_dir, and postprocess wrong_yml_dict = { diff --git a/fre/pp/tests/yamls/model.yaml b/fre/pp/tests/yamls/model.yaml new file mode 100644 index 000000000..4ea595842 --- /dev/null +++ b/fre/pp/tests/yamls/model.yaml @@ -0,0 +1,56 @@ +# reusable variables +fre_properties: + AM5_VERSION: "am5f7b12r1" + FRE_STEM: "am5/{{ fre_properties.AM5_VERSION }}" + + # amip + EXP_AMIP_START: "19790101T0000Z" + EXP_AMIP_END: "20200101T0000Z" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" + + PP_AMIP_CHUNK96: "P1Y" + PP_AMIP_CHUNK384: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" + + # climo + EXP_CLIMO_START96: "0001" + EXP_CLIMO_END96: "0011" + ANA_CLIMO_START96: "0002" + ANA_CLIMO_END96: "0011" + + EXP_CLIMO_START384: "0001" + EXP_CLIMO_END384: "0006" + ANA_CLIMO_START384: "0002" + ANA_CLIMO_END384: "0006" + + # coupled + PP_CPLD_CHUNK_A: "P5Y" + PP_CPLD_CHUNK_B: "P20Y" + + # grids + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + RELEASE: "f1a1r1" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: +# c96L65_am5f7b12r1_amip: + experiment_1: + settings: "yaml_include/settings.yaml" + postprocess: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" +# c96L65_am5f7b12r1_pdclim1850F: + experiment_2: + postprocess: + - "yaml_include/pp.c96_clim.yaml" diff --git a/fre/pp/tests/yamls/yaml_include/platforms.yaml b/fre/pp/tests/yamls/yaml_include/platforms.yaml new file mode 100644 index 000000000..01689dab7 --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/platforms.yaml @@ -0,0 +1,12 @@ +platforms: + - name: hpcme.intel25 + compiler: intel + RUNenv: "" + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3" + mkTemplate: "/apps/mkmf/templates/hpcme-intel2025.03-oneapi.mk" + container2step: True + container2base: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3rte" diff --git a/fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml similarity index 73% rename from fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml rename to fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml index 76f078523..25412100d 100644 --- a/fre/pp/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml +++ b/fre/pp/tests/yamls/yaml_include/pp-test.c96_amip.yaml @@ -1,33 +1,32 @@ # local reusable variable overrides fre_properties: - - &custom_interp "200,200" + custom_interp_test: "200,200" -#c96_amip_postprocess: postprocess: components: - - type: "atmos_cmip-TEST" + atmos_cmip-TEST: sources: - history_file: "atmos_month_cmip" - history_file: "atmos_8xdaily_cmip" - history_file: "atmos_daily_cmip" sourceGrid: "cubedsphere" - xyInterp: *custom_interp + xyInterp: "{{ fre_properties.custom_interp_test }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos-TEST" + atmos-TEST: sources: - history_file: "atmos_month" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False - - type: "atmos_level_cmip-TEST" + atmos_level_cmip-TEST: sources: - history_file: "atmos_level_cmip" sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" interpMethod: "conserve_order2" inputRealm: 'atmos' postprocess_on: False diff --git a/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..b5e1b7a8a --- /dev/null +++ b/fre/pp/tests/yamls/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,54 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "180,360" + +postprocess: + # main pp instructions + components: + atmos_cmip: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.custom_interp }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_level_cmip: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_level: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_month_aer: + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + atmos_diurnal: + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: "{{ fre_properties.PP_XYINTERP96 }}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_scalar: + sources: + - history_file: "atmos_scalar" diff --git a/fre/pp/tests/AM5_example/yaml_include/settings.yaml b/fre/pp/tests/yamls/yaml_include/settings.yaml similarity index 51% rename from fre/pp/tests/AM5_example/yaml_include/settings.yaml rename to fre/pp/tests/yamls/yaml_include/settings.yaml index 1057fc6d5..9ccd6a755 100644 --- a/fre/pp/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/pp/tests/yamls/yaml_include/settings.yaml @@ -1,22 +1,20 @@ -#c96_amip_directories: directories: - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - refined_history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history_dir_refined] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + refined_history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history_dir_refined" + pp_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "/nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" ptmp_dir: "/xtmp/$USER/ptmp" -#c96_amip_postprocess: postprocess: settings: history_segment: "P1Y" site: "ppan" - pp_start: *ANA_AMIP_START - pp_stop: *ANA_AMIP_END + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" pp_chunks: - - *PP_AMIP_CHUNK96 - pp_grid_spec: *GRID_SPEC96 - switches: &shared_switches + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: clean_work: True do_atmos_plevel_masking: True preanalysis: From 3bd89e4078bd79a3905cfe3fcd7e991db91ee66c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 13:22:38 -0400 Subject: [PATCH 23/47] clean up --- fre/pp/configure_script_yaml.py | 5 ----- fre/pp/tests/test_configure_script_yaml.py | 18 ------------------ 2 files changed, 23 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 97327f69a..7844eec50 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -11,7 +11,6 @@ from jsonschema import validate, SchemaError, ValidationError import metomi.rose.config -#import fre.yamltools.combine_yamls_script as cy import fre.yamltools.combine_yamls_script_new as cy import fre.list_.list_yamls_script as ly @@ -268,10 +267,6 @@ def yaml_info(yamlfile: str = None, experiment: str = None, platform: str = None cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") outfile = os.path.join(cylc_dir, f"{e}.yaml") -# full_yamldict = cy.consolidate_yamls(yamlfile = yml, -# experiment = e, platform = p, target = t, -# use = "pp", -# output = outfile) yamls = ly.list_yamls_subtool(yamlfile = yml, experiment = e, application= "postprocess") diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index 01535280c..20269ffef 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -16,7 +16,6 @@ import metomi.rose.config from fre.pp import configure_script_yaml as csy -#from fre.yamltools import combine_yamls_script as cy from fre.yamltools import combine_yamls_script_new as cy import fre.list_.list_yamls_script as ly @@ -44,9 +43,6 @@ def test_configure_script(monkeypatch): # Set home for ~/cylc-src location in script monkeypatch.setenv("HOME", str(Path(f"{TEST_DIR}/configure_yaml_out"))) -# old_home = os.environ["HOME"] -# os.environ["HOME"] = str(Path(f"{TEST_DIR}/configure_yaml_out")) - # Set output directory OUT_DIR = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}") Path(OUT_DIR).mkdir(parents = True, exist_ok = True) @@ -57,8 +53,6 @@ def test_configure_script(monkeypatch): # Invoke configure_yaml_script.py csy.yaml_info(model_yaml, EXPERIMENT, PLATFORM, TARGET) -# os.environ["HOME"] = old_home - # Check for configuration creation and final combined yaml assert all([ Path(f"{OUT_DIR}/{EXPERIMENT}.yaml").exists(), Path(f"{OUT_DIR}/rose-suite.conf").exists()]) @@ -67,12 +61,6 @@ def test_validate(): """ Test the success of validation. """ -# yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", -# experiment = EXPERIMENT, -# platform = PLATFORM, -# target = TARGET, -# use = "pp", -# output = None) yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", experiment = EXPERIMENT, application= "postprocess") @@ -91,12 +79,6 @@ def test_validate_fail(): """ Test that validation fails when given the wrong yaml dictionary. """ -# yml_dict = cy.consolidate_yamls(yamlfile = f"{TEST_DIR}/{TEST_YAML}", -# experiment = EXPERIMENT, -# platform = PLATFORM, -# target = TARGET, -# use = "pp", -# output = f"{Path(__file__).parent}/csy_out.yaml") yamls = ly.list_yamls_subtool(yamlfile = f"{TEST_DIR}/{TEST_YAML}", experiment = EXPERIMENT, application= "postprocess") From 5d815969de25e6459eaa9156cb266dfe3ce3470c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 17:25:11 -0400 Subject: [PATCH 24/47] update fre list subtools to use tool with uwtools --- fre/list_/frelist.py | 9 +++- fre/list_/list_experiments_script.py | 51 ++++++----------------- fre/list_/list_platforms_script.py | 38 ++++++++++++----- fre/list_/list_pp_components_script.py | 45 +++++++++++++------- fre/yamltools/combine_yamls_script_new.py | 30 +++++++------ fre/yamltools/freyamltools.py | 6 ++- 6 files changed, 98 insertions(+), 81 deletions(-) diff --git a/fre/list_/frelist.py b/fre/list_/frelist.py index afad49a07..bc1f65756 100644 --- a/fre/list_/frelist.py +++ b/fre/list_/frelist.py @@ -44,9 +44,14 @@ def platforms(yamlfile): type=str, help="Experiment to be post-processed", required=True) -def pp_components(yamlfile, experiment): +@click.option("-a", + "--application", + type=str, + help="Post-processing application name.", + required=True) +def pp_components(yamlfile, experiment, application): """ - List components to be post-processed for a defined experiment""" - list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment) + list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment, application) @list_cli.command() @click.option("-y", diff --git a/fre/list_/list_experiments_script.py b/fre/list_/list_experiments_script.py index ecf205868..237c6cd75 100644 --- a/fre/list_/list_experiments_script.py +++ b/fre/list_/list_experiments_script.py @@ -5,15 +5,7 @@ """ import logging -import yaml - -from fre.yamltools.info_parsers import pp_info_parser as ppip - -## to-do: figure out validation -# Imports associated with commented block -# from pathlib import Path -# from fre.yamltools import helpers -# from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy fre_logger = logging.getLogger(__name__) @@ -29,34 +21,15 @@ def list_experiments_subtool(yamlfile: str): platform = None target = None - # Combine model - # Create pp yaml instance - yamldict = ppip.InitPPYaml(yamlfile, exp, platform, target) - yaml_str = yamldict.combine_model() - yaml_dict = yaml.load(yaml_str, Loader = yaml.Loader) + yaml_dict = cy.yamltools_combine_subtool(yamls = yamlfile, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = True) -## COULD HAVE been one way to validate but section we'd want to parse was -## cleaned in final/"combined" yaml information -## Currently not a way to validate model yaml information because we only -## have schemas for the final "combined" compile or pp information (both -## of which remove the "experiments" section I believe -# exp = yamlfile.split("/")[-1].split(".")[0] -# platform = "None" -# target = "None" -# -# # Combine model / experiment -# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, -# experiment = exp, -# platform = platform, -# target = target, -# use = "compile", -# output = None) -# -# # Validate combined yaml information -# frelist_dir = Path(__file__).resolve().parents[2] -# schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json" -# # from fre.yamltools -# helpers.validate_yaml(yml_dict, schema_path) +#### TO-DO: CURRENTLY NO YAML PIECE VALIDATION +### No way to validate just the model yaml # set logger level to INFO former_log_level = fre_logger.level @@ -64,9 +37,9 @@ def list_experiments_subtool(yamlfile: str): # log the experiment names, which should show up on screen for sure fre_logger.info("Experiments found:") - for i in yaml_dict.get("experiments"): - fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") + + for i in yaml_dict["experiments"].keys(): + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_platforms_script.py b/fre/list_/list_platforms_script.py index 2ebc0a0e5..7d36d3703 100644 --- a/fre/list_/list_platforms_script.py +++ b/fre/list_/list_platforms_script.py @@ -6,7 +6,8 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) @@ -22,17 +23,33 @@ def list_platforms_subtool(yamlfile: str): former_log_level = fre_logger.level fre_logger.setLevel(logging.INFO) - exp = yamlfile.split("/")[-1].split(".")[0] + exp = None platform = None target = None - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "compile", - output = None) +# model_yf_path = Path(yamlfile).resolve().parent +# with open(yamlfile, 'r') as yf: +# yml = yaml.safe_load(yf) +# +# platform_yf_path = f"{model_yf_path}/{yml['build']['platformYaml']}" + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = None) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "compile", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) # Validate the yaml fre_pkg_dir = Path(__file__).resolve().parents[1] @@ -41,8 +58,7 @@ def list_platforms_subtool(yamlfile: str): helpers.validate_yaml(yml_dict, schema_path) fre_logger.info("Platforms available:") - for i in yml_dict.get("platforms"): + for i in yml_dict["platforms"]: fre_logger.info(' - %s', i.get("name")) - fre_logger.info("\n") fre_logger.setLevel(former_log_level) diff --git a/fre/list_/list_pp_components_script.py b/fre/list_/list_pp_components_script.py index 0be6128c7..5f475c5fb 100644 --- a/fre/list_/list_pp_components_script.py +++ b/fre/list_/list_pp_components_script.py @@ -19,12 +19,13 @@ from pathlib import Path import logging -from fre.yamltools import combine_yamls_script as cy +from fre.yamltools import combine_yamls_script_new as cy +from fre.list_ import list_yamls_script as ly from fre.yamltools import helpers fre_logger = logging.getLogger(__name__) -def list_ppcomps_subtool(yamlfile: str, experiment: str): +def list_ppcomps_subtool(yamlfile: str, experiment: str, application: str): """ List_ppcomps_subtool lists the components to be post-processed. @@ -32,6 +33,8 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): :type yamlfile: str :param experiment: is the experiment name defined in the model.yaml :type experiment: str + :param application: + :type applicatin: str """ # set logger level to INFO former_log_level = fre_logger.level @@ -40,14 +43,26 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): exp = experiment platform = None target = None + app = application + + yml_list = ly.list_yamls_subtool(yamlfile = yamlfile, + experiment = exp, + application = app) + +# # Combine model / experiment +# yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, +# experiment = exp, +# platform = platform, +# target = target, +# use = "pp", +# output = None) + yml_dict = cy.yamltools_combine_subtool(yamls = yml_list, + experiment = exp, + platform = platform, + target = target, + output = None, + no_clean = False) - # Combine model / experiment - yml_dict = cy.consolidate_yamls(yamlfile = yamlfile, - experiment = exp, - platform = platform, - target = target, - use = "pp", - output = None) # Validate combined yaml information frelist_dir = Path(__file__).resolve().parents[2] @@ -55,15 +70,15 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): # from fre.yamltools helpers.validate_yaml(yml_dict, schema_path) + comp_info = yml_dict["postprocess"]["components"] # log the experiment names, which should show up on screen for sure fre_logger.info("Components to be post-processed:") - for i in yml_dict["postprocess"]["components"]: - if "postprocess_on" in i: - if i.get("postprocess_on") is True: - fre_logger.info(' - %s', i.get("type")) + for i in comp_info.keys(): + if "postprocess_on" in comp_info[i]: + if comp_info[i].get("postprocess_on") is True: + fre_logger.info(' - %s', i) else: - fre_logger.info(' - %s', i.get("type")) - fre_logger.info("\n") + fre_logger.info(' - %s', i) # set logger back to normal level fre_logger.setLevel(former_log_level) diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index 82410cc07..7ce5f13a9 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -37,12 +37,13 @@ class LetsGo(): :ivar str output: is the file path to a file that will incude the final combined, resolved YAML configuration file """ - def __init__(self, yamls, experiment, platform, target, output): + def __init__(self, yamls, experiment, platform, target, output, no_clean): self.y = yamls.split(",") self.e = str(experiment) self.p = str(platform) self.t = str(target) self.o = output + self.no_clean = no_clean def list_check_yamls(self, init_file): """ @@ -93,10 +94,13 @@ def use_uwtools(self, yaml_list): print("This will not be displayed") combined_yaml_dict = config.compose(configs=yaml_list, realize=True).as_dict() - # CLEAN SERIALIZED YAML ## - cleaned_yaml_dict = clean_yaml(combined_yaml_dict) - if not cleaned_yaml_dict: - raise ValueError("YAML configuration could not be cleaned (experiments)") + if self.no_clean: + final_yaml_dict = combined_yaml_dict + else: + # CLEAN SERIALIZED YAML ## + final_yaml_dict = clean_yaml(combined_yaml_dict) + if not final_yaml_dict: + raise ValueError("YAML configuration could not be cleaned (experiments)") ### SHOLD BE RESOLVED BUT THIS IS TO CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## ## uw config realize: resolve final yaml @@ -110,7 +114,7 @@ def use_uwtools(self, yaml_list): uwlogger = use_uwtools_logger() - config.realize(input_config = cleaned_yaml_dict, + config.realize(input_config = final_yaml_dict, values_needed = True, total = True) @@ -121,16 +125,16 @@ def use_uwtools(self, yaml_list): if self.o: out_path = Path.cwd()/self.o fre_logger.info("Writing resolved YAML file: %s", out_path) - output_yaml(cleaned_yaml_dict, out_path) + output_yaml(final_yaml_dict, out_path) # Is this helpful? (if config.realize fails, it will not be written to output file, # but error will hopefully come up; this is just more output if the user wants to # see what the combined yaml dictonary would look like - #fre_logger.debug(pformat(cleaned_yaml_dict)) + #fre_logger.debug(pformat(final_yaml_dict)) else: fre_logger.info("Resolved YAML saved as dictionary. To display dictionary, pass fre -v ...") - fre_logger.info(cleaned_yaml_dict) + fre_logger.info(final_yaml_dict) - return cleaned_yaml_dict + return final_yaml_dict # def validate(self, final_dict) # """ @@ -155,7 +159,7 @@ def use_uwtools(self, yaml_list): #with open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json") as j1, open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_pp.json") as j2, open("j3.json", 'w') as j3: # dump([load(j1), load(j2)], j3) -def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None) -> dict: +def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None, no_clean: Optional[bool]=False) -> dict: """ :param yamls: is the list of YAML configuration files to combine :type yamls: str @@ -168,10 +172,12 @@ def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:st :param output: is the file path to a file that will incude the final combined, resolved YAML configuration file :type output: str + :param no_clean: is a True/False value to determine whether or not to remove any keys/sections from the YAML. Default is False. + :type no_clean: boolean """ # fre_logger.info('checking fre_cli_version compatibility...') # check_fre_version(combined) - init_obj = LetsGo(yamls, experiment, platform, target, output) + init_obj = LetsGo(yamls, experiment, platform, target, output, no_clean) init_file = f"{Path.cwd()}/init.yaml" ymls = init_obj.list_check_yamls(init_file) diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index aab992354..853679ec8 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -50,7 +50,9 @@ def combine_yamls(yamlfile, help="Target name") @click.option("-o", "--output", type=str, help="Output") -def combine(yamls, experiment, platform, target, output): +@click.option("--no-clean", type=bool, + help="Do not remove anything from the combinedm resolved YAML file.") +def combine(yamls, experiment, platform, target, output, no_clean): """ - Combine the model yaml with the compile, platform, experiment, and analysis yamls @@ -66,7 +68,7 @@ def combine(yamls, experiment, platform, target, output): # stdout from last tool adds a newline for some reason yaml_paths = sys.stdin.read().strip() - combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output) + combine_yamls_script_new.yamltools_combine_subtool(yaml_paths, experiment, platform, target, output, no_clean) #@yamltools_cli.command() #@click.option("-y", "--yaml", From b432561be553df09ddda55671edc61e100a7d8c2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 1 Oct 2026 17:26:15 -0400 Subject: [PATCH 25/47] fix tests and add test yamls --- .../tests/test_list_experiments_script.py | 21 +++---- fre/list_/tests/test_list_platforms_script.py | 33 ++++------- .../tests/test_list_pp_components_script.py | 34 +++++------ fre/list_/tests/test_list_yamls_script.py | 30 +++++----- fre/list_/tests/yamls/compile.yaml | 49 +++++++++++++++ fre/list_/tests/yamls/empty_platforms.yaml | 1 + fre/list_/tests/yamls/model.yaml | 28 +++++---- fre/list_/tests/yamls/platforms.yaml | 34 +++++++++++ fre/list_/tests/yamls/wrong_null_model.yaml | 15 +++++ .../yamls/yaml_include/pp-test.c96_amip.yaml | 33 +++++++++++ .../tests/yamls/yaml_include/pp.c96_amip.yaml | 59 +++++++++++++++++++ .../tests/yamls/yaml_include/settings.yaml | 37 ++++++++++++ 12 files changed, 297 insertions(+), 77 deletions(-) create mode 100644 fre/list_/tests/yamls/compile.yaml create mode 100644 fre/list_/tests/yamls/empty_platforms.yaml create mode 100644 fre/list_/tests/yamls/platforms.yaml create mode 100644 fre/list_/tests/yamls/wrong_null_model.yaml create mode 100644 fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml create mode 100644 fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml create mode 100644 fre/list_/tests/yamls/yaml_include/settings.yaml diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index f78bf1f93..4a7af0c29 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -4,34 +4,29 @@ from pathlib import Path import pytest -import yaml from fre.list_ import list_experiments_script from fre.yamltools import helpers - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") -YAMLFILE = "null_model.yaml" +TEST_DIR = Path("fre/list_/tests/yamls") +YAMLFILE = "model.yaml" EXP_NAME = "None" # yaml file checks def test_modelyaml_exists(): ''' Test that model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() # Test whole tool def test_exp_list(caplog): ''' Test fre list exps subtool ''' - list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = [ 'Experiments found:', - ' - null_model_full', - ' - null_model_0', - ' - null_model_1', - ' - null_model_2' ] + ' - experiment1', + ' - experiment2' ] for i in check_out: assert i in caplog.text @@ -48,10 +43,10 @@ def test_exp_list(caplog): ) def test_yamlvalidate(): ''' Test yaml is being validated ''' - yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yamlfilepath = Path(f"{TEST_DIR}/{YAMLFILE}") # Combine model / experiment - yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}") # Validate and capture output assert helpers.validate_yaml(yml_dict, VAL_SCHEMA) diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index 8dd7eba56..1e0af98cb 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -4,63 +4,56 @@ from pathlib import Path import pytest -import yaml - from fre.list_ import list_platforms_script -from fre.yamltools import combine_yamls_script as cy - # SET-UP -TEST_DIR = Path("fre/make/tests") -NM_EXAMPLE = Path("null_example") +TEST_DIR = Path("fre/list_/tests/yamls") PLATFORM = None TARGET = None -YAMLFILE = "null_model.yaml" -EXP_NAME = YAMLFILE.split(".")[0] +YAMLFILE = "model.yaml" +EXP_NAME = YAMLFILE.split(".", maxsplit=1)[0] VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_make.json") # Bad yaml example -BADYAMLFILE_PATH = f"{TEST_DIR}/{NM_EXAMPLE}/wrong_model/wrong_null_model.yaml" +BADYAMLFILE_PATH = f"{TEST_DIR}/wrong_null_model.yaml" # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{YAMLFILE}").exists() def test_compileyaml_exists(): ''' Test compile yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() + assert Path(f"{TEST_DIR}/compile.yaml").exists() def test_platformyaml_exists(): ''' Test platforms yaml exists ''' - assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() + assert Path(f"{TEST_DIR}/platforms.yaml").exists() # Test whole tool def test_platforms_list_correct(caplog): ''' Test fre list platforms subtool ''' - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") # check the logging output check_out = ["Platforms available:", " - ncrc5.intel23", - " - hpcme.2023", + " - hpcmini.2025", " - ci.gnu", - " - con.twostep" ] + " - hpcme.intel25" ] for i in check_out: assert i in caplog.text # make sure level is INFO for record in caplog.records: - record.levelname == "INFO" + assert record.levelname == "INFO" # Test validation def test_yamlvalidate(caplog): ''' Test yaml is being validated and is actually valid''' - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - # Combine model / experiment - list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}") + list_platforms_script.list_platforms_subtool(f"{TEST_DIR}/{YAMLFILE}") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -69,7 +62,7 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - record.levelname == "INFO" + assert record.levelname == "INFO" def test_not_valid_yaml(): ''' Test the correct output matches the ValueError raised when yaml is invalid ''' diff --git a/fre/list_/tests/test_list_pp_components_script.py b/fre/list_/tests/test_list_pp_components_script.py index f273d955c..ff024537c 100644 --- a/fre/list_/tests/test_list_pp_components_script.py +++ b/fre/list_/tests/test_list_pp_components_script.py @@ -6,12 +6,13 @@ # SET-UP -TEST_DIR = Path("fre/pp/tests") -AM5_EXAMPLE = Path("AM5_example") -MODEL_YAMLFILE = "am5.yaml" +#TEST_DIR = Path("fre/pp/tests") +TEST_DIR = Path("fre/list_/tests/yamls") +#AM5_EXAMPLE = Path("AM5_example") +MODEL_YAMLFILE = "model.yaml" SETTINGS_YAMLFILE = "yaml_include/settings.yaml" -PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml", "yaml_include/settings.yaml"] -EXP_NAME = "c96L65_am5f7b12r1_amip" +PP_YAMLFILES = ["yaml_include/pp.c96_amip.yaml", "yaml_include/pp-test.c96_amip.yaml"] #, "yaml_include/settings.yaml"] +EXP_NAME = "experiment1" VAL_SCHEMA = Path("fre/gfdl_msd_schemas/FRE/fre_pp.json") PLATFORM = "FOO" TARGET = "BAR" @@ -19,35 +20,33 @@ # yaml file checks def test_modelyaml_exists(): ''' Test model yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{MODEL_YAMLFILE}").exists() def test_settingsyaml_exist(): ''' Test settings yaml exists ''' - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{SETTINGS_YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{SETTINGS_YAMLFILE}").exists() def test_ppyamls_exist(): ''' Test post-processing yamls exist ''' for pp_yaml in PP_YAMLFILES: - assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{pp_yaml}").exists() + assert Path(f"{TEST_DIR}/{pp_yaml}").exists() # Test whole tool def test_pp_comp_list(caplog): ''' Test fre list pp-components subtool ''' - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") # check the logging output check_out = [ 'Components to be post-processed:', ' - atmos_cmip', ' - atmos', - ' - atmos_level_cmip', - ' - atmos_level', - ' - atmos_month_aer', ' - atmos_diurnal', - ' - atmos_scalar'] - + ' - atmos_scalar', + ' - atmos_cmip-TEST' ] for i in check_out: assert i in caplog.text - # make sure the level is WARNING for version mismatch and INFO otherwise for record in caplog.records: assert record.levelname in ["WARNING", "INFO"] @@ -56,7 +55,9 @@ def test_pp_comp_list(caplog): def test_yamlvalidate(caplog): ''' Test yaml is being validated ''' # Combine model / experiment - list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) + list_pp_components_script.list_ppcomps_subtool(yamlfile = f"{TEST_DIR}/{MODEL_YAMLFILE}", + experiment = EXP_NAME, + application = "postprocess") validate = ["Validating YAML information...", " YAML dictionary VALID."] @@ -66,4 +67,3 @@ def test_yamlvalidate(caplog): for record in caplog.records: assert record.levelname in ["WARNING", "INFO"] - diff --git a/fre/list_/tests/test_list_yamls_script.py b/fre/list_/tests/test_list_yamls_script.py index e89ca0cbe..094e1b7f5 100644 --- a/fre/list_/tests/test_list_yamls_script.py +++ b/fre/list_/tests/test_list_yamls_script.py @@ -85,7 +85,7 @@ def test_list_yamls_postprocess_only(self, model_yaml_path): # Should NOT contain compile, platform, run, or analysis YAMLs assert "compile.yaml" not in yaml_names - assert "platforms.yaml" not in yaml_names +# assert "platforms.yaml" not in yaml_names assert "run1.yaml" not in yaml_names assert "analysis1.yaml" not in yaml_names @@ -138,23 +138,23 @@ def test_list_yamls_postprocess_and_analysis(self, model_yaml_path): def test_list_yamls_no_experiment(self, model_yaml_path): """Test with no experiment name provided (default compile behavior)""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="", - application=None - ) +# with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="", + application=None + ) - yaml_names = [Path(y).name for y in result.split(",")] + yaml_names = [Path(y).name for y in result.split(",")] - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain compile and platform YAMLs - assert "compile.yaml" in yaml_names - assert "platforms.yaml" in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain compile and platform YAMLs + assert "compile.yaml" in yaml_names + assert "platforms.yaml" in yaml_names - # Should NOT contain experiment-specific YAMLs - assert "run1.yaml" not in yaml_names + # Should NOT contain experiment-specific YAMLs + assert "run1.yaml" not in yaml_names def test_list_yamls_returns_full_paths(self, model_yaml_path): """Test that returned YAMLs have full paths""" diff --git a/fre/list_/tests/yamls/compile.yaml b/fre/list_/tests/yamls/compile.yaml new file mode 100644 index 000000000..51a72b73c --- /dev/null +++ b/fre/list_/tests/yamls/compile.yaml @@ -0,0 +1,49 @@ +compile: + experiment: "null_model_full" + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + additionalInstructions: "echo \"Testing additionalInstructions\"" + cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" + otherFlags: "-fallow-argument-mismatch" # only needed for gcc + branch: "2025.04" + - component: "atmos_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/atmos_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "land_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/land_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_param" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ocean_null" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_null.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "ice_null" + requires: ["FMS", "ice_param", "ocean_null"] + repo: "https://github.com/NOAA-GFDL/ice_null.git" + branch: "{{ fre_properties.branch }}" + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: "{{ fre_properties.FMSincludes }}" + - component: "coupler" + requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: "{{ fre_properties.branch }}" + otherFlags: "{{ fre_properties.FMSincludes }}" + cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML" + paths: [ "coupler/full", + "coupler/shared" ] + diff --git a/fre/list_/tests/yamls/empty_platforms.yaml b/fre/list_/tests/yamls/empty_platforms.yaml new file mode 100644 index 000000000..9bceccc06 --- /dev/null +++ b/fre/list_/tests/yamls/empty_platforms.yaml @@ -0,0 +1 @@ +platforms: diff --git a/fre/list_/tests/yamls/model.yaml b/fre/list_/tests/yamls/model.yaml index 43cc46edb..035dbdcf3 100644 --- a/fre/list_/tests/yamls/model.yaml +++ b/fre/list_/tests/yamls/model.yaml @@ -1,22 +1,25 @@ # reusable variables fre_properties: - VERSION: "foo" - FRE_STEM: "test_list/{{ fre_properties.VERSION }}" + VERSION: "foo" + FRE_STEM: "test_list/{{ fre_properties.VERSION }}" # amip - EXP_AMIP_START: "19790101T0000Z" - EXP_AMIP_END: "20200101T0000Z" - ANA_AMIP_START: "1980" - ANA_AMIP_END: "2020" + EXP_AMIP_START: "19790101T0000Z" + EXP_AMIP_END: "20200101T0000Z" + ANA_AMIP_START: "1980" + ANA_AMIP_END: "2020" - PP_AMIP_CHUNK96: "P1Y" - PP_XYINTERP96: "180,288" - PP_XYINTERP384: "720,1152" + PP_AMIP_CHUNK96: "P1Y" + PP_XYINTERP96: "180,288" + PP_XYINTERP384: "720,1152" # compile information - INTEL: "intel-classic" - FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + INTEL: "intel-classic" + FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + branch: "test" + + GRID_SPEC96: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" # compile information build: @@ -33,3 +36,4 @@ experiments: - "yaml_include/pp-test.c96_amip.yaml" analysis: - "analysis1.yaml" + experiment2: diff --git a/fre/list_/tests/yamls/platforms.yaml b/fre/list_/tests/yamls/platforms.yaml new file mode 100644 index 000000000..00d1c74fc --- /dev/null +++ b/fre/list_/tests/yamls/platforms.yaml @@ -0,0 +1,34 @@ +platforms: + - name: ncrc5.intel23 + compiler: intel + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", + "source $MODULESHOME/init/sh", + "module load {{ fre_properties.INTEL }}/2023.2.0", + "module load fre/bronx-23", + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" + modelRoot: ${HOME}/fremake_canopy/test + - name: ci.gnu + compiler: gnu + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test + - name: hpcmini.2025 + compiler: gnu + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "ghcr.io/rem1776/hpc-mini:alpine3gcc14.2" + mkTemplate: "linux-ubuntu-xenial-gnu.mk" + - name: hpcme.intel25 + compiler: intel + RUNenv: "" + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3" + mkTemplate: "/apps/mkmf/templates/hpcme-intel2025.03-oneapi.mk" + container2step: True + container2base: "gitlab.gfdl.noaa.gov:5050/fre/hpc-me/base-ubuntu24.04-intel:2025.3rte" diff --git a/fre/list_/tests/yamls/wrong_null_model.yaml b/fre/list_/tests/yamls/wrong_null_model.yaml new file mode 100644 index 000000000..ccec12827 --- /dev/null +++ b/fre/list_/tests/yamls/wrong_null_model.yaml @@ -0,0 +1,15 @@ +# reusable variables +fre_properties: + VERSION: "full" + FRE_STEM: "null_model/{{ fre_properties.VERSION }}" + # compile information + branch: "main" + INTEL: "intel-classic" + FMSincludes: "-IFMS/include" + +build: + compileYaml: "compile.yaml" + platformYaml: "empty_platforms.yaml" + +experiments: + experiment1: diff --git a/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..b54f0fc77 --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "200,200" + +#c96_amip_postprocess: +postprocess: + components: + atmos_cmip-TEST: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.custom_interp}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos-TEST: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level_cmip-TEST: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..a06f8162e --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,59 @@ +# local reusable variable overrides +fre_properties: + custom_interp: "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + atmos_cmip: + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.custom_interp}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + atmos_level_cmip: + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_level: + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + atmos_month_aer: + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + atmos_diurnal: + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: "{{fre_properties.PP_XYINTERP96}}" + interpMethod: "conserve_order2" + inputRealm: 'atmos' + atmos_scalar: + sources: + - history_file: "atmos_scalar" diff --git a/fre/list_/tests/yamls/yaml_include/settings.yaml b/fre/list_/tests/yamls/yaml_include/settings.yaml new file mode 100644 index 000000000..b790137a0 --- /dev/null +++ b/fre/list_/tests/yamls/yaml_include/settings.yaml @@ -0,0 +1,37 @@ +#c96_amip_directories: +directories: + history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history" + refined_history_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/history_dir_refined" + pp_dir: "/archive/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}/{{ platform }}-{{ target }}/pp" + analysis_dir: "/nbhome/$USER/{{ fre_properties.FRE_STEM }}/{{ name }}" + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: "{{ fre_properties.ANA_AMIP_START }}" + pp_stop: "{{ fre_properties.ANA_AMIP_END }}" + pp_chunks: + - "{{ fre_properties.PP_AMIP_CHUNK96 }}" + pp_grid_spec: "{{ fre_properties.GRID_SPEC96 }}" + switches: + clean_work: True + do_atmos_plevel_masking: True + preanalysis: + vitals: + script: "/path/to/vitals-script" + inputs: ['atmos_month', 'aerosol_month'] + do_preanalysis: True + refinediag: + ocean: + script: "/path/to/ocean-script" + inputs: ['ocean_monthly', 'ocean_bgc'] + outputs: ['ocean_refined'] + do_refinediag: True + aerosol: + script: "/path/to/aerosol-script" + inputs: ['aerosol_month'] + outputs: ['aerosol_refined'] + do_refinediag: True From 559ede12e368573e51dde7e68acf7164c77437e2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 16:16:15 -0400 Subject: [PATCH 26/47] update fre yamltools compile tests - add yamls for this tool for now as I update yaml formats for uwtools --- .../tests/test_combine_yamls_script.py | 12 ++++++---- .../tests/yamls/compile_yamls/platforms.yaml | 23 +++++++------------ 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index 16af8e6ed..f9c4ce790 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -155,10 +155,13 @@ def test_check_expected_platformyamlcontent(): # EXP_NAME = YAMLFILE.split(".")[0] # Combine model / experiment - yamls = ("fre/make/tests/null_example/null_model.yaml," - "fre/make/tests/null_example/compile.yaml," - "fre/make/tests/null_example/platforms.yaml") - yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None) +# yamls = ("fre/make/tests/null_example/null_model.yaml," +# "fre/make/tests/null_example/compile.yaml," +# "fre/make/tests/null_example/platforms.yaml") + yamls = ("fre/yamltools/tests/yamls/model.yaml," + "fre/yamltools/tests/yamls/compile_yamls/compile.yaml," + "fre/yamltools/tests/yamls/compile_yamls/platforms.yaml") + yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = "/home/Dana.Singh/fre/singh/cli-dev/generalize-yaml-serialization/HERE.yml") # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', @@ -185,7 +188,6 @@ def test_check_expected_platformyamlcontent(): 'containerBase': 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25', 'mkTemplate': '/apps/mkmf/templates/hpcme-intel21.mk', 'volume': '/gpfs/f5'} - for key,value in yml_dict.items(): if key == "platforms": assert expected_platform_info_1 in value diff --git a/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml index 144261ce1..639f54c33 100644 --- a/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml +++ b/fre/yamltools/tests/yamls/compile_yamls/platforms.yaml @@ -1,23 +1,13 @@ platforms: - - name: ncrc5.intel - compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", - "source $MODULESHOME/init/sh", - "module load {{ fre_properties.INTEL }}/2022.2.1", - "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" - modelRoot: ${HOME}/fremake_canopy/test - name: ncrc5.intel23 compiler: intel - envSetup: [" module use -a /ncrc/home2/fms/local/modulefiles", + envSetup: ["module use -a /ncrc/home2/fms/local/modulefiles", "source $MODULESHOME/init/sh", - "module load {{ fre_properties.INTEL }}/2023.1.0", + "module load {{ fre_properties.INTEL }}/2023.2.0", "module load fre/bronx-23", - "module load cray-hdf5/1.12.2.3", - "module load cray-netcdf/4.9.0.3"] - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" + "module load cray-hdf5/1.12.2.11", + "module load cray-netcdf/4.9.0.11"] + mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/{{ fre_properties.INTEL }}.mk" modelRoot: ${HOME}/fremake_canopy/test - name: hpcme.2023 compiler: intel @@ -26,3 +16,6 @@ platforms: container: True containerBuild: "podman" containerRun: "apptainer" + containerBase: 'docker.io/ecpe4s/noaa-intel-prototype:2023.09.25' + mkTemplate: '/apps/mkmf/templates/hpcme-intel21.mk' + volume: '/gpfs/f5' From d3546ead6683c328796d3bb4b100eaa95b3a61d1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 16:42:14 -0400 Subject: [PATCH 27/47] whoops remove this --- fre/yamltools/tests/test_combine_yamls_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index f9c4ce790..fe349bc2e 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -161,7 +161,7 @@ def test_check_expected_platformyamlcontent(): yamls = ("fre/yamltools/tests/yamls/model.yaml," "fre/yamltools/tests/yamls/compile_yamls/compile.yaml," "fre/yamltools/tests/yamls/compile_yamls/platforms.yaml") - yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = "/home/Dana.Singh/fre/singh/cli-dev/generalize-yaml-serialization/HERE.yml") + yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None) # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', From 66b5f6b2b0876c0550828b84588af7234fe25444 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 17:33:42 -0400 Subject: [PATCH 28/47] only target fre package NOT uwtools logger --- fre/list_/tests/test_list_experiments_script.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fre/list_/tests/test_list_experiments_script.py b/fre/list_/tests/test_list_experiments_script.py index 4a7af0c29..2b9c96f2d 100644 --- a/fre/list_/tests/test_list_experiments_script.py +++ b/fre/list_/tests/test_list_experiments_script.py @@ -32,7 +32,8 @@ def test_exp_list(caplog): # make sure the level is INFO for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation @pytest.mark.skip( From 6a75cfc9b4197c298ff85dc55b20f5077db6a189 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 17:42:45 -0400 Subject: [PATCH 29/47] do the same for other tests --- fre/list_/tests/test_list_platforms_script.py | 6 ++++-- fre/list_/tests/test_list_pp_components_script.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fre/list_/tests/test_list_platforms_script.py b/fre/list_/tests/test_list_platforms_script.py index 1e0af98cb..16e6ecd2f 100644 --- a/fre/list_/tests/test_list_platforms_script.py +++ b/fre/list_/tests/test_list_platforms_script.py @@ -47,7 +47,8 @@ def test_platforms_list_correct(caplog): # make sure level is INFO for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" # Test validation def test_yamlvalidate(caplog): @@ -62,7 +63,8 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - assert record.levelname == "INFO" + if record.name.startswith("fre"): + assert record.levelname == "INFO" def test_not_valid_yaml(): ''' Test the correct output matches the ValueError raised when yaml is invalid ''' diff --git a/fre/list_/tests/test_list_pp_components_script.py b/fre/list_/tests/test_list_pp_components_script.py index ff024537c..fb0b30388 100644 --- a/fre/list_/tests/test_list_pp_components_script.py +++ b/fre/list_/tests/test_list_pp_components_script.py @@ -49,7 +49,8 @@ def test_pp_comp_list(caplog): assert i in caplog.text # make sure the level is WARNING for version mismatch and INFO otherwise for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] # Test validation def test_yamlvalidate(caplog): @@ -66,4 +67,5 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - assert record.levelname in ["WARNING", "INFO"] + if record.name.startswith("fre"): + assert record.levelname in ["WARNING", "INFO"] From f12f10318f50ae434af86c62eaabd0bdfd76ccc8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 2 Oct 2026 18:03:49 -0400 Subject: [PATCH 30/47] fix test for yaml additions --- fre/list_/tests/test_list_yamls_script.py | 35 +++++++++++------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/fre/list_/tests/test_list_yamls_script.py b/fre/list_/tests/test_list_yamls_script.py index 094e1b7f5..d909d355e 100644 --- a/fre/list_/tests/test_list_yamls_script.py +++ b/fre/list_/tests/test_list_yamls_script.py @@ -67,27 +67,24 @@ def test_list_yamls_runtime_only(self, model_yaml_path): def test_list_yamls_postprocess_only(self, model_yaml_path): """Test postprocess_only flag (without analysis) returns model YAML plus postprocessing YAMLs""" - with pytest.raises(ValueError): #, match = " *** PROVIDE THE MISSING YAML CONFIGURATIONS ***"): - result = list_yamls_subtool( - yamlfile=model_yaml_path, - experiment="experiment1", - application="postprocess" - ) - - yaml_names = [Path(y).name for y in result.split(",")] + result = list_yamls_subtool( + yamlfile=model_yaml_path, + experiment="experiment1", + application="postprocess" + ) - # Model YAML should always be included - assert "model.yaml" in yaml_names - # Should contain settings and postprocessing YAMLs - assert "settings.yaml" in yaml_names - assert "pp.c96_amip.yaml" in yaml_names - assert "pp-test.c96_amip.yaml" in yaml_names + yaml_names = [Path(y).name for y in result.split(",")] - # Should NOT contain compile, platform, run, or analysis YAMLs - assert "compile.yaml" not in yaml_names -# assert "platforms.yaml" not in yaml_names - assert "run1.yaml" not in yaml_names - assert "analysis1.yaml" not in yaml_names + # Model YAML should always be included + assert "model.yaml" in yaml_names + # Should contain settings and postprocessing YAMLs + assert "settings.yaml" in yaml_names + assert "pp.c96_amip.yaml" in yaml_names + assert "pp-test.c96_amip.yaml" in yaml_names + # Should NOT contain compile, platform, run, or analysis YAMLs + assert "compile.yaml" not in yaml_names + assert "run1.yaml" not in yaml_names + assert "analysis1.yaml" not in yaml_names def test_list_yamls_analysis_only(self, model_yaml_path): """Test analysis_only flag (without postprocess) returns model YAML plus analysis YAMLs""" From 361eac9c4ed371f1270d3ca2bfcfd0490601cd1c Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 5 Oct 2026 09:59:45 -0400 Subject: [PATCH 31/47] Fix conda install command in build_conda.yml Updated conda install command to include the '-y' flag for automatic confirmation. --- .github/workflows/build_conda.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index c7a630df5..4afcba2b4 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,7 +1,11 @@ name: build_conda +# conda-build v26.9.1 caused an error in the build related to "AttributeError" +# when looking for certain conda plugins +# Don't think plugins are needed here env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + CONDA_NO_PLUGINS: true on: pull_request: @@ -43,7 +47,7 @@ jobs: - name: Conda install conda-build run: | echo "conda install conda-build" - conda install conda-forge::conda-build + conda install -y conda-forge::conda-build - name: Check conda-build version and supported package formats run: | From f0803f30b31fdba44bfaab9b8a2b124a5ae83282 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:10:44 -0400 Subject: [PATCH 32/47] Add conda solver configuration to build workflow --- .github/workflows/build_conda.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 4afcba2b4..938fce79c 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -44,6 +44,13 @@ jobs: echo "printing conda config just in case" conda config --show + # Container being used may rely on "classic" solver but conda + # versions >=23.10.0 use a "libmamba solver as default + - name: Set conda solver + run: | + conda install --solver=classic conda-forge::conda-libmamba-solver + conda config --set solver libmamba + - name: Conda install conda-build run: | echo "conda install conda-build" From fc0c4fc1014364903691bde531890e2580ecf92e Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:27:28 -0400 Subject: [PATCH 33/47] Actually set the solver as classic Updated comments regarding conda-build and solver settings. --- .github/workflows/build_conda.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 938fce79c..b23a9f13b 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,11 +1,14 @@ name: build_conda -# conda-build v26.9.1 caused an error in the build related to "AttributeError" -# when looking for certain conda plugins -# Don't think plugins are needed here +# 1. conda-build v26.9.1 caused an error in the build related to +# "AttributeError" when looking for certain conda plugins +# Don't think plugins are needed here +# 2. Container being used may rely on "classic" solver but conda +# versions >=23.10.0 use a "libmamba solver as default env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true CONDA_NO_PLUGINS: true + CONDA_SOLVER: classic on: pull_request: @@ -44,13 +47,6 @@ jobs: echo "printing conda config just in case" conda config --show - # Container being used may rely on "classic" solver but conda - # versions >=23.10.0 use a "libmamba solver as default - - name: Set conda solver - run: | - conda install --solver=classic conda-forge::conda-libmamba-solver - conda config --set solver libmamba - - name: Conda install conda-build run: | echo "conda install conda-build" From e9440fb4c1cb64c9c29d4a5c305aa6e755678158 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 5 Oct 2026 11:22:35 -0400 Subject: [PATCH 34/47] test version that worked before --- .github/workflows/build_conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index b23a9f13b..491438eb5 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -8,7 +8,7 @@ name: build_conda env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true CONDA_NO_PLUGINS: true - CONDA_SOLVER: classic +# CONDA_SOLVER: classic on: pull_request: @@ -50,7 +50,7 @@ jobs: - name: Conda install conda-build run: | echo "conda install conda-build" - conda install -y conda-forge::conda-build + conda install -y conda-forge::conda-build=26.7.3 - name: Check conda-build version and supported package formats run: | From 6931715dfc31138b35dba687a737ade72dd9dc09 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 5 Oct 2026 17:05:34 -0400 Subject: [PATCH 35/47] install libmamba --- .github/workflows/build_conda.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 491438eb5..4c93c1542 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,14 +1,10 @@ name: build_conda -# 1. conda-build v26.9.1 caused an error in the build related to -# "AttributeError" when looking for certain conda plugins -# Don't think plugins are needed here -# 2. Container being used may rely on "classic" solver but conda -# versions >=23.10.0 use a "libmamba solver as default +# Container being used may rely on "classic" solver but conda +# versions >=23.10.0 use a "libmamba solver as default env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - CONDA_NO_PLUGINS: true -# CONDA_SOLVER: classic + CONDA_SOLVER: libmamba on: pull_request: @@ -47,7 +43,13 @@ jobs: echo "printing conda config just in case" conda config --show + - name: Conda install libmamba conda solver + run: | + conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda config --set solver libmamba + - name: Conda install conda-build + timeout-minutes: 10 run: | echo "conda install conda-build" conda install -y conda-forge::conda-build=26.7.3 From 76fb85a5ad4e1c8a9c304820828fcbc7270bd32e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 5 Oct 2026 17:19:32 -0400 Subject: [PATCH 36/47] remove version specification --- .github/workflows/build_conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 4c93c1542..fb21d5323 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -52,7 +52,7 @@ jobs: timeout-minutes: 10 run: | echo "conda install conda-build" - conda install -y conda-forge::conda-build=26.7.3 + conda install conda-forge::conda-build - name: Check conda-build version and supported package formats run: | From 79bfe5197609ccb22ad46984adbcdf3005eb584e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 6 Oct 2026 14:00:12 -0400 Subject: [PATCH 37/47] hopefully fix build --- .github/workflows/build_conda.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index fb21d5323..935efac81 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,10 +1,7 @@ name: build_conda -# Container being used may rely on "classic" solver but conda -# versions >=23.10.0 use a "libmamba solver as default env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - CONDA_SOLVER: libmamba on: pull_request: @@ -43,7 +40,7 @@ jobs: echo "printing conda config just in case" conda config --show - - name: Conda install libmamba conda solver + - name: Conda install and set libmamba conda solver run: | conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba conda config --set solver libmamba @@ -97,11 +94,22 @@ jobs: echo "printing conda config just in case" conda config --show + - name: Conda install and set libmamba conda solver + run: | + conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda config --set solver libmamba + - name: Conda install conda-build run: | echo "conda install conda-build" conda install conda-forge::conda-build + - name: Check conda-build version and supported package formats + run: | + conda --version + conda build --version + conda build --help + - name: Download tarball uses: actions/download-artifact@v5 with: From 98337b61fcc0c87f7bd63ae6d910e7cf25fdb449 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 7 Oct 2026 14:40:16 -0400 Subject: [PATCH 38/47] #874 clean up --- fre/yamltools/combine_yamls_script_new.py | 33 +++-------------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index 7ce5f13a9..ce08ec6d3 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -13,7 +13,7 @@ - can use this tool to pass in multiple yamls either comma separated string (-y y1,y2) OR multiple options (-y y1 -y y2 ...) - checks fre-version -- uses uw config compose (to combine), resolve (to check unrendered values), and validate (to validate yaml) +- uses uw config compose (to combine), resolve (to check unrendered values) """ import os @@ -28,7 +28,7 @@ fre_logger = logging.getLogger(__name__) -class LetsGo(): +class MergeYamls(): """ :ivar str yamls: is the list of YAML configuration files to combine :ivar str experiment: is the experiment name (relates to the run and postprocessing) @@ -82,7 +82,6 @@ def use_uwtools(self, yaml_list): ## COMBINE YAMLS AND RESOLVE WHERE WE CAN ## ## uw config compose: pass yaml list to compose final yaml - # CAN THIS BE DICTIONARIES?? # config.compose returns a base class specifying methods to read, manipulate, # and write several configuration-file formats. (use as_dict to return dictionary) # use realize=True to resolve what we can here; if any unresolved, it does not error out @@ -136,29 +135,6 @@ def use_uwtools(self, yaml_list): return final_yaml_dict -# def validate(self, final_dict) -# """ -# """ - -# VALIDATE SERIALIZED YAML ## -# ## uw config validate -# # output from validate is True or False -# validate_out = config.validate(schema_file="/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json", -# config_data=cleaned_yaml_dict) -# -# # If the YAML is not valid, exit with error instead of continuing -# if validate_out: -# fre_logger.info("VALID") -# else: -# fre_logger.error("INVALID") -# raise ValueError("INVALID. CHECK YA YAMLS") -# -## Combine schemas? -#from json import load, dump -# -#with open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_make.json") as j1, open("/home/Dana.Singh/fre/singh/generalize-yaml-serialization/fre/gfdl_msd_schemas/FRE/fre_pp.json") as j2, open("j3.json", 'w') as j3: -# dump([load(j1), load(j2)], j3) - def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:str, output: Optional[str]=None, no_clean: Optional[bool]=False) -> dict: """ :param yamls: is the list of YAML configuration files to combine @@ -177,15 +153,12 @@ def yamltools_combine_subtool(yamls:str, experiment:str, platform:str, target:st """ # fre_logger.info('checking fre_cli_version compatibility...') # check_fre_version(combined) - init_obj = LetsGo(yamls, experiment, platform, target, output, no_clean) + init_obj = MergeYamls(yamls, experiment, platform, target, output, no_clean) init_file = f"{Path.cwd()}/init.yaml" ymls = init_obj.list_check_yamls(init_file) combined = init_obj.use_uwtools(ymls) -# #validate -# init_obj.validate(combined) - # clean init_file (not needed anymore) Path(init_file).unlink() From 6955bef1c799af5caa0a1a93990af344601c87c3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 7 Oct 2026 15:09:55 -0400 Subject: [PATCH 39/47] #874 add test --- fre/tests/test_fre_yamltools_cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fre/tests/test_fre_yamltools_cli.py b/fre/tests/test_fre_yamltools_cli.py index ae86b0b64..4586fccc6 100644 --- a/fre/tests/test_fre_yamltools_cli.py +++ b/fre/tests/test_fre_yamltools_cli.py @@ -61,3 +61,9 @@ def test_cli_fre_yamltools_piped_from_list(caplog): freyamltools_pipe = runner.invoke(fre.fre, args=["-v", "yamltools", "combine"], input=frelist_out.output) assert freyamltools_pipe.exit_code == 0 assert "PIPED" in caplog.text + +def test_cli_fre_yamltools_NOT_piped_from_list(caplog): + ''' fre yamtools combine usage without fre list ''' + result = runner.invoke(fre.fre, args=["-v", "yamltools", "combine", "-y", "fre/yamltools/tests/yamls/model.yaml,fre/yamltools/tests/yamls/settings.yaml"]) + assert result.exit_code == 0 + assert "NOT PIPED" in caplog.text From 7b52b9a26215dc1b029f1ad036f79f30dd0d33fa Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 7 Oct 2026 15:27:48 -0400 Subject: [PATCH 40/47] #874 cover tests --- fre/tests/test_fre_list_cli.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/fre/tests/test_fre_list_cli.py b/fre/tests/test_fre_list_cli.py index 7315e507d..36de934d0 100644 --- a/fre/tests/test_fre_list_cli.py +++ b/fre/tests/test_fre_list_cli.py @@ -62,3 +62,35 @@ def test_cli_fre_list_exps_opt_dne(): ''' fre list platforms optionDNE ''' result = runner.invoke(fre.fre, args=["list", "platforms", "optionDNE"]) assert result.exit_code == 2 + +## fre list pp-components +def test_cli_fre_list_ppcomponents(): + ''' fre list pp-components ''' + result = runner.invoke(fre.fre, args=["list", "pp-components"]) + assert result.exit_code == 2 + +def test_cli_fre_list_ppcomponents_help(): + ''' fre list pp-components --help ''' + result = runner.invoke(fre.fre, args=["list", "pp-components", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_list_ppcomponents_opt_dne(): + ''' fre list pp-components optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "pp-components", "optionDNE"]) + assert result.exit_code == 2 + +## fre list yamls +def test_cli_fre_list_yamls(): + ''' fre list yamls ''' + result = runner.invoke(fre.fre, args=["list", "yamls"]) + assert result.exit_code == 2 + +def test_cli_fre_list_yamls_help(): + ''' fre list yamls --help ''' + result = runner.invoke(fre.fre, args=["list", "yamls", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_list_yamls_opt_dne(): + ''' fre list yamls optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "yamls", "optionDNE"]) + assert result.exit_code == 2 From 119f8d38c5538fca28f4e290e70f9c949d984495 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 7 Oct 2026 15:57:30 -0400 Subject: [PATCH 41/47] #874 improve test coverage for failing test --- fre/tests/test_fre_list_cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fre/tests/test_fre_list_cli.py b/fre/tests/test_fre_list_cli.py index 36de934d0..f8645b387 100644 --- a/fre/tests/test_fre_list_cli.py +++ b/fre/tests/test_fre_list_cli.py @@ -78,7 +78,15 @@ def test_cli_fre_list_ppcomponents_opt_dne(): ''' fre list pp-components optionDNE ''' result = runner.invoke(fre.fre, args=["list", "pp-components", "optionDNE"]) assert result.exit_code == 2 - + +def test_cli_fre_list_ppcomponents_app_opt(caplog): + ''' fre list pp-components -a postprocess optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "pp-components", "-y", "fre/list_/tests/yamls/model.yaml", + "-e", "experiment1", "-a", "postprocess"]) + assert result.exit_code == 0 + for c in ["atmos_cmip", "atmos", "atmos_diurnal", "atmos_scalar", "atmos_cmip-TEST"]: + assert c in caplog.text + ## fre list yamls def test_cli_fre_list_yamls(): ''' fre list yamls ''' From fb49784522672d49560e32792781463753f0716f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 7 Oct 2026 16:45:27 -0400 Subject: [PATCH 42/47] #874 attempt to fix failing code coverage --- fre/yamltools/combine_yamls_script_new.py | 4 +- fre/yamltools/helpers.py | 5 +- .../tests/test_combine_yamls_script.py | 46 ++++++++++++------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/fre/yamltools/combine_yamls_script_new.py b/fre/yamltools/combine_yamls_script_new.py index ce08ec6d3..213162fe6 100644 --- a/fre/yamltools/combine_yamls_script_new.py +++ b/fre/yamltools/combine_yamls_script_new.py @@ -98,8 +98,8 @@ def use_uwtools(self, yaml_list): else: # CLEAN SERIALIZED YAML ## final_yaml_dict = clean_yaml(combined_yaml_dict) - if not final_yaml_dict: - raise ValueError("YAML configuration could not be cleaned (experiments)") +# if not final_yaml_dict: +# raise ValueError("YAML configuration could not be cleaned (experiments)") ### SHOLD BE RESOLVED BUT THIS IS TO CATCH ANY UNRESOLVED JUST IN CASE AND OUTPUT TO FILE IF SPECIFIED ## ## uw config realize: resolve final yaml diff --git a/fre/yamltools/helpers.py b/fre/yamltools/helpers.py index f7de2cd2f..0884a8aef 100644 --- a/fre/yamltools/helpers.py +++ b/fre/yamltools/helpers.py @@ -175,7 +175,10 @@ def clean_yaml(yml_dict): keys_clean=["experiments", "fre_cli_version"] for kc in keys_clean: if kc in yml_dict.keys(): - del yml_dict[kc] + try: + del yml_dict[kc] + except: + raise ValueError("YAML configuration could not be cleaned (experiments, fre_cli_version)") # Dump cleaned dictionary back into combined yaml file #cleaned_yaml = yaml.safe_dump(yml_dict,default_flow_style=False,sort_keys=False) diff --git a/fre/yamltools/tests/test_combine_yamls_script.py b/fre/yamltools/tests/test_combine_yamls_script.py index fe349bc2e..0b0def0b3 100644 --- a/fre/yamltools/tests/test_combine_yamls_script.py +++ b/fre/yamltools/tests/test_combine_yamls_script.py @@ -72,10 +72,25 @@ def test_merged_compile_yamls(): f"{IN_DIR}/compile_yamls/platforms.yaml") # Merge the yamls try: - cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: assert False +def test_merged_compile_yamls_noclean(): + """ + Check for the creation of the combined-[experiment] yaml + Check that the model yaml was merged into the combined yaml + """ + yamls = (f"{IN_DIR}/model.yaml," + f"{IN_DIR}/compile_yamls/compile.yaml," + f"{IN_DIR}/compile_yamls/platforms.yaml") + # Merge the yamls + result = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = True) + assert (["fre_properties" in result, + "experiments" in result]) + def test_combined_compileyaml_validation(): """ Validate the combined compile yaml @@ -85,7 +100,8 @@ def test_combined_compileyaml_validation(): f"{IN_DIR}/compile_yamls/platforms.yaml") # Merge the yamls try: - out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: assert False @@ -113,7 +129,8 @@ def test_combined_compileyaml_combinefail(): # Merge the yamls - should fail since there is no compile yaml specified in the model yaml try: #out = - cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: print("EXPECTED FAILURE") assert True @@ -129,7 +146,8 @@ def test_combined_compileyaml_validatefail(): # Merge the yamls try: - out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, output = None) + out = cy.yamltools_combine_subtool(yamls, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, + output = None, no_clean = False) except: assert False @@ -149,19 +167,12 @@ def test_combined_compileyaml_validatefail(): def test_check_expected_platformyamlcontent(): ''' Test that expected yaml information (platform info) is included in dictionary content ''' -# TEST_DIR = Path("fre/make/tests") -# NM_EXAMPLE = Path("null_example") -# YAMLFILE = "null_model.yaml" -# EXP_NAME = YAMLFILE.split(".")[0] - # Combine model / experiment -# yamls = ("fre/make/tests/null_example/null_model.yaml," -# "fre/make/tests/null_example/compile.yaml," -# "fre/make/tests/null_example/platforms.yaml") yamls = ("fre/yamltools/tests/yamls/model.yaml," "fre/yamltools/tests/yamls/compile_yamls/compile.yaml," "fre/yamltools/tests/yamls/compile_yamls/platforms.yaml") - yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None) + yml_dict = cy.yamltools_combine_subtool(yamls, None, None, None, output = None, + no_clean = False) # compare combined yaml info with some information that's supposed to be parsed expected_platform_info_1 = {'name': 'ncrc5.intel23', @@ -218,7 +229,8 @@ def test_merged_pp_yamls(): # Merge the yamls try: - cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) + cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, + output = None, no_clean = False) except: assert False @@ -232,7 +244,8 @@ def test_combined_ppyaml_validation(): f"{IN_DIR}/pp_yamls/pp-TEST.c96_amip.yaml") # Merge the yamls try: - out = cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, output = None) + out = cy.yamltools_combine_subtool(yamls, PP_EXPERIMENT, PP_PLATFORM, PP_TARGET, + output = None, no_clean = False) except: assert False @@ -337,7 +350,8 @@ def test_combine_pp_yamls(tmp_path): f"{tmp_path}/settings.yaml," f"{tmp_path}/pp1.yaml,{tmp_path}/pp2.yaml") - output = cy.yamltools_combine_subtool(yamls, 'expname', 'platform', 'target', output = None) + output = cy.yamltools_combine_subtool(yamls, 'expname', 'platform', 'target', output = None, + no_clean = False) pp = pprint.PrettyPrinter(indent=4) pp.pprint(output) pp.pprint(combined) From 4e21ba66dbc88834b1ab3a32525c1f89351296ed Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 7 Oct 2026 17:16:17 -0400 Subject: [PATCH 43/47] #874 improve test coverage for failing test --- fre/yamltools/helpers.py | 10 +++++----- fre/yamltools/tests/test_helpers.py | 8 +++++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/fre/yamltools/helpers.py b/fre/yamltools/helpers.py index 0884a8aef..eb69253dc 100644 --- a/fre/yamltools/helpers.py +++ b/fre/yamltools/helpers.py @@ -173,12 +173,12 @@ def clean_yaml(yml_dict): # Clean the yaml # If keys exists, delete: keys_clean=["experiments", "fre_cli_version"] - for kc in keys_clean: - if kc in yml_dict.keys(): - try: + try: + for kc in keys_clean: + if kc in yml_dict.keys(): del yml_dict[kc] - except: - raise ValueError("YAML configuration could not be cleaned (experiments, fre_cli_version)") + except: + raise ValueError("YAML configuration could not be cleaned (experiments, fre_cli_version)") # Dump cleaned dictionary back into combined yaml file #cleaned_yaml = yaml.safe_dump(yml_dict,default_flow_style=False,sort_keys=False) diff --git a/fre/yamltools/tests/test_helpers.py b/fre/yamltools/tests/test_helpers.py index 84e43c5bc..be5b53c9f 100644 --- a/fre/yamltools/tests/test_helpers.py +++ b/fre/yamltools/tests/test_helpers.py @@ -1,11 +1,12 @@ import os +import re import tempfile import pytest import yaml import fre -from fre.yamltools.helpers import yaml_load, check_fre_version +from fre.yamltools.helpers import yaml_load, check_fre_version, clean_yaml @pytest.fixture @@ -69,3 +70,8 @@ def test_check_fre_version_missing(yaml_without_version, caplog): with caplog.at_level(logging.WARNING): check_fre_version(yaml_without_version) assert "fre_cli_version not specified" in caplog.text, f"i'd suspect the 'import fre' in fre/yamltools/helpers" + +def test_clean_fail(): + """clean_yaml should fail if nto given a dictionary""" + with pytest.raises(ValueError, match=re.escape("YAML configuration could not be cleaned (experiments, fre_cli_version)")): + clean_yaml("fre/yamltools/tests/yamls/model.yaml") From e73d4a8c8ca0947a7522788797e36a2bf10bd91a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 13 Oct 2026 10:33:07 -0400 Subject: [PATCH 44/47] #874 try testing package this way --- .github/workflows/build_conda.yml | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 935efac81..04430aa98 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -108,7 +108,6 @@ jobs: run: | conda --version conda build --version - conda build --help - name: Download tarball uses: actions/download-artifact@v5 @@ -120,12 +119,34 @@ jobs: timeout-minutes: 20 run: | pkg=$(find /app/fre-cli-tarball/noarch -name 'fre-cli-*.conda' | head -n 1) - + if [ -z "$pkg" ]; then echo "Could not find uploaded fre-cli conda package" exit 1 fi - + echo "Testing the package: $pkg" - conda -v build --test "$pkg" - + # install package in conda environment + conda create -y -n fre-cli "$pkg" + conda activate fre-cli + fre --help + fre app --help + fre catalog --help + fre check --help + fre cmor --help + fre list --help + fre make --help + fre pp --help + fre run --help + fre yamltools --help + + #run: | + # pkg=$(find /app/fre-cli-tarball/noarch -name 'fre-cli-*.conda' | head -n 1) + # + # if [ -z "$pkg" ]; then + # echo "Could not find uploaded fre-cli conda package" + # exit 1 + # fi + # + # echo "Testing the package: $pkg" + # conda -v build --test "$pkg" From 0aad4cc378d17e54326314a15ff03346df5f54ab Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 13 Oct 2026 10:58:05 -0400 Subject: [PATCH 45/47] #874 try adding python --- .github/workflows/build_conda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 04430aa98..54eaade7d 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -127,8 +127,9 @@ jobs: echo "Testing the package: $pkg" # install package in conda environment - conda create -y -n fre-cli "$pkg" + conda create -y -n fre-cli python conda activate fre-cli + conda install -y "$pkg" fre --help fre app --help fre catalog --help From c558766294a733f88db96265474f0391a0ce9113 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 14 Oct 2026 02:27:39 -0400 Subject: [PATCH 46/47] #874 pin libmamba version --- .github/workflows/build_conda.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 54eaade7d..1f6b7e1bc 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -42,7 +42,7 @@ jobs: - name: Conda install and set libmamba conda solver run: | - conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda install -n base -c conda-forge conda-libmamba-solver=25.* libmambapy libmamba conda config --set solver libmamba - name: Conda install conda-build @@ -96,7 +96,7 @@ jobs: - name: Conda install and set libmamba conda solver run: | - conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba + conda install -n base -c conda-forge conda-libmamba-solver=25.* libmambapy libmamba conda config --set solver libmamba - name: Conda install conda-build @@ -116,7 +116,7 @@ jobs: path: /app/fre-cli-tarball/noarch - name: Test fre-cli conda package - timeout-minutes: 20 + #timeout-minutes: 20 run: | pkg=$(find /app/fre-cli-tarball/noarch -name 'fre-cli-*.conda' | head -n 1) From 58017b51ed2a882cb0115f21a62491ebda9860b8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 14 Oct 2026 10:42:35 -0400 Subject: [PATCH 47/47] #874 revert changes --- .github/workflows/build_conda.yml | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 1f6b7e1bc..ee81c0811 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -119,35 +119,11 @@ jobs: #timeout-minutes: 20 run: | pkg=$(find /app/fre-cli-tarball/noarch -name 'fre-cli-*.conda' | head -n 1) - + if [ -z "$pkg" ]; then echo "Could not find uploaded fre-cli conda package" exit 1 fi echo "Testing the package: $pkg" - # install package in conda environment - conda create -y -n fre-cli python - conda activate fre-cli - conda install -y "$pkg" - fre --help - fre app --help - fre catalog --help - fre check --help - fre cmor --help - fre list --help - fre make --help - fre pp --help - fre run --help - fre yamltools --help - - #run: | - # pkg=$(find /app/fre-cli-tarball/noarch -name 'fre-cli-*.conda' | head -n 1) - # - # if [ -z "$pkg" ]; then - # echo "Could not find uploaded fre-cli conda package" - # exit 1 - # fi - # - # echo "Testing the package: $pkg" - # conda -v build --test "$pkg" + conda -v build --test "$pkg"