update fre list subtools and tests for uwtools integration

.github/workflows/build_conda.yml

@@ -40,7 +40,13 @@ jobs:
           echo "printing conda config just in case"
           conda config --show
 
+      - name: Conda install and set libmamba conda solver
+        run: |
+          conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba
+          conda config --set solver libmamba
+
       - name: Conda install conda-build
+        timeout-minutes: 10
         run: |
           echo "conda install conda-build"
           conda install conda-forge::conda-build
@@ -88,11 +94,22 @@ jobs:
           echo "printing conda config just in case"
           conda config --show
 
+      - name: Conda install and set libmamba conda solver
+        run: |
+          conda install -n base -c conda-forge conda-libmamba-solver libmambapy libmamba
+          conda config --set solver libmamba
+
       - name: Conda install conda-build
         run: |
           echo "conda install conda-build"
           conda install conda-forge::conda-build
 
+      - name: Check conda-build version and supported package formats
+        run: |
+          conda --version
+          conda build --version
+          conda build --help
+
       - name: Download tarball
         uses: actions/download-artifact@v5
         with:

fre/list_/frelist.py

@@ -44,9 +44,14 @@ def platforms(yamlfile):
               type=str,
               help="Experiment to be post-processed",
               required=True)
-def pp_components(yamlfile, experiment):
+@click.option("-a",
+              "--application",
+              type=str,
+              help="Post-processing application name.",
+              required=True)
+def pp_components(yamlfile, experiment, application):
     """ - List components to be post-processed for a defined experiment"""
-    list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment)
+    list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment, application)
 
 @list_cli.command()
 @click.option("-y",

fre/list_/list_experiments_script.py

@@ -5,15 +5,7 @@
 """
 
 import logging
-import yaml
-
-from fre.yamltools.info_parsers import pp_info_parser as ppip
-
-## to-do: figure out validation
-# Imports associated with commented block
-# from pathlib import Path
-# from fre.yamltools import helpers
-# from fre.yamltools import combine_yamls_script as cy
+from fre.yamltools import combine_yamls_script_new as cy
 
 fre_logger = logging.getLogger(__name__)
 
@@ -29,44 +21,25 @@ def list_experiments_subtool(yamlfile: str):
     platform = None
     target = None
 
-    # Combine model
-    # Create pp yaml instance
-    yamldict = ppip.InitPPYaml(yamlfile, exp, platform, target)
-    yaml_str = yamldict.combine_model()
-    yaml_dict = yaml.load(yaml_str, Loader = yaml.Loader)
+    yaml_dict = cy.yamltools_combine_subtool(yamls = yamlfile,
+                                             experiment = exp,
+                                             platform = platform,
+                                             target = target,
+                                             output = None,
+                                             no_clean = True)
 
-## COULD HAVE been one way to validate but section we'd want to parse was
-## cleaned in final/"combined" yaml information
-## Currently not a way to validate model yaml information because we only
-## have schemas for the final "combined" compile or pp information (both
-## of which remove the "experiments" section I believe
-#    exp = yamlfile.split("/")[-1].split(".")[0]
-#    platform = "None"
-#    target = "None"
-#
-#    # Combine model / experiment
-#    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
-#                                    experiment = exp,
-#                                    platform = platform,
-#                                    target = target,
-#                                    use = "compile",
-#                                    output = None)
-#
-#    # Validate combined yaml information
-#    frelist_dir = Path(__file__).resolve().parents[2]
-#    schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json"
-#    # from fre.yamltools
-#    helpers.validate_yaml(yml_dict, schema_path)
+#### TO-DO: CURRENTLY NO YAML PIECE VALIDATION
+### No way to validate just the model yaml
 
     # set logger level to INFO
     former_log_level = fre_logger.level
     fre_logger.setLevel(logging.INFO)
 
     # log the experiment names, which should show up on screen for sure
     fre_logger.info("Experiments found:")
-    for i in yaml_dict.get("experiments"):
-        fre_logger.info('   - %s', i.get("name"))
-    fre_logger.info("\n")
+
+    for i in yaml_dict["experiments"].keys():
+        fre_logger.info('   - %s', i)
 
     # set logger back to normal level
     fre_logger.setLevel(former_log_level)

fre/list_/list_platforms_script.py

@@ -6,7 +6,8 @@
 
 from pathlib import Path
 import logging
-from fre.yamltools import combine_yamls_script as cy
+from fre.yamltools import combine_yamls_script_new as cy
+from fre.list_ import list_yamls_script as ly
 from fre.yamltools import helpers
 
 fre_logger = logging.getLogger(__name__)
@@ -22,17 +23,33 @@ def list_platforms_subtool(yamlfile: str):
     former_log_level = fre_logger.level
     fre_logger.setLevel(logging.INFO)
 
-    exp = yamlfile.split("/")[-1].split(".")[0]
+    exp = None
     platform = None
     target = None
 
-    # Combine model / experiment
-    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
-                                    experiment = exp,
-                                    platform = platform,
-                                    target = target,
-                                    use = "compile",
-                                    output = None)
+#    model_yf_path = Path(yamlfile).resolve().parent
+#    with open(yamlfile, 'r') as yf:
+#        yml = yaml.safe_load(yf)
+#
+#    platform_yf_path = f"{model_yf_path}/{yml['build']['platformYaml']}"
+
+    yml_list = ly.list_yamls_subtool(yamlfile = yamlfile,
+                                     experiment = exp,
+                                     application = None)
+
+#    # Combine model / experiment
+#    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
+#                                    experiment = exp,
+#                                    platform = platform,
+#                                    target = target,
+#                                    use = "compile",
+#                                    output = None)
+    yml_dict = cy.yamltools_combine_subtool(yamls = yml_list,
+                                             experiment = exp,
+                                             platform = platform,
+                                             target = target,
+                                             output = None,
+                                             no_clean = False)
 
     # Validate the yaml
     fre_pkg_dir = Path(__file__).resolve().parents[1]
@@ -41,8 +58,7 @@ def list_platforms_subtool(yamlfile: str):
     helpers.validate_yaml(yml_dict, schema_path)
 
     fre_logger.info("Platforms available:")
-    for i in yml_dict.get("platforms"):
+    for i in yml_dict["platforms"]:
         fre_logger.info('    - %s', i.get("name"))
-    fre_logger.info("\n")
 
     fre_logger.setLevel(former_log_level)

fre/list_/list_pp_components_script.py

@@ -19,19 +19,22 @@
 
 from pathlib import Path
 import logging
-from fre.yamltools import combine_yamls_script as cy
+from fre.yamltools import combine_yamls_script_new as cy
+from fre.list_ import list_yamls_script as ly
 from fre.yamltools import helpers
 
 fre_logger = logging.getLogger(__name__)
 
-def list_ppcomps_subtool(yamlfile: str, experiment: str):
+def list_ppcomps_subtool(yamlfile: str, experiment: str, application: str):
     """
     List_ppcomps_subtool lists the components to be post-processed.
 
     :param yamlfile: is the path to the model.yaml configuration file
     :type yamlfile: str
     :param experiment: is the experiment name defined in the model.yaml
     :type experiment: str
+    :param application:
+    :type applicatin: str
     """
     # set logger level to INFO
     former_log_level = fre_logger.level
@@ -40,30 +43,42 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str):
     exp = experiment
     platform = None
     target = None
+    app = application
+
+    yml_list = ly.list_yamls_subtool(yamlfile = yamlfile,
+                                     experiment = exp,
+                                     application = app)
+
+#    # Combine model / experiment
+#    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
+#                                    experiment = exp,
+#                                    platform = platform,
+#                                    target = target,
+#                                    use = "pp",
+#                                    output = None)
+    yml_dict = cy.yamltools_combine_subtool(yamls = yml_list,
+                                            experiment = exp,
+                                            platform = platform,
+                                            target = target,
+                                            output = None,
+                                            no_clean = False)
 
-    # Combine model / experiment
-    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
-                                    experiment = exp,
-                                    platform = platform,
-                                    target = target,
-                                    use = "pp",
-                                    output = None)
 
     # Validate combined yaml information
     frelist_dir = Path(__file__).resolve().parents[2]
     schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_pp.json"
     # from fre.yamltools
     helpers.validate_yaml(yml_dict, schema_path)
 
+    comp_info = yml_dict["postprocess"]["components"]
     # log the experiment names, which should show up on screen for sure
     fre_logger.info("Components to be post-processed:")
-    for i in yml_dict["postprocess"]["components"]:
-        if "postprocess_on" in i:
-            if i.get("postprocess_on") is True:
-                fre_logger.info('   - %s', i.get("type"))
+    for i in comp_info.keys():
+        if "postprocess_on" in comp_info[i]:
+            if comp_info[i].get("postprocess_on") is True:
+                fre_logger.info('   - %s', i)
         else:
-            fre_logger.info('   - %s', i.get("type"))
-    fre_logger.info("\n")
+            fre_logger.info('   - %s', i)
 
     # set logger back to normal level
     fre_logger.setLevel(former_log_level)

fre/list_/tests/test_list_experiments_script.py

@@ -4,40 +4,36 @@
 from pathlib import Path
 
 import pytest
-import yaml
 
 from fre.list_ import list_experiments_script
 from fre.yamltools import helpers
 
-
 # SET-UP
-TEST_DIR = Path("fre/make/tests")
-NM_EXAMPLE = Path("null_example")
-YAMLFILE = "null_model.yaml"
+TEST_DIR = Path("fre/list_/tests/yamls")
+YAMLFILE = "model.yaml"
 EXP_NAME = "None"
 
 # yaml file checks
 def test_modelyaml_exists():
     ''' Test that model yaml exists '''
-    assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists()
+    assert Path(f"{TEST_DIR}/{YAMLFILE}").exists()
 
 # Test whole tool
 def test_exp_list(caplog):
     ''' Test fre list exps subtool '''
-    list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+    list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}")
 
     # check the logging output
     check_out = [ 'Experiments found:',
-                  '   - null_model_full',
-                  '   - null_model_0',
-                  '   - null_model_1',
-                  '   - null_model_2'     ]
+                  '   - experiment1',
+                  '   - experiment2' ]
     for i in check_out:
         assert i in caplog.text
 
     # make sure the level is INFO
     for record in caplog.records:
-        assert record.levelname == "INFO"
+        if record.name.startswith("fre"):
+            assert record.levelname == "INFO"
 
 # Test validation
 @pytest.mark.skip(
@@ -48,10 +44,10 @@ def test_exp_list(caplog):
 )
 def test_yamlvalidate():
     ''' Test yaml is being validated '''
-    yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+    yamlfilepath = Path(f"{TEST_DIR}/{YAMLFILE}")
 
     # Combine model / experiment
-    yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+    yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}")
 
     # Validate and capture output
     assert helpers.validate_yaml(yml_dict, VAL_SCHEMA)