874.refactor fre yamltools

fre/list_/frelist.py

@@ -44,9 +44,14 @@ def platforms(yamlfile):
               type=str,
               help="Experiment to be post-processed",
               required=True)
-def pp_components(yamlfile, experiment):
+@click.option("-a",
+              "--application",
+              type=str,
+              help="Post-processing application name.",
+              required=True)
+def pp_components(yamlfile, experiment, application):
     """ - List components to be post-processed for a defined experiment"""
-    list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment)
+    list_pp_components_script.list_ppcomps_subtool(yamlfile, experiment, application)
 
 @list_cli.command()
 @click.option("-y",

fre/list_/list_experiments_script.py

@@ -5,15 +5,7 @@
 """
 
 import logging
-import yaml
-
-from fre.yamltools.info_parsers import pp_info_parser as ppip
-
-## to-do: figure out validation
-# Imports associated with commented block
-# from pathlib import Path
-# from fre.yamltools import helpers
-# from fre.yamltools import combine_yamls_script as cy
+from fre.yamltools import combine_yamls_script_new as cy
 
 fre_logger = logging.getLogger(__name__)
 
@@ -29,44 +21,25 @@ def list_experiments_subtool(yamlfile: str):
     platform = None
     target = None
 
-    # Combine model
-    # Create pp yaml instance
-    yamldict = ppip.InitPPYaml(yamlfile, exp, platform, target)
-    yaml_str = yamldict.combine_model()
-    yaml_dict = yaml.load(yaml_str, Loader = yaml.Loader)
+    yaml_dict = cy.yamltools_combine_subtool(yamls = yamlfile,
+                                             experiment = exp,
+                                             platform = platform,
+                                             target = target,
+                                             output = None,
+                                             no_clean = True)
 
-## COULD HAVE been one way to validate but section we'd want to parse was
-## cleaned in final/"combined" yaml information
-## Currently not a way to validate model yaml information because we only
-## have schemas for the final "combined" compile or pp information (both
-## of which remove the "experiments" section I believe
-#    exp = yamlfile.split("/")[-1].split(".")[0]
-#    platform = "None"
-#    target = "None"
-#
-#    # Combine model / experiment
-#    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
-#                                    experiment = exp,
-#                                    platform = platform,
-#                                    target = target,
-#                                    use = "compile",
-#                                    output = None)
-#
-#    # Validate combined yaml information
-#    frelist_dir = Path(__file__).resolve().parents[2]
-#    schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json"
-#    # from fre.yamltools
-#    helpers.validate_yaml(yml_dict, schema_path)
+#### TO-DO: CURRENTLY NO YAML PIECE VALIDATION
+### No way to validate just the model yaml
 
     # set logger level to INFO
     former_log_level = fre_logger.level
     fre_logger.setLevel(logging.INFO)
 
     # log the experiment names, which should show up on screen for sure
     fre_logger.info("Experiments found:")
-    for i in yaml_dict.get("experiments"):
-        fre_logger.info('   - %s', i.get("name"))
-    fre_logger.info("\n")
+
+    for i in yaml_dict["experiments"].keys():
+        fre_logger.info('   - %s', i)
 
     # set logger back to normal level
     fre_logger.setLevel(former_log_level)

fre/list_/list_platforms_script.py

@@ -6,7 +6,8 @@
 
 from pathlib import Path
 import logging
-from fre.yamltools import combine_yamls_script as cy
+from fre.yamltools import combine_yamls_script_new as cy
+from fre.list_ import list_yamls_script as ly
 from fre.yamltools import helpers
 
 fre_logger = logging.getLogger(__name__)
@@ -22,17 +23,33 @@ def list_platforms_subtool(yamlfile: str):
     former_log_level = fre_logger.level
     fre_logger.setLevel(logging.INFO)
 
-    exp = yamlfile.split("/")[-1].split(".")[0]
+    exp = None
     platform = None
     target = None
 
-    # Combine model / experiment
-    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
-                                    experiment = exp,
-                                    platform = platform,
-                                    target = target,
-                                    use = "compile",
-                                    output = None)
+#    model_yf_path = Path(yamlfile).resolve().parent
+#    with open(yamlfile, 'r') as yf:
+#        yml = yaml.safe_load(yf)
+#
+#    platform_yf_path = f"{model_yf_path}/{yml['build']['platformYaml']}"
+
+    yml_list = ly.list_yamls_subtool(yamlfile = yamlfile,
+                                     experiment = exp,
+                                     application = None)
+
+#    # Combine model / experiment
+#    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
+#                                    experiment = exp,
+#                                    platform = platform,
+#                                    target = target,
+#                                    use = "compile",
+#                                    output = None)
+    yml_dict = cy.yamltools_combine_subtool(yamls = yml_list,
+                                             experiment = exp,
+                                             platform = platform,
+                                             target = target,
+                                             output = None,
+                                             no_clean = False)
 
     # Validate the yaml
     fre_pkg_dir = Path(__file__).resolve().parents[1]
@@ -41,8 +58,7 @@ def list_platforms_subtool(yamlfile: str):
     helpers.validate_yaml(yml_dict, schema_path)
 
     fre_logger.info("Platforms available:")
-    for i in yml_dict.get("platforms"):
+    for i in yml_dict["platforms"]:
         fre_logger.info('    - %s', i.get("name"))
-    fre_logger.info("\n")
 
     fre_logger.setLevel(former_log_level)

fre/list_/list_pp_components_script.py

@@ -19,19 +19,22 @@
 
 from pathlib import Path
 import logging
-from fre.yamltools import combine_yamls_script as cy
+from fre.yamltools import combine_yamls_script_new as cy
+from fre.list_ import list_yamls_script as ly
 from fre.yamltools import helpers
 
 fre_logger = logging.getLogger(__name__)
 
-def list_ppcomps_subtool(yamlfile: str, experiment: str):
+def list_ppcomps_subtool(yamlfile: str, experiment: str, application: str):
     """
     List_ppcomps_subtool lists the components to be post-processed.
 
     :param yamlfile: is the path to the model.yaml configuration file
     :type yamlfile: str
     :param experiment: is the experiment name defined in the model.yaml
     :type experiment: str
+    :param application:
+    :type applicatin: str
     """
     # set logger level to INFO
     former_log_level = fre_logger.level
@@ -40,30 +43,42 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str):
     exp = experiment
     platform = None
     target = None
+    app = application
+
+    yml_list = ly.list_yamls_subtool(yamlfile = yamlfile,
+                                     experiment = exp,
+                                     application = app)
+
+#    # Combine model / experiment
+#    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
+#                                    experiment = exp,
+#                                    platform = platform,
+#                                    target = target,
+#                                    use = "pp",
+#                                    output = None)
+    yml_dict = cy.yamltools_combine_subtool(yamls = yml_list,
+                                            experiment = exp,
+                                            platform = platform,
+                                            target = target,
+                                            output = None,
+                                            no_clean = False)
 
-    # Combine model / experiment
-    yml_dict = cy.consolidate_yamls(yamlfile = yamlfile,
-                                    experiment = exp,
-                                    platform = platform,
-                                    target = target,
-                                    use = "pp",
-                                    output = None)
 
     # Validate combined yaml information
     frelist_dir = Path(__file__).resolve().parents[2]
     schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_pp.json"
     # from fre.yamltools
     helpers.validate_yaml(yml_dict, schema_path)
 
+    comp_info = yml_dict["postprocess"]["components"]
     # log the experiment names, which should show up on screen for sure
     fre_logger.info("Components to be post-processed:")
-    for i in yml_dict["postprocess"]["components"]:
-        if "postprocess_on" in i:
-            if i.get("postprocess_on") is True:
-                fre_logger.info('   - %s', i.get("type"))
+    for i in comp_info.keys():
+        if "postprocess_on" in comp_info[i]:
+            if comp_info[i].get("postprocess_on") is True:
+                fre_logger.info('   - %s', i)
         else:
-            fre_logger.info('   - %s', i.get("type"))
-    fre_logger.info("\n")
+            fre_logger.info('   - %s', i)
 
     # set logger back to normal level
     fre_logger.setLevel(former_log_level)

fre/list_/list_yamls_script.py

@@ -11,7 +11,9 @@
 """
 import logging
 from pathlib import Path
+import sys
 import yaml
+import click
 
 fre_logger = logging.getLogger(__name__)
 
@@ -25,6 +27,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str):
     :type experiment: str
     :param application: is the application name
     :type application: str
+    :return: is a comma separated string of yaml files (absolute paths)
+    :rtype: str
 
     :raise ValueError: if the experiment, application passed does not exist and 
                        if yaml files do not exist
@@ -40,8 +44,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str):
     with open(yamlfile, 'r', encoding="utf-8") as yf:
         yaml_dict = yaml.load(yf, Loader = yaml.Loader)
 
-    compile_data = yaml_dict["build"].get("compileYaml")
     platform_data = yaml_dict["build"].get("platformYaml")
+    compile_data = yaml_dict["build"].get("compileYaml")
     exp_data = yaml_dict["experiments"].get(experiment)
 
     yamls = [model_yaml]
@@ -68,15 +72,15 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str):
                 else:
                     yamls.append(exp_data[a])
         else:
-            yamls.extend([compile_data, platform_data])
+            yamls.extend([platform_data, compile_data])
             for value in exp_data.values():
                 if isinstance(value, list):
                     yamls.extend(value)
                 else:
                     yamls.append(value)
     else:
         fre_logger.info("No experiment name passed. Will only provide YAMLs related to compilation.")
-        yamls.extend([compile_data, platform_data])
+        yamls.extend([platform_data, compile_data])
 
     yamls_full_path = ""
     # Add full path for yaml configurations
@@ -94,13 +98,13 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str):
         fre_logger.info("  - %s", y)
 
 ### Might add this in when fre yamltools combine-yamls is refactored
-#    fre_logger.info("")
-#    fre_logger.info('If combining these yamls, there are 2 options:')
-#    fre_logger.info('   1. Pipe this tool to "fre yamltools combine"')
-#    fre_logger.info('   2. Copy and paste this string (including quotes)
-#                           as the -y option in "fre yamltools combine -y <yamls>:')
-#    fre_logger.info('       "%s"', yamls_full_path)
-#    fre_logger.info("")
+    fre_logger.info("")
+    fre_logger.info('If combining these yamls, there are 2 options:')
+    fre_logger.info('   1. Pipe this tool to "fre yamltools combine"')
+    fre_logger.info('   2. Copy and paste this string (including quotes) '
+                           'as the -y option in "fre yamltools combine -y <yamls>:')
+    fre_logger.info('       "%s"', yamls_full_path)
+    fre_logger.info("")
     fre_logger.setLevel(former_log_level)
 
     # Check if the paths exist; give warning
@@ -113,4 +117,8 @@ def list_yamls_subtool(yamlfile: str, experiment: str, application:str):
     if "True" in fail:
         raise ValueError(" *** PROVIDE THE MISSING YAML CONFIGURATIONS ***")
 
+    # if not piped to fre yamltools combine, no need to show all the yamls
+    if not sys.stdout.isatty():
+        click.echo(yamls_full_path)
+
     return yamls_full_path

fre/list_/tests/test_list_experiments_script.py

@@ -4,40 +4,36 @@
 from pathlib import Path
 
 import pytest
-import yaml
 
 from fre.list_ import list_experiments_script
 from fre.yamltools import helpers
 
-
 # SET-UP
-TEST_DIR = Path("fre/make/tests")
-NM_EXAMPLE = Path("null_example")
-YAMLFILE = "null_model.yaml"
+TEST_DIR = Path("fre/list_/tests/yamls")
+YAMLFILE = "model.yaml"
 EXP_NAME = "None"
 
 # yaml file checks
 def test_modelyaml_exists():
     ''' Test that model yaml exists '''
-    assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists()
+    assert Path(f"{TEST_DIR}/{YAMLFILE}").exists()
 
 # Test whole tool
 def test_exp_list(caplog):
     ''' Test fre list exps subtool '''
-    list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+    list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}")
 
     # check the logging output
     check_out = [ 'Experiments found:',
-                  '   - null_model_full',
-                  '   - null_model_0',
-                  '   - null_model_1',
-                  '   - null_model_2'     ]
+                  '   - experiment1',
+                  '   - experiment2' ]
     for i in check_out:
         assert i in caplog.text
 
     # make sure the level is INFO
     for record in caplog.records:
-        assert record.levelname == "INFO"
+        if record.name.startswith("fre"):
+            assert record.levelname == "INFO"
 
 # Test validation
 @pytest.mark.skip(
@@ -48,10 +44,10 @@ def test_exp_list(caplog):
 )
 def test_yamlvalidate():
     ''' Test yaml is being validated '''
-    yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+    yamlfilepath = Path(f"{TEST_DIR}/{YAMLFILE}")
 
     # Combine model / experiment
-    yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+    yml_dict = list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{YAMLFILE}")
 
     # Validate and capture output
     assert helpers.validate_yaml(yml_dict, VAL_SCHEMA)