diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daa4806..9c7e13a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: docs: name: docs - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - uses: mamba-org/setup-micromamba@v1 diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 139f1a0..9dbcb2c 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -37,7 +37,7 @@ jobs: with: activate-environment: test environment-file: environment.yml - python-version: 3.8 + python-version: 3.12 auto-activate-base: false - name: Conda check shell: bash -l {0} diff --git a/.github/workflows/readthedocs-pr.yml b/.github/workflows/readthedocs-pr.yml index 7939a4c..3591d87 100644 --- a/.github/workflows/readthedocs-pr.yml +++ b/.github/workflows/readthedocs-pr.yml @@ -2,7 +2,7 @@ # This does NOT trigger a build of the documentation, this is handled through webhooks. name: Read the Docs PR Preview on: - pull_request: + pull_request_target: types: - opened - synchronize @@ -19,6 +19,6 @@ jobs: documentation-links: runs-on: ubuntu-latest steps: - - uses: readthedocs/actions/preview@v1 + - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1 with: project-slug: "pyflow-workflow-generator" \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..bf207ad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: + - repo: https://github.com/pycqa/isort + rev: 8.0.1 + hooks: + - id: isort + name: ci qa isort --check + args: ["--check-only", "."] + pass_filenames: false + require_serial: true + + - repo: https://github.com/psf/black + rev: 26.5.1 + hooks: + - id: black + name: ci qa black --check + args: ["--check", "."] + pass_filenames: false + require_serial: true + + - repo: https://github.com/pycqa/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + name: ci qa flake8 + args: ["."] + pass_filenames: false + require_serial: true diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d347971..ac90b82 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,6 +19,12 @@ conda: environment: docs/environment.yml build: - os: "ubuntu-20.04" + os: "ubuntu-22.04" tools: python: "mambaforge-4.10" + jobs: + # Install the checked-out source (e.g. the current PR) into the conda + # environment, so the docs are built against this code. Dependencies are + # already provided by docs/environment.yml, hence --no-deps. + post_install: + - pip install --no-deps . diff --git a/README.md b/README.md index 00fbb71..6d1dde2 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ ## Installation + To install pyflow using conda (including ecFlow): conda env create -n pyflow -f environment.yml @@ -43,8 +44,37 @@ To install pyflow using pip (requires a local installation of ecFlow): pip install pyflow-workflow-generator ## Documentation + The documentation can be found at . +## QA Checks (CI-equivalent) + +The CI `qa` job runs the following checks, in order: + +1. `isort --check .` +2. `black --check .` +3. `flake8 .` + +This repository includes a matching pre-commit configuration in +`.pre-commit-config.yaml` using standard upstream hooks, pinned to the +tool versions from the CI run: + +- `isort==8.0.1` +- `black==26.5.1` +- `flake8==7.3.0` + +Run locally: + +```bash +python -m pip install ".[dev]" +pre-commit run --all-files +``` + +The hooks are split into three sequential checks (isort, then black, then +flake8) to avoid conflicts and to match CI behavior. Running `isort` before +`black` prevents import-format churn, and running `flake8` last ensures linting +sees code after formatting checks. + ## License [Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities diff --git a/docs/_ext/ecflow_lexers.py b/docs/_ext/ecflow_lexers.py index a202c1c..1c2ee3f 100644 --- a/docs/_ext/ecflow_lexers.py +++ b/docs/_ext/ecflow_lexers.py @@ -33,7 +33,7 @@ class EcflowDefLexer(RegexLexer): bygroups(Keyword, Name.Constant), ), ( - r"(repeat)(\s+(?:date(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))", + r"(repeat)(\s+(?:date(?:time)?(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))", # noqa: E501 bygroups(Keyword, Name.Other, Name.Variable, Literal.Date), ), # Required diff --git a/docs/content/api-reference.rst b/docs/content/api-reference.rst index f7df6d5..d518e46 100644 --- a/docs/content/api-reference.rst +++ b/docs/content/api-reference.rst @@ -160,6 +160,14 @@ Repeat .. autoclass:: pyflow.attributes.RepeatDateList +.. _RepeatDateTime: + +.. autoclass:: pyflow.attributes.RepeatDateTime + +.. _RepeatDateTimeList: + +.. autoclass:: pyflow.attributes.RepeatDateTimeList + .. _RepeatDay: .. autoclass:: pyflow.attributes.RepeatDay diff --git a/docs/environment.yml b/docs/environment.yml index db553a6..a244d83 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -18,7 +18,6 @@ dependencies: - sphinx-rtd-theme - sphinx-copybutton - sphinx-tabs - - git+https://github.com/ecmwf/pyflow.git variables: QT_MAC_WANTS_LAYER: 1 diff --git a/docs/requirements.txt b/docs/requirements.txt index e4b5648..cc30bbd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ ipykernel nbsphinx +pypandoc_binary sphinx-rtd-theme==0.5.2 sphinx-copybutton==0.3.1 -sphinx-tabs -git+https://github.com/ecmwf/pyflow.git \ No newline at end of file +sphinx-tabs \ No newline at end of file diff --git a/pyflow/__init__.py b/pyflow/__init__.py index 305eb3d..20c0e80 100644 --- a/pyflow/__init__.py +++ b/pyflow/__init__.py @@ -30,6 +30,7 @@ RepeatDate, RepeatDateList, RepeatDateTime, + RepeatDateTimeList, RepeatDay, RepeatEnumerated, RepeatInteger, diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 559ceb4..f9c1efe 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -25,7 +25,7 @@ expression_from_json, make_expression, ) -from .importer import ecflow +from .importer import ecflow, supported from .state import aborted, active, complete, queued, submitted, suspended, unknown NO_TRIGGER = False @@ -641,6 +641,7 @@ def day_of_week(self): setattr(RepeatDate, day, property(lambda self: Eq(self.day_of_week, dow))) +@supported(">=5.12.0") class RepeatDateTime(Exportable): """ An attribute that allows a node to be repeated by a date+time value. @@ -658,11 +659,17 @@ class RepeatDateTime(Exportable): datetime.datetime(year=2019, month=12, day=31, hour=12, minute=0, second=0), datetime.timedelta(hours=12, minutes=0, seconds=0)) - Date and increment can also be strings:: + Start/End values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with + hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, + or simply a date ``yyyymmdd`` (the missing components are assumed to be 0), and increment can also be a string:: pyflow.RepeatDateTime('REPEAT_DATETIME', '20190101T120000', '20191231T120000', '12:00:00') + Note:: + + This repeat type is only supported in ecFlow 5.12.0 and later. + """ def __init__( @@ -727,6 +734,92 @@ def day_of_week(self): return Mod(Add(Div(self, 86400), 4), 7) +@supported(">=5.17.0") +class RepeatDateTimeList(Repeat): + """ + An attribute that allows a node to be repeated over a list of datetime values. + + Parameters: + name(str): The name of the repeat attribute. + values(list of datetime/date): The list of datetime/date values, as datetime/date objects or strings. + + Example:: + + pyflow.RepeatDateTimeList('REPEAT_DATETIME', + [datetime.date(year=2019, month=1, day=1), + datetime.datetime(year=2019, month=1, day=3, hour=12, minute=0, second=0)]) + + Values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with + hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, + or simply a date ``yyyymmdd`` (the missing components are assumed to be 0):: + + pyflow.RepeatDateTimeList('REPEAT_DATETIME', ['20190101T120000', '20190103']) + + Note:: + + This repeat type is only supported in ecFlow 5.17.0 and later. + """ + + def __init__(self, name, values): + if values is None: + raise ValueError("values cannot be None") + if not isinstance(values, list): + raise TypeError("values must be a list") + if isinstance(values, list) and not values: + raise ValueError("values cannot be an empty list") + if not all( + isinstance(value, (datetime.datetime, datetime.date, str)) + for value in values + ): + raise TypeError("values must be a list of datetime/date objects or strings") + + super().__init__(name, values) + + def _build(self, ecflow_parent): + # Format all datetime values as ISO 8601 basic format `yyyymmddTHHMMSS` + values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.values] + + repeat = ecflow.RepeatDateTimeList( + str(self.name), + values, + ) + + ecflow_parent.add_repeat(repeat) + + @property + def values(self): + """*list*: The list of datetime values.""" + return [ + x if isinstance(x, datetime.datetime) else as_date(x) for x in self.value + ] + + def __add__(self, other): + return Add(self, other) + + def __sub__(self, other): + return Sub(self, other) + + @property + def second(self): + """*int*: The second of the repeat datetime.""" + return Mod(self, 60) + + @property + def minute(self): + """*int*: The minute of the repeat datetime.""" + return Mod(Div(self, 60), 60) + + @property + def hour(self): + """*int*: The hour of the repeat datetime.""" + return Mod(Div(self, 3600), 24) + + @property + def day_of_week(self): + """*int*: The day of the week of the repeat datetime.""" + return Mod(Add(Div(self, 86400), 4), 7) + + def is_date(value): return ( isinstance(value, (datetime.date, datetime.datetime)) @@ -939,6 +1032,10 @@ class InLimit(Attribute): Parameters: value(str,Limit_): The name of the limit or a limit object. + path(str): The optional path to the limit if the limit is not in the same node as the InLimit attribute. + tokens(int): The number of tokens to consume from the limit when a task is submitted. + limit_this_node_only(bool): Whether the limit should only apply to current node. + limit_submission(bool): Whether the limit should only apply to submissions Example:: @@ -946,8 +1043,19 @@ class InLimit(Attribute): pyflow.InLimit(l) """ - def __init__(self, value): + def __init__( + self, + value: str | Limit, + path: str = "", + tokens: int = 1, + limit_this_node_only: bool = False, + limit_submission: bool = False, + ): super().__init__("_" + str(value), value) + self.path = path + self.tokens = tokens + self.limit_this_node_only = limit_this_node_only + self.limit_submission = limit_submission def _build(self, ecflow_parent): value = self.value @@ -955,9 +1063,32 @@ def _build(self, ecflow_parent): return if isinstance(value, Limit): value = value.fullname.split(":") - ecflow_parent.add_inlimit(ecflow.InLimit(value[1], value[0])) + if self.path: + if self.path != value[0]: + raise ValueError( + "InLimit path {} does not match limit path {}".format( + self.path, value[0] + ) + ) + ecflow_parent.add_inlimit( + ecflow.InLimit( + value[1], + value[0], + self.tokens, + self.limit_this_node_only, + self.limit_submission, + ) + ) else: - ecflow_parent.add_inlimit(ecflow.InLimit(str(value))) + ecflow_parent.add_inlimit( + ecflow.InLimit( + str(value), + self.path, + self.tokens, + self.limit_this_node_only, + self.limit_submission, + ) + ) class Inlimit(InLimit): diff --git a/pyflow/host.py b/pyflow/host.py index 9f407a5..106c7d7 100644 --- a/pyflow/host.py +++ b/pyflow/host.py @@ -437,6 +437,9 @@ def preamble_error_function(self, ecflowpath, exit_hook=None): set +x # Define a error handler ERROR() { + export EXIT_REASON="$1" + export EXIT_DETAIL="$2" + export EXIT_RC="${3:-1}" export PATH=%(ecf_path)s:$PATH set +eu # Clear -eu flag, so we don't fail wait # wait for background process to stop @@ -454,11 +457,11 @@ def preamble_error_function(self, ecflowpath, exit_hook=None): export SIGNAL_LIST='%(signal_list)s' for signal in $SIGNAL_LIST; do - trap "ERROR $signal \\"Signal $(kill -l $signal) ($signal) received \\"" $signal + trap "rc=\\$?; ERROR $signal \\"Signal $(kill -l $signal) ($signal) received\\" \\"\\$rc\\"" $signal done # Trap any calls to exit and errors caught by the -e flag - trap ERROR 0 + trap 'rc=$?; ERROR EXIT "" "$rc"' 0 set -x """) % {"ecf_path": ecflowpath, "signal_list": signal_list}) # noqa: E501 return script diff --git a/pyflow/importer.py b/pyflow/importer.py index 870c7b8..bda753b 100644 --- a/pyflow/importer.py +++ b/pyflow/importer.py @@ -1,5 +1,9 @@ +import functools import os import sys +import types + +from packaging.specifiers import SpecifierSet try: import ecflow @@ -30,3 +34,78 @@ raise ImportError( "Could not find ecflow Python library, try to set ECFLOW_DIR environment variable to correct path" ) + + +def supported(specifier: str, current: str = ecflow.__version__): + """ + A decorator that ensures the decorated class can only be used when + the available ecFlow version satisfies ``specifier``. + + Every method and property of the class is wrapped so that invoking it + (e.g. instantiating the class via ``__init__``, or accessing a property) + raises :class:`NotImplementedError` when ``current`` does not satisfy + ``specifier``. + + The version comparison is evaluated once, when the class is decorated + (i.e. at import time, using ``current`` which defaults to the version of + the imported ecFlow module). + The ``current`` argument allows tests to instrument the behaviour for an arbitrary version. + + Parameters: + specifier(str): A :class:`packaging.specifiers.SpecifierSet` string, e.g. ``">=5.12.0,<6.0.0"``. + current(str): The current version, e.g. ``"5.12.0"``. Defaults to the version of the imported ecFlow module. + """ + + # Evaluate the comparison once, since both versions are fixed for the + # lifetime of the decorated class. + is_supported = SpecifierSet(specifier).contains(current) + + def decorator(cls): + + # Define a wrapper factory that checks the supported version before calling the original function + def make_wrapper(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not is_supported: + raise NotImplementedError( + "{} functionality is only supported for ecFlow {}, but current version is {}".format( + cls.__name__, specifier, current + ) + ) + + return func(*args, **kwargs) + + return wrapper + + # Iterate over a copy as we mutate the class namespace while iterating. + for attr_name, attr_value in list(cls.__dict__.items()): + + if isinstance(attr_value, types.FunctionType): + # Wrapped plain methods directly. + setattr(cls, attr_name, make_wrapper(attr_value)) + + elif isinstance(attr_value, property): + # Ensure properties must remain properties, + # by wrapping accessors and rebuilding the property so attribute access keeps working. + setattr( + cls, + attr_name, + property( + make_wrapper(attr_value.fget) if attr_value.fget else None, + make_wrapper(attr_value.fset) if attr_value.fset else None, + make_wrapper(attr_value.fdel) if attr_value.fdel else None, + attr_value.__doc__, + ), + ) + + elif isinstance(attr_value, staticmethod): + # Unwrap the inner function, wrap it, and re-wrap as staticmethod. + setattr(cls, attr_name, staticmethod(make_wrapper(attr_value.__func__))) + + elif isinstance(attr_value, classmethod): + # Unwrap the inner function, wrap it, and re-wrap as classmethod. + setattr(cls, attr_name, classmethod(make_wrapper(attr_value.__func__))) + + return cls + + return decorator diff --git a/pyflow/nodes.py b/pyflow/nodes.py index 67b5a46..30e98c0 100644 --- a/pyflow/nodes.py +++ b/pyflow/nodes.py @@ -95,6 +95,25 @@ def json_build(cls, name, tree): json_build(Task, k, v) +def _normalize_exit_hook(hook): + if isinstance(hook, str): + return [hook] + if isinstance(hook, Script): + return [hook.value] + + try: + hook_items = list(hook) + except TypeError as exc: + raise TypeError( + "exit_hook must be a string, Script, or iterable containing those types" + ) from exc + + normalized = [] + for item in hook_items: + normalized.extend(_normalize_exit_hook(item)) + return normalized + + class DuplicateNodeError(RuntimeError): def __init__(self, parent, new, existing): super().__init__( @@ -937,15 +956,13 @@ def _add_single_node(self, node): super()._add_single_node(node) def _add_exit_hook(self, hook): - if isinstance(hook, str): - hook = [hook] - for hk in hook: - if hk not in self._exit_hook: - self._exit_hook.append(hk) + normalized_hook = _normalize_exit_hook(hook) + for hk in normalized_hook: + self._exit_hook.append(hk) # Check if properly initialised if "_nodes" in self.__dict__: for chld in self.executable_children: - chld._add_exit_hook(hook) + chld._add_exit_hook(normalized_hook) class AnchorFamily(AnchorMixin, Family): @@ -1203,15 +1220,13 @@ def _add_single_node(self, node): super()._add_single_node(node) def _add_exit_hook(self, hook): - if isinstance(hook, str): - hook = [hook] - for hk in hook: - if hk not in self._exit_hook: - self._exit_hook.append(hk) + normalized_hook = _normalize_exit_hook(hook) + for hk in normalized_hook: + self._exit_hook.append(hk) # Check if properly initialised if "_nodes" in self.__dict__: for chld in self.executable_children: - chld._add_exit_hook(hook) + chld._add_exit_hook(normalized_hook) class Task(Node): @@ -1401,12 +1416,10 @@ def task_purge_modules(self): return self.host.purge_modules or super().task_purge_modules() - def _add_exit_hook(self, hook: str): - if isinstance(hook, str): - hook = [hook] - for hk in hook: - if hk not in self._exit_hook: - self._exit_hook.append(hk) + def _add_exit_hook(self, hook): + normalized_hook = _normalize_exit_hook(hook) + for hk in normalized_hook: + self._exit_hook.append(hk) def generate_script(self): """ diff --git a/pyproject.toml b/pyproject.toml index 440f8ce..e71fb17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyflow-workflow-generator" -requires-python = ">=3.8" +requires-python = ">=3.12" authors = [ {name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int"}, ] @@ -22,14 +22,14 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", ] dynamic = ["version", "readme"] dependencies = [ "jinja2", + "packaging", "requests", ] @@ -42,6 +42,13 @@ dependencies = [ test = [ "pytest", ] + qa = [ + "black==26.5.1", + "flake8==7.3.0", + "isort==8.0.1", + "pre-commit", + ] + dev = ["pyflow-workflow-generator[test,qa]"] diagrams = [ "graphviz", ] @@ -61,8 +68,8 @@ where = ["."] exclude = ["tests"] [tool.setuptools_scm] -write_to = "pyflow/_version.py" -write_to_template = ''' +version_file = "pyflow/_version.py" +version_file_template = ''' # Do not change! Do not track in version control! __version__ = "{version}" ''' diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 594eac9..dec685f 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -554,6 +554,96 @@ def test_repeat_datetime(self): s.check_definition() + def test_repeat_datetimelist_basic_usage(self): + from datetime import datetime as dt + + i = dt(2000, 1, 1, 12, 0, 0) + j = dt(2000, 1, 2) + + input_tests = ( + ("A", [i]), + ("B", [i, j]), + ("C", ["20000103T120000", "20000104T000000"]), + ("D", [i, "20010105T123456"]), + ) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = ( + 'repeat datetimelist A "20000101T120000"', + 'repeat datetimelist B "20000101T120000" "20000102T000000"', + 'repeat datetimelist C "20000103T120000" "20000104T000000"', + 'repeat datetimelist D "20000101T120000" "20010105T123456"', + ) + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + + def test_repeat_datetimelist_allowing_date_and_datetime_object(self): + import datetime + + i = datetime.datetime(2000, 1, 1, 12, 34, 56) + j = datetime.date(2000, 1, 2) + + input_tests = (("A", [i, j]),) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + assert_ = 'repeat datetimelist A "20000101T123456" "20000102T000000"' + defn = str(s.ecflow_definition()) + assert assert_ in defn + + s.check_definition() + + def test_repeat_datetimelist_with_truncated_string_values(self): + input_tests = ( + ("A", ["20000101T01", "20000102T0102", "20000103T010203"]), + ("B", ["20000104", "20000105T12", "20010106T1234"]), + ) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = ( + 'repeat datetimelist A "20000101T010000" "20000102T010200" "20000103T010203"', + 'repeat datetimelist B "20000104T000000" "20000105T120000" "20010106T123400"', + ) + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + + def test_repeat_datetimelist_with_none_value(self): + with pytest.raises(ValueError, match="values cannot be None"): + pyflow.RepeatDateTimeList("N", None) + + def test_repeat_datetimelist_with_empty_values_list(self): + with pytest.raises(ValueError, match="values cannot be an empty list"): + pyflow.RepeatDateTimeList("E", []) + + def test_repeat_datetimelist_with_invalid_type_values_list(self): + with pytest.raises( + TypeError, match="values must be a list of datetime/date objects or strings" + ): + pyflow.RepeatDateTimeList("I", [20050101]) + + def test_repeat_datetimelist_with_literal_type_value(self): + from datetime import datetime as dt + + with pytest.raises(TypeError, match="values must be a list"): + pyflow.RepeatDateTimeList("I", dt(2000, 1, 1, 0, 0, 0)) + def test_repeat_date_list(self): i = date(year=2019, month=12, day=31) j = date(year=2020, month=1, day=1) diff --git a/tests/test_families.py b/tests/test_families.py index 7944c60..3c5d04c 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -1,4 +1,6 @@ -from pyflow import AnchorFamily, Family, Limit, Suite, Task, Variable +import pytest + +from pyflow import AnchorFamily, Family, Limit, Script, Suite, Task, Variable def test_families(): @@ -99,7 +101,7 @@ def test_files_locations(): assert t7.deploy_path == "/a/base/path/f5/f6/t7.ecf" -def test_exit_hook(): +def test_exit_hook_strings(): """ Propagate exit hook to children of a given family """ @@ -110,7 +112,8 @@ def test_exit_hook(): t5 = Task("t5") - with Suite("S"): + common_hook = "one_hook" + with Suite("S", exit_hook=common_hook): with Family("f", exit_hook="hook_f", tasks=t5) as f: Limit("limit1", 15) Variable("VARIABLE1", 1234) @@ -118,17 +121,99 @@ def test_exit_hook(): with Family("f2", families=f3, exit_hook="hook_f2") as f2: t2 = Task("t2", exit_hook="hook_t2") - assert f._exit_hook == ["hook_f"] - assert t1._exit_hook == ["hook_f", "hook_t1"] - assert f2._exit_hook == ["hook_f", "hook_f2"] - assert t2._exit_hook == ["hook_f", "hook_f2", "hook_t2"] - assert f3._exit_hook == ["hook_f3", "hook_f", "hook_f2"] - assert t3._exit_hook == ["hook_f3", "hook_t3", "hook_f", "hook_f2"] - assert t4._exit_hook == ["hook_f3", "hook_f", "hook_f2"] - assert t5._exit_hook == ["hook_f"] + assert f._exit_hook == [common_hook, "hook_f"] + assert t1._exit_hook == [common_hook, "hook_f", "hook_t1"] + assert f2._exit_hook == [common_hook, "hook_f", "hook_f2"] + assert t2._exit_hook == [common_hook, "hook_f", "hook_f2", "hook_t2"] + assert f3._exit_hook == ["hook_f3", common_hook, "hook_f", "hook_f2"] + assert t3._exit_hook == ["hook_f3", "hook_t3", common_hook, "hook_f", "hook_f2"] + assert t4._exit_hook == ["hook_f3", common_hook, "hook_f", "hook_f2"] + assert t5._exit_hook == [common_hook, "hook_f"] + + +def test_exit_hook_scripts(): + """ + Support Script objects as exit hooks and propagate to children of a given family + """ + script_hook_f3 = Script(["hook_f3_line_1", "hook_f3_line_2"]) + script_hook_t3 = Script("hook_t3_line") + with Family("f3", exit_hook=script_hook_f3) as f3: + t3 = Task("t3", exit_hook=script_hook_t3) + t4 = Task("t4") + + t5 = Task("t5") + + common_hook = Script("common_hook_line") + hook_f = Script("hook_f_line") + hook_t1 = Script("hook_t1_line") + hook_f2 = Script("hook_f2_line") + hook_t2 = Script("hook_t2_line") + with Suite("S", exit_hook=common_hook): + with Family("f", exit_hook=hook_f, tasks=t5) as f: + Limit("limit1", 15) + Variable("VARIABLE1", 1234) + t1 = Task("t1", exit_hook=hook_t1) + with Family("f2", families=f3, exit_hook=hook_f2) as f2: + t2 = Task("t2", exit_hook=hook_t2) + + assert f._exit_hook == ["common_hook_line", "hook_f_line"] + assert t1._exit_hook == ["common_hook_line", "hook_f_line", "hook_t1_line"] + assert f2._exit_hook == ["common_hook_line", "hook_f_line", "hook_f2_line"] + assert t2._exit_hook == [ + "common_hook_line", + "hook_f_line", + "hook_f2_line", + "hook_t2_line", + ] + assert f3._exit_hook == [ + "hook_f3_line_1\nhook_f3_line_2", + "common_hook_line", + "hook_f_line", + "hook_f2_line", + ] + assert t3._exit_hook == [ + "hook_f3_line_1\nhook_f3_line_2", + "hook_t3_line", + "common_hook_line", + "hook_f_line", + "hook_f2_line", + ] + assert t4._exit_hook == [ + "hook_f3_line_1\nhook_f3_line_2", + "common_hook_line", + "hook_f_line", + "hook_f2_line", + ] + assert t5._exit_hook == ["common_hook_line", "hook_f_line"] + + +def test_exit_hook_list_strings_preserves_duplicate_lines(): + """ + Lists of string exit hooks should preserve ordering and duplicate lines. + + Changed in v3.7.0. In previous versions duplication in exit_hook scripts were not allowed. + """ + + parent_lines = ["echo pre_cleanup", "echo duplicate_line"] + child_lines = ["echo duplicate_line", "echo post_cleanup"] + + with Suite("S", exit_hook=parent_lines): + with Family("f", exit_hook=child_lines) as f: + t1 = Task("t1") + + expected = [ + "echo pre_cleanup", + "echo duplicate_line", + "echo duplicate_line", + "echo post_cleanup", + ] + + assert f._exit_hook == expected + assert t1._exit_hook == expected if __name__ == "__main__": + # flake8: noqa from os import path import pytest diff --git a/tests/test_host.py b/tests/test_host.py index 9978600..def9ba7 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -446,6 +446,16 @@ def test_traps(): assert signal_list1 in s1 assert signal_list2 in s2 + # Ensure ERROR exposes contract variables used by custom exit hooks. + assert 'export EXIT_REASON="$1"' in s1 + assert 'export EXIT_DETAIL="$2"' in s1 + assert 'export EXIT_RC="${3:-1}"' in s1 + + # Ensure traps pass the command return code through to ERROR. + assert 'trap "rc=\\$?; ERROR $signal' in s1 + assert '\\"\\$rc\\"" $signal' in s1 + assert 'trap \'rc=$?; ERROR EXIT "" "$rc"\' 0' in s1 + @pytest.mark.parametrize( "key,expected_class,kwargs", diff --git a/tests/test_inlimits.py b/tests/test_inlimits.py index a0e9f7e..a4866fc 100644 --- a/tests/test_inlimits.py +++ b/tests/test_inlimits.py @@ -1,4 +1,6 @@ -from pyflow import Limits, Suite, Tasks +import pytest + +from pyflow import InLimit, Limit, Limits, Suite, Tasks def test_inlimits(): @@ -14,6 +16,29 @@ def test_inlimits(): s.generate_node() +@pytest.mark.parametrize( + "options", + [ + {}, + {"path": "/s"}, + {"value": "tlimit"}, + {"value": "tlimit", "path": "/s"}, + {"value": "tlimit", "tokens": 1}, + {"value": "tlimit", "limit_this_node_only": True}, + {"value": "tlimit", "limit_submission": True}, + ], +) +def test_options(options): + with Suite("s") as s: + limit = Limit("tlimit", value=3) + if "value" not in options: + options["value"] = limit + Tasks("t", "t2", inlimits=InLimit(**options)) + + s.check_definition() + s.generate_node() + + if __name__ == "__main__": from os import path diff --git a/tests/test_supported.py b/tests/test_supported.py new file mode 100644 index 0000000..42e2000 --- /dev/null +++ b/tests/test_supported.py @@ -0,0 +1,218 @@ +import ecflow +import pytest +from packaging import version + +import pyflow +from pyflow.importer import supported + + +def make_widget(specifier, current): + """Build a class decorated with an explicitly instrumented specifier/current version.""" + + @supported(specifier, current=current) + class Widget: + def __init__(self, value=0): + self.value = value + + def a_method(self): + "A Widget method." + return self.value + + @property + def a_property(self): + "A Widget property." + return self.value * 2 + + @staticmethod + def a_static(): + "A Widget static method." + return 42 + + @classmethod + def a_classmethod(cls): + "A Widget class method." + return cls.__name__ + + return Widget + + +def test_widget_use_is_disallowed_on_older_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + with pytest.raises(NotImplementedError): + Widget() + + +def test_widget_use_is_allowed_on_newer_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.13.0") + assert Widget(3).value == 3 + + +def test_widget_use_is_allowed_on_boundary_version(): + """The exact lower-bound version must be accepted (>= semantics).""" + Widget = make_widget(specifier=">=5.12.0", current="5.12.0") + assert Widget(7).value == 7 + + +def test_widget_use_is_disallowed_on_just_below_boundary(): + Widget = make_widget(specifier=">=5.12.0", current="5.11.9") + with pytest.raises(NotImplementedError): + Widget() + + +def test_error_message_contains_specifier_and_name(): + Widget = make_widget(specifier=">=5.20.1", current="5.6.7") + with pytest.raises(NotImplementedError) as exc: + Widget() + message = str(exc.value) + assert "Widget" in message + assert ">=5.20.1" in message + assert "5.6.7" in message + + +def test_methods_are_guarded_when_unsupported(): + """Ensure non-__init__ methods also raise when unsupported.""" + + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + # __init__ is guarded too, so build with an unguarded instance via __new__. + w = Widget.__new__(Widget) + with pytest.raises(NotImplementedError): + w.a_method() + + +def test_methods_work_when_supported(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(5) + assert w.a_method() == 5 + + +def test_properties_remain_properties(): + """ + Ensure properties behave as properties (return a computed value) after decoration. + """ + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(4) + assert w.a_property == 8 + assert isinstance(type(w).__dict__["a_property"], property) + + +def test_properties_are_disallowed_on_older_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + w = Widget.__new__(Widget) + with pytest.raises(NotImplementedError): + _ = w.a_property + + +def test_properties_are_allowed_on_newer_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(4) + assert w.a_property == 8 + + +def test_wraps_preserves_metadata(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert Widget.a_method.__name__ == "a_method" + assert Widget.a_method.__doc__ == "A Widget method." + assert Widget.__dict__["a_property"].__doc__ == "A Widget property." + + +def test_current_defaults_to_installed_ecflow_version(): + """Without an explicit ``current``, the installed ecFlow version is used.""" + + @supported(">=9999.0.0") + class Future: + def __init__(self): + pass + + with pytest.raises(NotImplementedError): + Future() + + @supported(">=0.0.1") + class Ancient: + def __init__(self): + self.ok = True + + assert Ancient().ok is True + + +def test_compound_specifier_excludes_upper_bound(): + """A compound specifier like >=5.12.0,<5.13.0 rejects versions outside the range.""" + Widget = make_widget(specifier=">=5.12.0,<5.13.0", current="5.13.0") + with pytest.raises(NotImplementedError): + Widget() + + +def test_compound_specifier_allows_version_in_range(): + Widget = make_widget(specifier=">=5.12.0,<5.13.0", current="5.12.5") + assert Widget(1).value == 1 + + +# ----------------------------------------------------------------------------- + + +def _installed_below(min_version): + return version.parse(ecflow.__version__) < version.parse(min_version) + + +def test_repeat_datetime_builds_on_installed_ecflow(): + if _installed_below("5.12.0"): + pytest.skip("RepeatDateTime requires ecFlow >= 5.12.0") + + with pyflow.Suite("s"): + with pyflow.Task("t"): + repeat = pyflow.RepeatDateTime( + "REPEAT_DATETIME", + "20190101T120000", + "20191231T120000", + "12:00:00", + ) + + assert repeat.name == "REPEAT_DATETIME" + assert not callable(repeat.second) + assert isinstance(type(repeat).__dict__["second"], property) + + +def test_repeat_datetimelist_builds_on_installed_ecflow(): + if _installed_below("5.17.0"): + pytest.skip("RepeatDateTimeList requires ecFlow >= 5.17.0") + + with pyflow.Suite("s"): + with pyflow.Task("t"): + repeat = pyflow.RepeatDateTimeList( + "REPEAT_DATETIME", ["20190101T120000", "20190103"] + ) + + assert repeat.name == "REPEAT_DATETIME" + assert not callable(repeat.values) + assert isinstance(type(repeat).__dict__["values"], property) + + +def test_staticmethod_is_guarded_when_unsupported(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + with pytest.raises(NotImplementedError): + Widget.a_static() + + +def test_staticmethod_works_when_supported(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert Widget.a_static() == 42 + + +def test_classmethod_is_guarded_when_unsupported(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + with pytest.raises(NotImplementedError): + Widget.a_classmethod() + + +def test_classmethod_works_when_supported(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert Widget.a_classmethod() == "Widget" + + +def test_staticmethod_remains_static_after_decoration(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert isinstance(Widget.__dict__["a_static"], staticmethod) + + +def test_classmethod_remains_classmethod_after_decoration(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert isinstance(Widget.__dict__["a_classmethod"], classmethod)