From 333c6e9c636649e8c50974c5b69dfa20b78798e3 Mon Sep 17 00:00:00 2001 From: Jenny Wong Date: Thu, 21 May 2026 13:33:32 +0000 Subject: [PATCH 01/32] Extend InLimit api --- pyflow/attributes.py | 44 +++++++++++++++++++++++++++++++++++++++--- tests/test_inlimits.py | 26 ++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 559ceb4..6ef5947 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -939,6 +939,10 @@ class InLimit(Attribute): Parameters: value(str,Limit_): The name of the limit or a limit object. + path(str): The optional path to the limit if the limit is not in the same node as the InLimit attribute. + tokens(int): The number of tokens to consume from the limit when a task is submitted. + limit_this_node_only(bool): Whether the limit should only apply to current node. + limit_submission(bool): Whether the limit should only apply to submissions Example:: @@ -946,8 +950,19 @@ class InLimit(Attribute): pyflow.InLimit(l) """ - def __init__(self, value): + def __init__( + self, + value: str | Limit, + path: str = "", + tokens: int = 1, + limit_this_node_only: bool = False, + limit_submission: bool = False, + ): super().__init__("_" + str(value), value) + self.path = path + self.tokens = tokens + self.limit_this_node_only = limit_this_node_only + self.limit_submission = limit_submission def _build(self, ecflow_parent): value = self.value @@ -955,9 +970,32 @@ def _build(self, ecflow_parent): return if isinstance(value, Limit): value = value.fullname.split(":") - ecflow_parent.add_inlimit(ecflow.InLimit(value[1], value[0])) + if self.path: + if self.path != value[0]: + raise ValueError( + "InLimit path {} does not match limit path {}".format( + self.path, value[0] + ) + ) + ecflow_parent.add_inlimit( + ecflow.InLimit( + value[1], + value[0], + self.tokens, + self.limit_this_node_only, + self.limit_submission, + ) + ) else: - ecflow_parent.add_inlimit(ecflow.InLimit(str(value))) + ecflow_parent.add_inlimit( + ecflow.InLimit( + str(value), + self.path, + self.tokens, + self.limit_this_node_only, + self.limit_submission, + ) + ) class Inlimit(InLimit): diff --git a/tests/test_inlimits.py b/tests/test_inlimits.py index a0e9f7e..668f3e3 100644 --- a/tests/test_inlimits.py +++ b/tests/test_inlimits.py @@ -1,4 +1,5 @@ -from pyflow import Limits, Suite, Tasks +import pytest +from pyflow import Limit, Limits, InLimit, Suite, Tasks def test_inlimits(): @@ -14,6 +15,29 @@ def test_inlimits(): s.generate_node() +@pytest.mark.parametrize( + "options", + [ + {}, + {"path": "/s"}, + {"value": "tlimit"}, + {"value": "tlimit", "path": "/s"}, + {"value": "tlimit", "tokens": 1}, + {"value": "tlimit", "limit_this_node_only": True}, + {"value": "tlimit", "limit_submission": True}, + ], +) +def test_options(options): + with Suite("s") as s: + limit = Limit("tlimit", value=3) + if "value" not in options: + options["value"] = limit + Tasks("t", "t2", inlimits=InLimit(**options)) + + s.check_definition() + s.generate_node() + + if __name__ == "__main__": from os import path From a7c929e264e8d026cc62525837554b1cc98113ac Mon Sep 17 00:00:00 2001 From: Jenny Wong Date: Thu, 21 May 2026 14:20:46 +0000 Subject: [PATCH 02/32] isort --- tests/test_inlimits.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_inlimits.py b/tests/test_inlimits.py index 668f3e3..a4866fc 100644 --- a/tests/test_inlimits.py +++ b/tests/test_inlimits.py @@ -1,5 +1,6 @@ import pytest -from pyflow import Limit, Limits, InLimit, Suite, Tasks + +from pyflow import InLimit, Limit, Limits, Suite, Tasks def test_inlimits(): From e48afe3fc0c39b793e777fe818e077470234292e Mon Sep 17 00:00:00 2001 From: Gert Mertes Date: Wed, 27 May 2026 13:01:28 +0000 Subject: [PATCH 03/32] Add RepeatDateTimeList --- pyflow/attributes.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 6ef5947..24dc8fd 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -727,6 +727,65 @@ def day_of_week(self): return Mod(Add(Div(self, 86400), 4), 7) +class RepeatDateTimeList(Repeat): + """ + An attribute that allows a node to be repeated by a list of datetime values. + + Parameters: + name(str): The name of the repeat attribute. + values(list of datetime): The list of datetime values, as datetime objects or strings. + + Example:: + + pyflow.RepeatDateTimeList('REPEAT_DATETIME', + [datetime.datetime(year=2019, month=1, day=1), + datetime.datetime(year=2019, month=1, day=3)]) + + Values can also be strings in ISO 8601 basic format `yyyymmddTHHMMSS`, or `YYYYMMDD`:: + + pyflow.RepeatDateTimeList('REPEAT_DATETIME', ['20190101T120000', '20190103']) + """ + + def __init__(self, name, values): + super().__init__(name, values) + + def _build(self, ecflow_parent): + values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.values] + + repeat = ecflow.RepeatDateTimeList( + str(self.name), + values, + ) + + ecflow_parent.add_repeat(repeat) + + def __add__(self, other): + return Add(self, other) + + def __sub__(self, other): + return Sub(self, other) + + @property + def second(self): + """*int*: The second of the repeat datetime.""" + return Mod(self, 60) + + @property + def minute(self): + """*int*: The minute of the repeat datetime.""" + return Mod(Div(self, 60), 60) + + @property + def hour(self): + """*int*: The hour of the repeat datetime.""" + return Mod(Div(self, 3600), 24) + + @property + def day_of_week(self): + """*int*: The day of the week of the repeat datetime.""" + return Mod(Add(Div(self, 86400), 4), 7) + + def is_date(value): return ( isinstance(value, (datetime.date, datetime.datetime)) From f2a68df6ba59caf639eddb861cb4bde6e715f213 Mon Sep 17 00:00:00 2001 From: Gert Mertes <13658335+gmertes@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:05:08 +0100 Subject: [PATCH 04/32] self.values -> self.value Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pyflow/attributes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 24dc8fd..6b323e5 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -750,8 +750,7 @@ def __init__(self, name, values): super().__init__(name, values) def _build(self, ecflow_parent): - values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.values] - + values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.value] repeat = ecflow.RepeatDateTimeList( str(self.name), values, From 8c6f6f915655bfa9cd6c45b50854c8b924525f10 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 13:50:22 +0100 Subject: [PATCH 05/32] Export RepeatDateList in the pyflow module --- pyflow/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyflow/__init__.py b/pyflow/__init__.py index 305eb3d..20c0e80 100644 --- a/pyflow/__init__.py +++ b/pyflow/__init__.py @@ -30,6 +30,7 @@ RepeatDate, RepeatDateList, RepeatDateTime, + RepeatDateTimeList, RepeatDay, RepeatEnumerated, RepeatInteger, From 20c9664184e41b45b586e991be91f24d18711206 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 13:56:20 +0100 Subject: [PATCH 06/32] Add missing `values` property --- pyflow/attributes.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 6b323e5..2a11068 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -750,7 +750,9 @@ def __init__(self, name, values): super().__init__(name, values) def _build(self, ecflow_parent): - values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.value] + # Format all datetime values as ISO 8601 basic format `yyyymmddTHHMMSS` + values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.values] + repeat = ecflow.RepeatDateTimeList( str(self.name), values, @@ -758,6 +760,13 @@ def _build(self, ecflow_parent): ecflow_parent.add_repeat(repeat) + @property + def values(self): + """*list*: The list of datetime values.""" + return [ + x if isinstance(x, datetime.datetime) else as_date(x) for x in self.value + ] + def __add__(self, other): return Add(self, other) From 05b622115caa0d6f882443d25be74bd971f3cacb Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 13:57:13 +0100 Subject: [PATCH 07/32] Add basic test to construct RepeatDateTimeList attributes --- tests/test_attributes.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 594eac9..023101a 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -554,6 +554,35 @@ def test_repeat_datetime(self): s.check_definition() + def test_repeat_datetime_list(self): + from datetime import datetime as dt + i = dt(2000, 1, 1, 12, 0, 0) + j = dt(2000, 1, 2) + + input_tests = ( + ("A", [i]), + ("B", [i, j]), + ("C", ["20000103T120000", "20000104"]), + ("D", [i, "20010105T123456"]), + ) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = ( + 'repeat datetimelist A "20000101T120000"', + 'repeat datetimelist B "20000101T120000" "20000102T000000"', + 'repeat datetimelist C "20000103T120000" "20000104T000000"', + 'repeat datetimelist D "20000101T120000" "20010105T123456"', + ) + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + def test_repeat_date_list(self): i = date(year=2019, month=12, day=31) j = date(year=2020, month=1, day=1) From 8b1efc51bcdc865a4b91c646c6f01451e201c504 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 14:50:20 +0100 Subject: [PATCH 08/32] Safeguard against invalid RepeatDateTimeList values --- pyflow/attributes.py | 13 +++++++++++-- tests/test_attributes.py | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 2a11068..d35ee6c 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -729,7 +729,7 @@ def day_of_week(self): class RepeatDateTimeList(Repeat): """ - An attribute that allows a node to be repeated by a list of datetime values. + An attribute that allows a node to be repeated over a list of datetime values. Parameters: name(str): The name of the repeat attribute. @@ -741,12 +741,21 @@ class RepeatDateTimeList(Repeat): [datetime.datetime(year=2019, month=1, day=1), datetime.datetime(year=2019, month=1, day=3)]) - Values can also be strings in ISO 8601 basic format `yyyymmddTHHMMSS`, or `YYYYMMDD`:: + Values can also be strings in ISO 8601 basic format `yyyymmddTHHMMSS`, or `yyyymmdd`:: pyflow.RepeatDateTimeList('REPEAT_DATETIME', ['20190101T120000', '20190103']) """ def __init__(self, name, values): + if values is None: + raise ValueError("values cannot be None") + if not isinstance(values, list): + raise TypeError("values must be a list") + if isinstance(values, list) and not values: + raise ValueError("values cannot be an empty list") + if not all(isinstance(value, (datetime.datetime, str)) for value in values): + raise TypeError("values must be a list of datetime objects or strings") + super().__init__(name, values) def _build(self, ecflow_parent): diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 023101a..48a43fb 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -554,7 +554,7 @@ def test_repeat_datetime(self): s.check_definition() - def test_repeat_datetime_list(self): + def test_repeat_datetimelist_basic_usage(self): from datetime import datetime as dt i = dt(2000, 1, 1, 12, 0, 0) j = dt(2000, 1, 2) @@ -583,6 +583,23 @@ def test_repeat_datetime_list(self): s.check_definition() + def test_repeat_datetimelist_with_none_value(self): + with pytest.raises(ValueError, match="values cannot be None" ): + pyflow.RepeatDateTimeList("N", None) + + def test_repeat_datetimelist_with_empty_values_list(self): + with pytest.raises(ValueError, match="values cannot be an empty list" ): + pyflow.RepeatDateTimeList("E", []) + + def test_repeat_datetimelist_with_invalid_type_values_list(self): + with pytest.raises(TypeError, match="values must be a list of datetime objects or strings" ): + pyflow.RepeatDateTimeList("I", [20050101]) + + def test_repeat_datetimelist_with_literal_type_value(self): + from datetime import datetime as dt + with pytest.raises(TypeError, match="values must be a list" ): + pyflow.RepeatDateTimeList("I", dt(2000, 1, 1, 0, 0, 0)) + def test_repeat_date_list(self): i = date(year=2019, month=12, day=31) j = date(year=2020, month=1, day=1) From 625cfc23f0f8342f53dd5fce7407ef287cedecef Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 15:29:27 +0100 Subject: [PATCH 09/32] Update docstring regarding the use of truncated datetime strings as values Add test to ensure the functionality is as documented. --- pyflow/attributes.py | 8 ++++++-- tests/test_attributes.py | 23 ++++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index d35ee6c..2b914f5 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -658,7 +658,9 @@ class RepeatDateTime(Exportable): datetime.datetime(year=2019, month=12, day=31, hour=12, minute=0, second=0), datetime.timedelta(hours=12, minutes=0, seconds=0)) - Date and increment can also be strings:: + Start/End values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with + hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, + or simply a date ``yyyymmdd`` (the missing components are assumed to be 0), and increment can also be a string:: pyflow.RepeatDateTime('REPEAT_DATETIME', '20190101T120000', '20191231T120000', '12:00:00') @@ -741,7 +743,9 @@ class RepeatDateTimeList(Repeat): [datetime.datetime(year=2019, month=1, day=1), datetime.datetime(year=2019, month=1, day=3)]) - Values can also be strings in ISO 8601 basic format `yyyymmddTHHMMSS`, or `yyyymmdd`:: + Values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with + hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, + or simply a date ``yyyymmdd`` (the missing components are assumed to be 0):: pyflow.RepeatDateTimeList('REPEAT_DATETIME', ['20190101T120000', '20190103']) """ diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 48a43fb..f9c4d26 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -562,7 +562,7 @@ def test_repeat_datetimelist_basic_usage(self): input_tests = ( ("A", [i]), ("B", [i, j]), - ("C", ["20000103T120000", "20000104"]), + ("C", ["20000103T120000", "20000104T000000"]), ("D", [i, "20010105T123456"]), ) @@ -583,6 +583,27 @@ def test_repeat_datetimelist_basic_usage(self): s.check_definition() + def test_repeat_datetimelist_with_truncated_string_values(self): + input_tests = ( + ("A", ["20000101T01", "20000102T0102", "20000103T010203"]), + ("B", ["20000104", "20000105T12", "20010106T1234"]), + ) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = ( + 'repeat datetimelist A "20000101T010000" "20000102T010200" "20000103T010203"', + 'repeat datetimelist B "20000104T000000" "20000105T120000" "20010106T123400"', + ) + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + def test_repeat_datetimelist_with_none_value(self): with pytest.raises(ValueError, match="values cannot be None" ): pyflow.RepeatDateTimeList("N", None) From d87d0cd22877bf42bf48ea4ba372a1e75f870af4 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 15:47:06 +0100 Subject: [PATCH 10/32] Allow taking datetime.date objects in as values --- pyflow/attributes.py | 8 ++++---- tests/test_attributes.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 2b914f5..389602a 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -735,13 +735,13 @@ class RepeatDateTimeList(Repeat): Parameters: name(str): The name of the repeat attribute. - values(list of datetime): The list of datetime values, as datetime objects or strings. + values(list of datetime/date): The list of datetime/date values, as datetime/date objects or strings. Example:: pyflow.RepeatDateTimeList('REPEAT_DATETIME', - [datetime.datetime(year=2019, month=1, day=1), - datetime.datetime(year=2019, month=1, day=3)]) + [datetime.date(year=2019, month=1, day=1), + datetime.datetime(year=2019, month=1, day=3, hour=12, minute=0, seconds=0)]) Values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, @@ -757,7 +757,7 @@ def __init__(self, name, values): raise TypeError("values must be a list") if isinstance(values, list) and not values: raise ValueError("values cannot be an empty list") - if not all(isinstance(value, (datetime.datetime, str)) for value in values): + if not all(isinstance(value, (datetime.datetime, datetime.date, str)) for value in values): raise TypeError("values must be a list of datetime objects or strings") super().__init__(name, values) diff --git a/tests/test_attributes.py b/tests/test_attributes.py index f9c4d26..a5bf0be 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -583,6 +583,29 @@ def test_repeat_datetimelist_basic_usage(self): s.check_definition() + def test_repeat_datetimelist_allowing_date_and_datetime_object(self): + import datetime + i = datetime.datetime(2000, 1, 1, 12, 34, 56) + j = datetime.date(2000, 1, 2) + + input_tests = ( + ("A", [i, j]), + ) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = ( + 'repeat datetimelist A "20000101T123456" "20000102T000000"' + ) + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + def test_repeat_datetimelist_with_truncated_string_values(self): input_tests = ( ("A", ["20000101T01", "20000102T0102", "20000103T010203"]), From 7f54de6e2e422e7028ee934a6130219fdee69351 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 15:56:44 +0100 Subject: [PATCH 11/32] Correct formatting issues --- pyflow/attributes.py | 5 ++++- tests/test_attributes.py | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 389602a..55e8ef2 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -757,7 +757,10 @@ def __init__(self, name, values): raise TypeError("values must be a list") if isinstance(values, list) and not values: raise ValueError("values cannot be an empty list") - if not all(isinstance(value, (datetime.datetime, datetime.date, str)) for value in values): + if not all( + isinstance(value, (datetime.datetime, datetime.date, str)) + for value in values + ): raise TypeError("values must be a list of datetime objects or strings") super().__init__(name, values) diff --git a/tests/test_attributes.py b/tests/test_attributes.py index a5bf0be..36ce55e 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -556,6 +556,7 @@ def test_repeat_datetime(self): def test_repeat_datetimelist_basic_usage(self): from datetime import datetime as dt + i = dt(2000, 1, 1, 12, 0, 0) j = dt(2000, 1, 2) @@ -585,21 +586,18 @@ def test_repeat_datetimelist_basic_usage(self): def test_repeat_datetimelist_allowing_date_and_datetime_object(self): import datetime + i = datetime.datetime(2000, 1, 1, 12, 34, 56) j = datetime.date(2000, 1, 2) - input_tests = ( - ("A", [i, j]), - ) + input_tests = (("A", [i, j]),) with pyflow.Suite("s") as s: for idx, args in enumerate(input_tests): with pyflow.Task(f"t{idx}"): pyflow.RepeatDateTimeList(*args) - asserts = ( - 'repeat datetimelist A "20000101T123456" "20000102T000000"' - ) + asserts = 'repeat datetimelist A "20000101T123456" "20000102T000000"' defn = str(s.ecflow_definition()) for a in asserts: assert a in defn @@ -628,20 +626,23 @@ def test_repeat_datetimelist_with_truncated_string_values(self): s.check_definition() def test_repeat_datetimelist_with_none_value(self): - with pytest.raises(ValueError, match="values cannot be None" ): + with pytest.raises(ValueError, match="values cannot be None"): pyflow.RepeatDateTimeList("N", None) def test_repeat_datetimelist_with_empty_values_list(self): - with pytest.raises(ValueError, match="values cannot be an empty list" ): + with pytest.raises(ValueError, match="values cannot be an empty list"): pyflow.RepeatDateTimeList("E", []) def test_repeat_datetimelist_with_invalid_type_values_list(self): - with pytest.raises(TypeError, match="values must be a list of datetime objects or strings" ): + with pytest.raises( + TypeError, match="values must be a list of datetime objects or strings" + ): pyflow.RepeatDateTimeList("I", [20050101]) def test_repeat_datetimelist_with_literal_type_value(self): from datetime import datetime as dt - with pytest.raises(TypeError, match="values must be a list" ): + + with pytest.raises(TypeError, match="values must be a list"): pyflow.RepeatDateTimeList("I", dt(2000, 1, 1, 0, 0, 0)) def test_repeat_date_list(self): From 3f0ded6e5b208962d115a231f1ac30fa5ceb15af Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 4 Jun 2026 16:44:03 +0100 Subject: [PATCH 12/32] Update docs to include RepeatDateTime/List attributes --- docs/_ext/ecflow_lexers.py | 2 +- docs/content/api-reference.rst | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/_ext/ecflow_lexers.py b/docs/_ext/ecflow_lexers.py index a202c1c..1c2ee3f 100644 --- a/docs/_ext/ecflow_lexers.py +++ b/docs/_ext/ecflow_lexers.py @@ -33,7 +33,7 @@ class EcflowDefLexer(RegexLexer): bygroups(Keyword, Name.Constant), ), ( - r"(repeat)(\s+(?:date(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))", + r"(repeat)(\s+(?:date(?:time)?(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))", # noqa: E501 bygroups(Keyword, Name.Other, Name.Variable, Literal.Date), ), # Required diff --git a/docs/content/api-reference.rst b/docs/content/api-reference.rst index f7df6d5..d518e46 100644 --- a/docs/content/api-reference.rst +++ b/docs/content/api-reference.rst @@ -160,6 +160,14 @@ Repeat .. autoclass:: pyflow.attributes.RepeatDateList +.. _RepeatDateTime: + +.. autoclass:: pyflow.attributes.RepeatDateTime + +.. _RepeatDateTimeList: + +.. autoclass:: pyflow.attributes.RepeatDateTimeList + .. _RepeatDay: .. autoclass:: pyflow.attributes.RepeatDay From eb5ee04564a0efe6f654c61aa43c9427ed20b2ac Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Mon, 8 Jun 2026 13:29:49 +0100 Subject: [PATCH 13/32] Install the latest sources when generating docs This allows using the latest changes when building the documentation. --- .readthedocs.yaml | 6 ++++++ docs/environment.yml | 1 - docs/requirements.txt | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d347971..3807566 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -22,3 +22,9 @@ build: os: "ubuntu-20.04" tools: python: "mambaforge-4.10" + jobs: + # Install the checked-out source (e.g. the current PR) into the conda + # environment, so the docs are built against this code. Dependencies are + # already provided by docs/environment.yml, hence --no-deps. + post_install: + - pip install --no-deps . diff --git a/docs/environment.yml b/docs/environment.yml index db553a6..a244d83 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -18,7 +18,6 @@ dependencies: - sphinx-rtd-theme - sphinx-copybutton - sphinx-tabs - - git+https://github.com/ecmwf/pyflow.git variables: QT_MAC_WANTS_LAYER: 1 diff --git a/docs/requirements.txt b/docs/requirements.txt index e4b5648..cc30bbd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ ipykernel nbsphinx +pypandoc_binary sphinx-rtd-theme==0.5.2 sphinx-copybutton==0.3.1 -sphinx-tabs -git+https://github.com/ecmwf/pyflow.git \ No newline at end of file +sphinx-tabs \ No newline at end of file From 0cb724b9de10c5aec7a77c87588597293b49189a Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Mon, 8 Jun 2026 15:29:13 +0100 Subject: [PATCH 14/32] Use pull_request_target when triggering docs build As per the documentation of readthedocs/actions/preview@v1. --- .github/workflows/readthedocs-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/readthedocs-pr.yml b/.github/workflows/readthedocs-pr.yml index 7939a4c..6dd0782 100644 --- a/.github/workflows/readthedocs-pr.yml +++ b/.github/workflows/readthedocs-pr.yml @@ -2,7 +2,7 @@ # This does NOT trigger a build of the documentation, this is handled through webhooks. name: Read the Docs PR Preview on: - pull_request: + pull_request_target: types: - opened - synchronize From ccec6728c0c15ffb4a331e8c7ad44470a8ac40b7 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 9 Jun 2026 09:03:49 +0100 Subject: [PATCH 15/32] Decorate RepeatDateTimeList with `supported('')` --- pyflow/attributes.py | 12 ++- pyflow/importer.py | 71 ++++++++++++++++ pyproject.toml | 1 + tests/test_supported.py | 176 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 tests/test_supported.py diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 55e8ef2..06f3653 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -25,7 +25,7 @@ expression_from_json, make_expression, ) -from .importer import ecflow +from .importer import ecflow, supported from .state import aborted, active, complete, queued, submitted, suspended, unknown NO_TRIGGER = False @@ -641,6 +641,7 @@ def day_of_week(self): setattr(RepeatDate, day, property(lambda self: Eq(self.day_of_week, dow))) +@supported(">=5.12.0") class RepeatDateTime(Exportable): """ An attribute that allows a node to be repeated by a date+time value. @@ -665,6 +666,10 @@ class RepeatDateTime(Exportable): pyflow.RepeatDateTime('REPEAT_DATETIME', '20190101T120000', '20191231T120000', '12:00:00') + Note:: + + This repeat type is only supported in ecFlow 5.12.0 and later. + """ def __init__( @@ -729,6 +734,7 @@ def day_of_week(self): return Mod(Add(Div(self, 86400), 4), 7) +@supported(">=5.17.0") class RepeatDateTimeList(Repeat): """ An attribute that allows a node to be repeated over a list of datetime values. @@ -748,6 +754,10 @@ class RepeatDateTimeList(Repeat): or simply a date ``yyyymmdd`` (the missing components are assumed to be 0):: pyflow.RepeatDateTimeList('REPEAT_DATETIME', ['20190101T120000', '20190103']) + + Note:: + + This repeat type is only supported in ecFlow 5.17.0 and later. """ def __init__(self, name, values): diff --git a/pyflow/importer.py b/pyflow/importer.py index 870c7b8..3969ee7 100644 --- a/pyflow/importer.py +++ b/pyflow/importer.py @@ -1,5 +1,9 @@ +import functools import os import sys +import types + +from packaging.specifiers import SpecifierSet try: import ecflow @@ -30,3 +34,70 @@ raise ImportError( "Could not find ecflow Python library, try to set ECFLOW_DIR environment variable to correct path" ) + + +def supported(specifier: str, current: str = ecflow.__version__): + """ + A decorator that ensures the decorated class can only be used when + the available ecFlow version satisfies ``specifier``. + + Every method and property of the class is wrapped so that invoking it + (e.g. instantiating the class via ``__init__``, or accessing a property) + raises :class:`NotImplementedError` when ``current`` does not satisfy + ``specifier``. + + The version comparison is evaluated once, when the class is decorated + (i.e. at import time, using ``current`` which defaults to the version of + the imported ecFlow module). + The ``current`` argument allows tests to instrument the behaviour for an arbitrary version. + + Parameters: + specifier(str): A :class:`packaging.specifiers.SpecifierSet` string, e.g. ``">=5.12.0,<6.0.0"``. + current(str): The current version, e.g. ``"5.12.0"``. Defaults to the version of the imported ecFlow module. + """ + + # Evaluate the comparison once, since both versions are fixed for the + # lifetime of the decorated class. + is_supported = SpecifierSet(specifier).contains(current) + + def decorator(cls): + + # Define a wrapper factory that checks the supported version before calling the original function + def make_wrapper(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not is_supported: + raise NotImplementedError( + "{} functionality is only supported for ecFlow {}, but current version is {}".format( + cls.__name__, specifier, current + ) + ) + + return func(*args, **kwargs) + + return wrapper + + # Iterate over a copy as we mutate the class namespace while iterating. + for attr_name, attr_value in list(cls.__dict__.items()): + + if isinstance(attr_value, types.FunctionType): + # Wrapped plain methods directly. + setattr(cls, attr_name, make_wrapper(attr_value)) + + elif isinstance(attr_value, property): + # Ensure properties must remain properties, + # by wrapping accessors and rebuilding the property so attribute access keeps working. + setattr( + cls, + attr_name, + property( + make_wrapper(attr_value.fget) if attr_value.fget else None, + make_wrapper(attr_value.fset) if attr_value.fset else None, + make_wrapper(attr_value.fdel) if attr_value.fdel else None, + attr_value.__doc__, + ), + ) + + return cls + + return decorator diff --git a/pyproject.toml b/pyproject.toml index 440f8ce..a07e905 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dynamic = ["version", "readme"] dependencies = [ "jinja2", + "packaging", "requests", ] diff --git a/tests/test_supported.py b/tests/test_supported.py new file mode 100644 index 0000000..cd58406 --- /dev/null +++ b/tests/test_supported.py @@ -0,0 +1,176 @@ +import ecflow +import pytest +from packaging import version + +import pyflow +from pyflow.importer import supported + + +def make_widget(specifier, current): + """Build a class decorated with an explicitly instrumented specifier/current version.""" + + @supported(specifier, current=current) + class Widget: + def __init__(self, value=0): + self.value = value + + def a_method(self): + "A Widget method." + return self.value + + @property + def a_property(self): + "A Widget property." + return self.value * 2 + + return Widget + + +def test_widget_use_is_disallowed_on_older_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + with pytest.raises(NotImplementedError): + Widget() + + +def test_widget_use_is_allowed_on_newer_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.13.0") + assert Widget(3).value == 3 + + +def test_widget_use_is_allowed_on_boundary_version(): + """The exact lower-bound version must be accepted (>= semantics).""" + Widget = make_widget(specifier=">=5.12.0", current="5.12.0") + assert Widget(7).value == 7 + + +def test_widget_use_is_disallowed_on_just_below_boundary(): + Widget = make_widget(specifier=">=5.12.0", current="5.11.9") + with pytest.raises(NotImplementedError): + Widget() + + +def test_error_message_contains_specifier_and_name(): + Widget = make_widget(specifier=">=5.20.1", current="5.6.7") + with pytest.raises(NotImplementedError) as exc: + Widget() + message = str(exc.value) + assert "Widget" in message + assert ">=5.20.1" in message + assert "5.6.7" in message + + +def test_methods_are_guarded_when_unsupported(): + """Ensure non-__init__ methods also raise when unsupported.""" + + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + # __init__ is guarded too, so build with an unguarded instance via __new__. + w = Widget.__new__(Widget) + with pytest.raises(NotImplementedError): + w.a_method() + + +def test_methods_work_when_supported(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(5) + assert w.a_method() == 5 + + +def test_properties_remain_properties(): + """ + Ensure properties behave as properties (return a computed value) after decoration. + """ + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(4) + assert w.a_property == 8 + assert isinstance(type(w).__dict__["a_property"], property) + + +def test_properties_are_disallowed_on_older_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + w = Widget.__new__(Widget) + with pytest.raises(NotImplementedError): + _ = w.a_property + + +def test_properties_are_allowed_on_newer_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(4) + assert w.a_property == 8 + + +def test_wraps_preserves_metadata(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert Widget.a_method.__name__ == "a_method" + assert Widget.a_method.__doc__ == "A Widget method." + assert Widget.__dict__["a_property"].__doc__ == "A Widget property." + + +def test_current_defaults_to_installed_ecflow_version(): + """Without an explicit ``current``, the installed ecFlow version is used.""" + + @supported(">=9999.0.0") + class Future: + def __init__(self): + pass + + with pytest.raises(NotImplementedError): + Future() + + @supported(">=0.0.1") + class Ancient: + def __init__(self): + self.ok = True + + assert Ancient().ok is True + + +def test_compound_specifier_excludes_upper_bound(): + """A compound specifier like >=5.12.0,<5.13.0 rejects versions outside the range.""" + Widget = make_widget(specifier=">=5.12.0,<5.13.0", current="5.13.0") + with pytest.raises(NotImplementedError): + Widget() + + +def test_compound_specifier_allows_version_in_range(): + Widget = make_widget(specifier=">=5.12.0,<5.13.0", current="5.12.5") + assert Widget(1).value == 1 + + +# ----------------------------------------------------------------------------- + + +def _installed_below(min_version): + return version.parse(ecflow.__version__) < version.parse(min_version) + + +def test_repeat_datetime_builds_on_installed_ecflow(): + if _installed_below("5.12.0"): + pytest.skip("RepeatDateTime requires ecFlow >= 5.12.0") + + with pyflow.Suite("s"): + with pyflow.Task("t"): + repeat = pyflow.RepeatDateTime( + "REPEAT_DATETIME", + "20190101T120000", + "20191231T120000", + "12:00:00", + ) + + assert repeat.name == "REPEAT_DATETIME" + assert not callable(repeat.second) + assert isinstance(type(repeat).__dict__["second"], property) + + +def test_repeat_datetimelist_builds_on_installed_ecflow(): + if _installed_below("5.17.0"): + pytest.skip("RepeatDateTimeList requires ecFlow >= 5.17.0") + + with pyflow.Suite("s"): + with pyflow.Task("t"): + repeat = pyflow.RepeatDateTimeList( + "REPEAT_DATETIME", ["20190101T120000", "20190103"] + ) + + assert repeat.name == "REPEAT_DATETIME" + assert not callable(repeat.values) + assert isinstance(type(repeat).__dict__["values"], property) From 6d31b3d83dab1a59df8bb6324c561219cb3d6105 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Thu, 2 Jul 2026 15:15:10 +0000 Subject: [PATCH 16/32] add pre-commit config to antecipate ci qa runs --- .pre-commit-config.yaml | 27 +++++++++++++++++++++++++++ README.md | 30 ++++++++++++++++++++++++++++++ pyproject.toml | 7 +++++++ 3 files changed, 64 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..bf207ad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: + - repo: https://github.com/pycqa/isort + rev: 8.0.1 + hooks: + - id: isort + name: ci qa isort --check + args: ["--check-only", "."] + pass_filenames: false + require_serial: true + + - repo: https://github.com/psf/black + rev: 26.5.1 + hooks: + - id: black + name: ci qa black --check + args: ["--check", "."] + pass_filenames: false + require_serial: true + + - repo: https://github.com/pycqa/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + name: ci qa flake8 + args: ["."] + pass_filenames: false + require_serial: true diff --git a/README.md b/README.md index 00fbb71..6d1dde2 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ ## Installation + To install pyflow using conda (including ecFlow): conda env create -n pyflow -f environment.yml @@ -43,8 +44,37 @@ To install pyflow using pip (requires a local installation of ecFlow): pip install pyflow-workflow-generator ## Documentation + The documentation can be found at . +## QA Checks (CI-equivalent) + +The CI `qa` job runs the following checks, in order: + +1. `isort --check .` +2. `black --check .` +3. `flake8 .` + +This repository includes a matching pre-commit configuration in +`.pre-commit-config.yaml` using standard upstream hooks, pinned to the +tool versions from the CI run: + +- `isort==8.0.1` +- `black==26.5.1` +- `flake8==7.3.0` + +Run locally: + +```bash +python -m pip install ".[dev]" +pre-commit run --all-files +``` + +The hooks are split into three sequential checks (isort, then black, then +flake8) to avoid conflicts and to match CI behavior. Running `isort` before +`black` prevents import-format churn, and running `flake8` last ensures linting +sees code after formatting checks. + ## License [Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities diff --git a/pyproject.toml b/pyproject.toml index a07e905..abae427 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,13 @@ dependencies = [ test = [ "pytest", ] + qa = [ + "black==26.5.1", + "flake8==7.3.0", + "isort==8.0.1", + "pre-commit", + ] + dev = ["pyflow-workflow-generator[test,qa]"] diagrams = [ "graphviz", ] From b11edecd22e8e4836ff1f2a8a018b50b717517f6 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Thu, 2 Jul 2026 15:16:37 +0000 Subject: [PATCH 17/32] update config options --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index abae427..6b91a3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ where = ["."] exclude = ["tests"] [tool.setuptools_scm] -write_to = "pyflow/_version.py" -write_to_template = ''' +version_file = "pyflow/_version.py" +version_file_template = ''' # Do not change! Do not track in version control! __version__ = "{version}" ''' From a706ceb5a4a53977d6700f0efa0a6e0b555f65fa Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Thu, 2 Jul 2026 15:19:58 +0000 Subject: [PATCH 18/32] update building machine name for rtd --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daa4806..079303b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: docs: name: docs - runs-on: ubuntu-latest + runs-on: ubuntu-lts-latest steps: - uses: actions/checkout@v2 - uses: mamba-org/setup-micromamba@v1 From 4cce674c5d42e1991aa57e9aca55ef839fc2812a Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Thu, 2 Jul 2026 15:25:21 +0000 Subject: [PATCH 19/32] fix and sync docs ci builds --- .github/workflows/ci.yml | 2 +- .readthedocs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 079303b..9c7e13a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: docs: name: docs - runs-on: ubuntu-lts-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - uses: mamba-org/setup-micromamba@v1 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3807566..ac90b82 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,7 +19,7 @@ conda: environment: docs/environment.yml build: - os: "ubuntu-20.04" + os: "ubuntu-22.04" tools: python: "mambaforge-4.10" jobs: From d0fe3e17494c049160242f9b32eb441cbb3424a3 Mon Sep 17 00:00:00 2001 From: Jenny Wong Date: Thu, 2 Jul 2026 23:24:47 +0000 Subject: [PATCH 20/32] Remove duplication check --- pyflow/nodes.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pyflow/nodes.py b/pyflow/nodes.py index 67b5a46..380ef97 100644 --- a/pyflow/nodes.py +++ b/pyflow/nodes.py @@ -940,8 +940,7 @@ def _add_exit_hook(self, hook): if isinstance(hook, str): hook = [hook] for hk in hook: - if hk not in self._exit_hook: - self._exit_hook.append(hk) + self._exit_hook.append(hk) # Check if properly initialised if "_nodes" in self.__dict__: for chld in self.executable_children: @@ -1206,8 +1205,7 @@ def _add_exit_hook(self, hook): if isinstance(hook, str): hook = [hook] for hk in hook: - if hk not in self._exit_hook: - self._exit_hook.append(hk) + self._exit_hook.append(hk) # Check if properly initialised if "_nodes" in self.__dict__: for chld in self.executable_children: @@ -1405,8 +1403,7 @@ def _add_exit_hook(self, hook: str): if isinstance(hook, str): hook = [hook] for hk in hook: - if hk not in self._exit_hook: - self._exit_hook.append(hk) + self._exit_hook.append(hk) def generate_script(self): """ From d992e1c230194993a8fc6ff87e66ed37fe2bad48 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Wed, 1 Jul 2026 16:34:32 +0000 Subject: [PATCH 21/32] expose ERROR arguments that can be used in conditional hook/cleanup user code --- pyflow/host.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyflow/host.py b/pyflow/host.py index 9f407a5..cddeced 100644 --- a/pyflow/host.py +++ b/pyflow/host.py @@ -437,6 +437,9 @@ def preamble_error_function(self, ecflowpath, exit_hook=None): set +x # Define a error handler ERROR() { + export EXIT_REASON="$1" + export EXIT_DETAIL="$2" + export EXIT_RC="${3:-1}" export PATH=%(ecf_path)s:$PATH set +eu # Clear -eu flag, so we don't fail wait # wait for background process to stop @@ -454,11 +457,11 @@ def preamble_error_function(self, ecflowpath, exit_hook=None): export SIGNAL_LIST='%(signal_list)s' for signal in $SIGNAL_LIST; do - trap "ERROR $signal \\"Signal $(kill -l $signal) ($signal) received \\"" $signal + trap "rc=\\$?; ERROR $signal \\"Signal $(kill -l $signal) ($signal) received \\"" $signal done # Trap any calls to exit and errors caught by the -e flag - trap ERROR 0 + trap 'rc=$?; ERROR EXIT "$rc"' 0 set -x """) % {"ecf_path": ecflowpath, "signal_list": signal_list}) # noqa: E501 return script From a59953a56cd117a4fbcd855532a0c2ebfa6cfd41 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Thu, 2 Jul 2026 14:00:26 +0000 Subject: [PATCH 22/32] pass on right arguments and return codes in error --- pyflow/host.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyflow/host.py b/pyflow/host.py index cddeced..106c7d7 100644 --- a/pyflow/host.py +++ b/pyflow/host.py @@ -457,11 +457,11 @@ def preamble_error_function(self, ecflowpath, exit_hook=None): export SIGNAL_LIST='%(signal_list)s' for signal in $SIGNAL_LIST; do - trap "rc=\\$?; ERROR $signal \\"Signal $(kill -l $signal) ($signal) received \\"" $signal + trap "rc=\\$?; ERROR $signal \\"Signal $(kill -l $signal) ($signal) received\\" \\"\\$rc\\"" $signal done # Trap any calls to exit and errors caught by the -e flag - trap 'rc=$?; ERROR EXIT "$rc"' 0 + trap 'rc=$?; ERROR EXIT "" "$rc"' 0 set -x """) % {"ecf_path": ecflowpath, "signal_list": signal_list}) # noqa: E501 return script From 9441095fef72c91fa0a2ff8b568971884c4f86d1 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Thu, 2 Jul 2026 14:00:59 +0000 Subject: [PATCH 23/32] allow exit_hook as pyflow.Script objects --- pyflow/nodes.py | 40 ++++++++++++------ tests/test_families.py | 96 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 113 insertions(+), 23 deletions(-) diff --git a/pyflow/nodes.py b/pyflow/nodes.py index 380ef97..30e98c0 100644 --- a/pyflow/nodes.py +++ b/pyflow/nodes.py @@ -95,6 +95,25 @@ def json_build(cls, name, tree): json_build(Task, k, v) +def _normalize_exit_hook(hook): + if isinstance(hook, str): + return [hook] + if isinstance(hook, Script): + return [hook.value] + + try: + hook_items = list(hook) + except TypeError as exc: + raise TypeError( + "exit_hook must be a string, Script, or iterable containing those types" + ) from exc + + normalized = [] + for item in hook_items: + normalized.extend(_normalize_exit_hook(item)) + return normalized + + class DuplicateNodeError(RuntimeError): def __init__(self, parent, new, existing): super().__init__( @@ -937,14 +956,13 @@ def _add_single_node(self, node): super()._add_single_node(node) def _add_exit_hook(self, hook): - if isinstance(hook, str): - hook = [hook] - for hk in hook: + normalized_hook = _normalize_exit_hook(hook) + for hk in normalized_hook: self._exit_hook.append(hk) # Check if properly initialised if "_nodes" in self.__dict__: for chld in self.executable_children: - chld._add_exit_hook(hook) + chld._add_exit_hook(normalized_hook) class AnchorFamily(AnchorMixin, Family): @@ -1202,14 +1220,13 @@ def _add_single_node(self, node): super()._add_single_node(node) def _add_exit_hook(self, hook): - if isinstance(hook, str): - hook = [hook] - for hk in hook: + normalized_hook = _normalize_exit_hook(hook) + for hk in normalized_hook: self._exit_hook.append(hk) # Check if properly initialised if "_nodes" in self.__dict__: for chld in self.executable_children: - chld._add_exit_hook(hook) + chld._add_exit_hook(normalized_hook) class Task(Node): @@ -1399,10 +1416,9 @@ def task_purge_modules(self): return self.host.purge_modules or super().task_purge_modules() - def _add_exit_hook(self, hook: str): - if isinstance(hook, str): - hook = [hook] - for hk in hook: + def _add_exit_hook(self, hook): + normalized_hook = _normalize_exit_hook(hook) + for hk in normalized_hook: self._exit_hook.append(hk) def generate_script(self): diff --git a/tests/test_families.py b/tests/test_families.py index 7944c60..b84a9e9 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -1,4 +1,6 @@ -from pyflow import AnchorFamily, Family, Limit, Suite, Task, Variable +import pytest + +from pyflow import AnchorFamily, Family, Limit, Script, Suite, Task, Variable def test_families(): @@ -99,7 +101,7 @@ def test_files_locations(): assert t7.deploy_path == "/a/base/path/f5/f6/t7.ecf" -def test_exit_hook(): +def test_exit_hook_strings(): """ Propagate exit hook to children of a given family """ @@ -110,7 +112,8 @@ def test_exit_hook(): t5 = Task("t5") - with Suite("S"): + common_hook = "one_hook" + with Suite("S", exit_hook=common_hook): with Family("f", exit_hook="hook_f", tasks=t5) as f: Limit("limit1", 15) Variable("VARIABLE1", 1234) @@ -118,15 +121,86 @@ def test_exit_hook(): with Family("f2", families=f3, exit_hook="hook_f2") as f2: t2 = Task("t2", exit_hook="hook_t2") - assert f._exit_hook == ["hook_f"] - assert t1._exit_hook == ["hook_f", "hook_t1"] - assert f2._exit_hook == ["hook_f", "hook_f2"] - assert t2._exit_hook == ["hook_f", "hook_f2", "hook_t2"] - assert f3._exit_hook == ["hook_f3", "hook_f", "hook_f2"] - assert t3._exit_hook == ["hook_f3", "hook_t3", "hook_f", "hook_f2"] - assert t4._exit_hook == ["hook_f3", "hook_f", "hook_f2"] - assert t5._exit_hook == ["hook_f"] + assert f._exit_hook == [common_hook, "hook_f"] + assert t1._exit_hook == [common_hook, "hook_f", "hook_t1"] + assert f2._exit_hook == [common_hook, "hook_f", "hook_f2"] + assert t2._exit_hook == [common_hook, "hook_f", "hook_f2", "hook_t2"] + assert f3._exit_hook == ["hook_f3", common_hook, "hook_f", "hook_f2"] + assert t3._exit_hook == ["hook_f3", "hook_t3", common_hook, "hook_f", "hook_f2"] + assert t4._exit_hook == ["hook_f3", common_hook, "hook_f", "hook_f2"] + assert t5._exit_hook == [common_hook, "hook_f"] + + +def test_exit_hook_scripts(): + """ + Support Script objects as exit hooks and propagate to children of a given family + """ + script_hook_f3 = Script(["hook_f3_line_1", "hook_f3_line_2"]) + script_hook_t3 = Script("hook_t3_line") + with Family("f3", exit_hook=script_hook_f3) as f3: + t3 = Task("t3", exit_hook=script_hook_t3) + t4 = Task("t4") + + t5 = Task("t5") + + common_hook = Script("common_hook_line") + hook_f = Script("hook_f_line") + hook_t1 = Script("hook_t1_line") + hook_f2 = Script("hook_f2_line") + hook_t2 = Script("hook_t2_line") + with Suite("S", exit_hook=common_hook): + with Family("f", exit_hook=hook_f, tasks=t5) as f: + Limit("limit1", 15) + Variable("VARIABLE1", 1234) + t1 = Task("t1", exit_hook=hook_t1) + with Family("f2", families=f3, exit_hook=hook_f2) as f2: + t2 = Task("t2", exit_hook=hook_t2) + + assert f._exit_hook == ["common_hook_line", "hook_f_line"] + assert t1._exit_hook == ["common_hook_line", "hook_f_line", "hook_t1_line"] + assert f2._exit_hook == ["common_hook_line", "hook_f_line", "hook_f2_line"] + assert t2._exit_hook == [ + "common_hook_line", + "hook_f_line", + "hook_f2_line", + "hook_t2_line", + ] + assert f3._exit_hook == ["hook_f3_line_1\nhook_f3_line_2", "common_hook_line", "hook_f_line", "hook_f2_line"] + assert t3._exit_hook == [ + "hook_f3_line_1\nhook_f3_line_2", + "hook_t3_line", + "common_hook_line", + "hook_f_line", + "hook_f2_line", + ] + assert t4._exit_hook == ["hook_f3_line_1\nhook_f3_line_2", "common_hook_line", "hook_f_line", "hook_f2_line"] + assert t5._exit_hook == ["common_hook_line", "hook_f_line"] + + +@pytest.mark.xfail( + reason="Known issue: duplicate list exit_hook lines are de-duplicated when inherited" +) +def test_exit_hook_list_strings_preserves_duplicate_lines(): + """ + Lists of string exit hooks should preserve ordering and duplicate lines. + """ + + parent_lines = ["echo pre_cleanup", "echo duplicate_line"] + child_lines = ["echo duplicate_line", "echo post_cleanup"] + + with Suite("S", exit_hook=parent_lines): + with Family("f", exit_hook=child_lines) as f: + t1 = Task("t1") + + expected = [ + "echo pre_cleanup", + "echo duplicate_line", + "echo duplicate_line", + "echo post_cleanup", + ] + assert f._exit_hook == expected + assert t1._exit_hook == expected if __name__ == "__main__": from os import path From 9b8158cac31cff9193191463e2f43c8155657712 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Thu, 2 Jul 2026 14:04:40 +0000 Subject: [PATCH 24/32] ensure new variables are covered in tests --- tests/test_families.py | 15 +++++++++++++-- tests/test_host.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/test_families.py b/tests/test_families.py index b84a9e9..3b4e33b 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -165,7 +165,12 @@ def test_exit_hook_scripts(): "hook_f2_line", "hook_t2_line", ] - assert f3._exit_hook == ["hook_f3_line_1\nhook_f3_line_2", "common_hook_line", "hook_f_line", "hook_f2_line"] + assert f3._exit_hook == [ + "hook_f3_line_1\nhook_f3_line_2", + "common_hook_line", + "hook_f_line", + "hook_f2_line", + ] assert t3._exit_hook == [ "hook_f3_line_1\nhook_f3_line_2", "hook_t3_line", @@ -173,7 +178,12 @@ def test_exit_hook_scripts(): "hook_f_line", "hook_f2_line", ] - assert t4._exit_hook == ["hook_f3_line_1\nhook_f3_line_2", "common_hook_line", "hook_f_line", "hook_f2_line"] + assert t4._exit_hook == [ + "hook_f3_line_1\nhook_f3_line_2", + "common_hook_line", + "hook_f_line", + "hook_f2_line", + ] assert t5._exit_hook == ["common_hook_line", "hook_f_line"] @@ -202,6 +212,7 @@ def test_exit_hook_list_strings_preserves_duplicate_lines(): assert f._exit_hook == expected assert t1._exit_hook == expected + if __name__ == "__main__": from os import path diff --git a/tests/test_host.py b/tests/test_host.py index 9978600..def9ba7 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -446,6 +446,16 @@ def test_traps(): assert signal_list1 in s1 assert signal_list2 in s2 + # Ensure ERROR exposes contract variables used by custom exit hooks. + assert 'export EXIT_REASON="$1"' in s1 + assert 'export EXIT_DETAIL="$2"' in s1 + assert 'export EXIT_RC="${3:-1}"' in s1 + + # Ensure traps pass the command return code through to ERROR. + assert 'trap "rc=\\$?; ERROR $signal' in s1 + assert '\\"\\$rc\\"" $signal' in s1 + assert 'trap \'rc=$?; ERROR EXIT "" "$rc"\' 0' in s1 + @pytest.mark.parametrize( "key,expected_class,kwargs", From 59e36098ade1ad68ce1996a8dcc61732888d19d7 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Wed, 8 Jul 2026 14:54:20 +0100 Subject: [PATCH 25/32] exit hook script allow duplicated lines. --- tests/test_families.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_families.py b/tests/test_families.py index 3b4e33b..4598966 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -187,12 +187,11 @@ def test_exit_hook_scripts(): assert t5._exit_hook == ["common_hook_line", "hook_f_line"] -@pytest.mark.xfail( - reason="Known issue: duplicate list exit_hook lines are de-duplicated when inherited" -) def test_exit_hook_list_strings_preserves_duplicate_lines(): """ Lists of string exit hooks should preserve ordering and duplicate lines. + + Changed in v3.7.0. In previous versions duplication in exit_hook scripts were not allowed. """ parent_lines = ["echo pre_cleanup", "echo duplicate_line"] From 97fdc91cadd05bf8ad7e972943097b4dc3af5476 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Wed, 8 Jul 2026 15:01:48 +0100 Subject: [PATCH 26/32] ignore qa --- tests/test_families.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_families.py b/tests/test_families.py index 4598966..3c5d04c 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -213,6 +213,7 @@ def test_exit_hook_list_strings_preserves_duplicate_lines(): if __name__ == "__main__": + # flake8: noqa from os import path import pytest From 7015e91a8c43b2e63b91a7a4b8147f8ca5977408 Mon Sep 17 00:00:00 2001 From: Juan Colonese <47576248+colonesej@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:07:42 +0100 Subject: [PATCH 27/32] fix: typo in docstrings Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pyflow/attributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 06f3653..566f06a 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -747,7 +747,7 @@ class RepeatDateTimeList(Repeat): pyflow.RepeatDateTimeList('REPEAT_DATETIME', [datetime.date(year=2019, month=1, day=1), - datetime.datetime(year=2019, month=1, day=3, hour=12, minute=0, seconds=0)]) + datetime.datetime(year=2019, month=1, day=3, hour=12, minute=0, second=0)]) Values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, From 37550bca961b4601474dd739cbcb0787da0de2ae Mon Sep 17 00:00:00 2001 From: Juan Colonese <47576248+colonesej@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:13:51 +0100 Subject: [PATCH 28/32] fix: explicit about supported types in error message Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pyflow/attributes.py | 2 +- tests/test_attributes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 566f06a..f9c1efe 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -771,7 +771,7 @@ def __init__(self, name, values): isinstance(value, (datetime.datetime, datetime.date, str)) for value in values ): - raise TypeError("values must be a list of datetime objects or strings") + raise TypeError("values must be a list of datetime/date objects or strings") super().__init__(name, values) diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 36ce55e..c8e89d6 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -635,7 +635,7 @@ def test_repeat_datetimelist_with_empty_values_list(self): def test_repeat_datetimelist_with_invalid_type_values_list(self): with pytest.raises( - TypeError, match="values must be a list of datetime objects or strings" + TypeError, match="values must be a list of datetime/date objects or strings" ): pyflow.RepeatDateTimeList("I", [20050101]) From fce39a9465fe92b02ed8a22b124a89ece70754c1 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Wed, 8 Jul 2026 15:21:39 +0100 Subject: [PATCH 29/32] pin external action SHA --- .github/workflows/readthedocs-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/readthedocs-pr.yml b/.github/workflows/readthedocs-pr.yml index 6dd0782..3591d87 100644 --- a/.github/workflows/readthedocs-pr.yml +++ b/.github/workflows/readthedocs-pr.yml @@ -19,6 +19,6 @@ jobs: documentation-links: runs-on: ubuntu-latest steps: - - uses: readthedocs/actions/preview@v1 + - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1 with: project-slug: "pyflow-workflow-generator" \ No newline at end of file From 333cc7369d0d2864f923767dc4449fd5039b3813 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Wed, 8 Jul 2026 15:28:35 +0100 Subject: [PATCH 30/32] include static and class methods in wrapped context --- pyflow/importer.py | 8 ++++++++ tests/test_supported.py | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/pyflow/importer.py b/pyflow/importer.py index 3969ee7..bda753b 100644 --- a/pyflow/importer.py +++ b/pyflow/importer.py @@ -98,6 +98,14 @@ def wrapper(*args, **kwargs): ), ) + elif isinstance(attr_value, staticmethod): + # Unwrap the inner function, wrap it, and re-wrap as staticmethod. + setattr(cls, attr_name, staticmethod(make_wrapper(attr_value.__func__))) + + elif isinstance(attr_value, classmethod): + # Unwrap the inner function, wrap it, and re-wrap as classmethod. + setattr(cls, attr_name, classmethod(make_wrapper(attr_value.__func__))) + return cls return decorator diff --git a/tests/test_supported.py b/tests/test_supported.py index cd58406..42e2000 100644 --- a/tests/test_supported.py +++ b/tests/test_supported.py @@ -23,6 +23,16 @@ def a_property(self): "A Widget property." return self.value * 2 + @staticmethod + def a_static(): + "A Widget static method." + return 42 + + @classmethod + def a_classmethod(cls): + "A Widget class method." + return cls.__name__ + return Widget @@ -174,3 +184,35 @@ def test_repeat_datetimelist_builds_on_installed_ecflow(): assert repeat.name == "REPEAT_DATETIME" assert not callable(repeat.values) assert isinstance(type(repeat).__dict__["values"], property) + + +def test_staticmethod_is_guarded_when_unsupported(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + with pytest.raises(NotImplementedError): + Widget.a_static() + + +def test_staticmethod_works_when_supported(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert Widget.a_static() == 42 + + +def test_classmethod_is_guarded_when_unsupported(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + with pytest.raises(NotImplementedError): + Widget.a_classmethod() + + +def test_classmethod_works_when_supported(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert Widget.a_classmethod() == "Widget" + + +def test_staticmethod_remains_static_after_decoration(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert isinstance(Widget.__dict__["a_static"], staticmethod) + + +def test_classmethod_remains_classmethod_after_decoration(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert isinstance(Widget.__dict__["a_classmethod"], classmethod) From 13911f68bb97487c0e0b78db1169f77bbcf9539a Mon Sep 17 00:00:00 2001 From: Juan Colonese <47576248+colonesej@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:30:31 +0100 Subject: [PATCH 31/32] ensure asserts are not just strings Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_attributes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_attributes.py b/tests/test_attributes.py index c8e89d6..dec685f 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -597,10 +597,9 @@ def test_repeat_datetimelist_allowing_date_and_datetime_object(self): with pyflow.Task(f"t{idx}"): pyflow.RepeatDateTimeList(*args) - asserts = 'repeat datetimelist A "20000101T123456" "20000102T000000"' + assert_ = 'repeat datetimelist A "20000101T123456" "20000102T000000"' defn = str(s.ecflow_definition()) - for a in asserts: - assert a in defn + assert assert_ in defn s.check_definition() From 62c1d9905a3cbb08f00bd6950577fcd747126a79 Mon Sep 17 00:00:00 2001 From: Juan Colonese Date: Wed, 8 Jul 2026 15:36:37 +0100 Subject: [PATCH 32/32] raise supported python version --- .github/workflows/on-push.yml | 2 +- pyproject.toml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 139f1a0..9dbcb2c 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -37,7 +37,7 @@ jobs: with: activate-environment: test environment-file: environment.yml - python-version: 3.8 + python-version: 3.12 auto-activate-base: false - name: Conda check shell: bash -l {0} diff --git a/pyproject.toml b/pyproject.toml index 6b91a3f..e71fb17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyflow-workflow-generator" -requires-python = ">=3.8" +requires-python = ">=3.12" authors = [ {name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int"}, ] @@ -22,8 +22,7 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", ] dynamic = ["version", "readme"]