Pre-release 3.7.0

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
 
   docs:
     name: docs
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       - uses: mamba-org/setup-micromamba@v1

.github/workflows/on-push.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           activate-environment: test
           environment-file: environment.yml
-          python-version: 3.8
+          python-version: 3.12
           auto-activate-base: false
       - name: Conda check
         shell: bash -l {0}

.github/workflows/readthedocs-pr.yml

@@ -2,7 +2,7 @@
 # This does NOT trigger a build of the documentation, this is handled through webhooks.
 name: Read the Docs PR Preview
 on:
-  pull_request:
+  pull_request_target:
     types:
       - opened
       - synchronize
@@ -19,6 +19,6 @@ jobs:
   documentation-links:
     runs-on: ubuntu-latest
     steps:
-      - uses: readthedocs/actions/preview@v1
+      - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333  # v1
         with:
           project-slug: "pyflow-workflow-generator"

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 8.0.1
+    hooks:
+      - id: isort
+        name: ci qa isort --check
+        args: ["--check-only", "."]
+        pass_filenames: false
+        require_serial: true
+
+  - repo: https://github.com/psf/black
+    rev: 26.5.1
+    hooks:
+      - id: black
+        name: ci qa black --check
+        args: ["--check", "."]
+        pass_filenames: false
+        require_serial: true
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.3.0
+    hooks:
+      - id: flake8
+        name: ci qa flake8
+        args: ["."]
+        pass_filenames: false
+        require_serial: true

.readthedocs.yaml

@@ -19,6 +19,12 @@ conda:
     environment: docs/environment.yml
 
 build:
-    os: "ubuntu-20.04"
+    os: "ubuntu-22.04"
     tools:
         python: "mambaforge-4.10"
+    jobs:
+        # Install the checked-out source (e.g. the current PR) into the conda
+        # environment, so the docs are built against this code. Dependencies are
+        # already provided by docs/environment.yml, hence --no-deps.
+        post_install:
+            - pip install --no-deps .

README.md

@@ -34,6 +34,7 @@
 
 
 ## Installation
+
 To install pyflow using conda (including ecFlow):
 
     conda env create -n pyflow -f environment.yml
@@ -43,8 +44,37 @@ To install pyflow using pip (requires a local installation of ecFlow):
     pip install pyflow-workflow-generator
 
 ## Documentation
+
 The documentation can be found at <https://pyflow-workflow-generator.readthedocs.io>.
 
+## QA Checks (CI-equivalent)
+
+The CI `qa` job runs the following checks, in order:
+
+1. `isort --check .`
+2. `black --check .`
+3. `flake8 .`
+
+This repository includes a matching pre-commit configuration in
+`.pre-commit-config.yaml` using standard upstream hooks, pinned to the
+tool versions from the CI run:
+
+- `isort==8.0.1`
+- `black==26.5.1`
+- `flake8==7.3.0`
+
+Run locally:
+
+```bash
+python -m pip install ".[dev]"
+pre-commit run --all-files
+```
+
+The hooks are split into three sequential checks (isort, then black, then
+flake8) to avoid conflicts and to match CI behavior. Running `isort` before
+`black` prevents import-format churn, and running `flake8` last ensures linting
+sees code after formatting checks.
+
 
 ## License
 [Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities 

docs/_ext/ecflow_lexers.py

@@ -33,7 +33,7 @@ class EcflowDefLexer(RegexLexer):
                 bygroups(Keyword, Name.Constant),
             ),
             (
-                r"(repeat)(\s+(?:date(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))",
+                r"(repeat)(\s+(?:date(?:time)?(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))",  # noqa: E501
                 bygroups(Keyword, Name.Other, Name.Variable, Literal.Date),
             ),
             # Required

docs/content/api-reference.rst

@@ -160,6 +160,14 @@ Repeat
 
 .. autoclass:: pyflow.attributes.RepeatDateList
 
+.. _RepeatDateTime:
+
+.. autoclass:: pyflow.attributes.RepeatDateTime
+
+.. _RepeatDateTimeList:
+
+.. autoclass:: pyflow.attributes.RepeatDateTimeList
+
 .. _RepeatDay:
 
 .. autoclass:: pyflow.attributes.RepeatDay

docs/environment.yml

@@ -18,7 +18,6 @@ dependencies:
     - sphinx-rtd-theme
     - sphinx-copybutton
     - sphinx-tabs
-    - git+https://github.com/ecmwf/pyflow.git
 
 variables:
   QT_MAC_WANTS_LAYER: 1

docs/requirements.txt

@@ -1,6 +1,6 @@
 ipykernel
 nbsphinx
+pypandoc_binary
 sphinx-rtd-theme==0.5.2
 sphinx-copybutton==0.3.1
-sphinx-tabs
-git+https://github.com/ecmwf/pyflow.git
+sphinx-tabs

pyflow/__init__.py

@@ -30,6 +30,7 @@
     RepeatDate,
     RepeatDateList,
     RepeatDateTime,
+    RepeatDateTimeList,
     RepeatDay,
     RepeatEnumerated,
     RepeatInteger,

pyflow/attributes.py

@@ -25,7 +25,7 @@
     expression_from_json,
     make_expression,
 )
-from .importer import ecflow
+from .importer import ecflow, supported
 from .state import aborted, active, complete, queued, submitted, suspended, unknown
 
 NO_TRIGGER = False
@@ -641,6 +641,7 @@ def day_of_week(self):
     setattr(RepeatDate, day, property(lambda self: Eq(self.day_of_week, dow)))
 
 
+@supported(">=5.12.0")
 class RepeatDateTime(Exportable):
     """
     An attribute that allows a node to be repeated by a date+time value.
@@ -658,11 +659,17 @@ class RepeatDateTime(Exportable):
                               datetime.datetime(year=2019, month=12, day=31, hour=12, minute=0, second=0),
                               datetime.timedelta(hours=12, minutes=0, seconds=0))
 
-    Date and increment can also be strings::
+    Start/End values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with
+    hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``,
+    or simply a date ``yyyymmdd`` (the missing components are assumed to be 0), and increment can also be a string::
 
         pyflow.RepeatDateTime('REPEAT_DATETIME',
                               '20190101T120000', '20191231T120000', '12:00:00')
 
+    Note::
+
+        This repeat type is only supported in ecFlow 5.12.0 and later.
+
     """
 
     def __init__(
@@ -727,6 +734,92 @@ def day_of_week(self):
         return Mod(Add(Div(self, 86400), 4), 7)
 
 
+@supported(">=5.17.0")
+class RepeatDateTimeList(Repeat):
+    """
+    An attribute that allows a node to be repeated over a list of datetime values.
+
+    Parameters:
+        name(str): The name of the repeat attribute.
+        values(list of datetime/date): The list of datetime/date values, as datetime/date objects or strings.
+
+    Example::
+
+        pyflow.RepeatDateTimeList('REPEAT_DATETIME',
+                                  [datetime.date(year=2019, month=1, day=1),
+                                   datetime.datetime(year=2019, month=1, day=3, hour=12, minute=0, second=0)])
+
+    Values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with
+    hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``,
+    or simply a date ``yyyymmdd`` (the missing components are assumed to be 0)::
+
+        pyflow.RepeatDateTimeList('REPEAT_DATETIME', ['20190101T120000', '20190103'])
+
+    Note::
+
+        This repeat type is only supported in ecFlow 5.17.0 and later.
+    """
+
+    def __init__(self, name, values):
+        if values is None:
+            raise ValueError("values cannot be None")
+        if not isinstance(values, list):
+            raise TypeError("values must be a list")
+        if isinstance(values, list) and not values:
+            raise ValueError("values cannot be an empty list")
+        if not all(
+            isinstance(value, (datetime.datetime, datetime.date, str))
+            for value in values
+        ):
+            raise TypeError("values must be a list of datetime/date objects or strings")
+
+        super().__init__(name, values)
+
+    def _build(self, ecflow_parent):
+        # Format all datetime values as ISO 8601 basic format `yyyymmddTHHMMSS`
+        values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.values]
+
+        repeat = ecflow.RepeatDateTimeList(
+            str(self.name),
+            values,
+        )
+
+        ecflow_parent.add_repeat(repeat)
+
+    @property
+    def values(self):
+        """*list*: The list of datetime values."""
+        return [
+            x if isinstance(x, datetime.datetime) else as_date(x) for x in self.value
+        ]
+
+    def __add__(self, other):
+        return Add(self, other)
+
+    def __sub__(self, other):
+        return Sub(self, other)
+
+    @property
+    def second(self):
+        """*int*: The second of the repeat datetime."""
+        return Mod(self, 60)
+
+    @property
+    def minute(self):
+        """*int*: The minute of the repeat datetime."""
+        return Mod(Div(self, 60), 60)
+
+    @property
+    def hour(self):
+        """*int*: The hour of the repeat datetime."""
+        return Mod(Div(self, 3600), 24)
+
+    @property
+    def day_of_week(self):
+        """*int*: The day of the week of the repeat datetime."""
+        return Mod(Add(Div(self, 86400), 4), 7)
+
+
 def is_date(value):
     return (
         isinstance(value, (datetime.date, datetime.datetime))
@@ -939,25 +1032,63 @@ class InLimit(Attribute):
 
     Parameters:
         value(str,Limit_): The name of the limit or a limit object.
+        path(str): The optional path to the limit if the limit is not in the same node as the InLimit attribute.
+        tokens(int): The number of tokens to consume from the limit when a task is submitted.
+        limit_this_node_only(bool): Whether the limit should only apply to current node.
+        limit_submission(bool): Whether the limit should only apply to submissions
 
     Example::
 
         l = pyflow.Limit('l', 3)
         pyflow.InLimit(l)
     """
 
-    def __init__(self, value):
+    def __init__(
+        self,
+        value: str | Limit,
+        path: str = "",
+        tokens: int = 1,
+        limit_this_node_only: bool = False,
+        limit_submission: bool = False,
+    ):
         super().__init__("_" + str(value), value)
+        self.path = path
+        self.tokens = tokens
+        self.limit_this_node_only = limit_this_node_only
+        self.limit_submission = limit_submission
 
     def _build(self, ecflow_parent):
         value = self.value
         if NO_INLIMIT:
             return
         if isinstance(value, Limit):
             value = value.fullname.split(":")
-            ecflow_parent.add_inlimit(ecflow.InLimit(value[1], value[0]))
+            if self.path:
+                if self.path != value[0]:
+                    raise ValueError(
+                        "InLimit path {} does not match limit path {}".format(
+                            self.path, value[0]
+                        )
+                    )
+            ecflow_parent.add_inlimit(
+                ecflow.InLimit(
+                    value[1],
+                    value[0],
+                    self.tokens,
+                    self.limit_this_node_only,
+                    self.limit_submission,
+                )
+            )
         else:
-            ecflow_parent.add_inlimit(ecflow.InLimit(str(value)))
+            ecflow_parent.add_inlimit(
+                ecflow.InLimit(
+                    str(value),
+                    self.path,
+                    self.tokens,
+                    self.limit_this_node_only,
+                    self.limit_submission,
+                )
+            )
 
 
 class Inlimit(InLimit):

pyflow/host.py

@@ -437,6 +437,9 @@ def preamble_error_function(self, ecflowpath, exit_hook=None):
             set +x
             # Define a error handler
             ERROR() {
+                export EXIT_REASON="$1"
+                export EXIT_DETAIL="$2"
+                export EXIT_RC="${3:-1}"
                 export PATH=%(ecf_path)s:$PATH
                 set +eu  # Clear -eu flag, so we don't fail
                 wait  # wait for background process to stop
@@ -454,11 +457,11 @@ def preamble_error_function(self, ecflowpath, exit_hook=None):
             export SIGNAL_LIST='%(signal_list)s'
 
             for signal in $SIGNAL_LIST; do
-                trap "ERROR $signal \\"Signal $(kill -l $signal) ($signal) received \\"" $signal
+                trap "rc=\\$?; ERROR $signal \\"Signal $(kill -l $signal) ($signal) received\\" \\"\\$rc\\"" $signal
             done
 
             # Trap any calls to exit and errors caught by the -e flag
-            trap ERROR 0
+            trap 'rc=$?; ERROR EXIT "" "$rc"' 0
             set -x
             """) % {"ecf_path": ecflowpath, "signal_list": signal_list})  # noqa: E501
         return script