From 287b7c1f103d3ab56ab52ba8b2b8f6755c8e68be Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 13 Nov 2026 11:38:45 +0000 Subject: [PATCH 1/7] perf: Batch works out the state of chosen rows only `Batch.state(settings, keys)` reads the records and statuses of the rows of `keys` only, and `reduce`, `tweak`, and `cancel` use it, so that submitting a few rows of a batch of 100k rows no longer works out the state of every row. A table without edits returns the scan's rows as they are, without a copy per row. A row's result is the newest record of the batch's spec, label, and the row's key, whoever submitted it: one folder or one service holds one history. Co-Authored-By: Claude Opus 5.5 --- .../essapps/src/ess/apps/batch/reduction.py | 48 +++++++++++-------- packages/essapps/src/ess/apps/batch/state.py | 14 ++---- packages/essapps/src/ess/apps/batch/table.py | 21 ++++---- .../essapps/tests/batch/reduction_test.py | 30 ++++++++++-- 4 files changed, 69 insertions(+), 44 deletions(-) diff --git a/packages/essapps/src/ess/apps/batch/reduction.py b/packages/essapps/src/ess/apps/batch/reduction.py index b2acc280..d50bd408 100644 --- a/packages/essapps/src/ess/apps/batch/reduction.py +++ b/packages/essapps/src/ess/apps/batch/reduction.py @@ -143,18 +143,24 @@ def _cells(self, row: Row) -> dict[str, Any]: for c in self.columns } - def state(self, settings: Mapping[str, Any] | None = None) -> dict[str, RowState]: + def state( + self, + settings: Mapping[str, Any] | None = None, + keys: Sequence[str] | None = None, + ) -> dict[str, RowState]: """ - Where each row stands now, read from the client. With ``settings``, - also which rows are stale. A result that came from :meth:`tweak` is - stale only if the row's cells have changed since. + Where each row stands now, read from the client, or the rows of + ``keys`` if given. With ``settings``, also which rows are stale. A + result that came from :meth:`tweak` is stale only if the row's cells + have changed since. """ - found = self._results() - upstream = self._records_read() + rows = list(self.rows) if keys is None else [self.rows[k] for k in keys] + found = self._results(keys) + upstream = self._records_read(rows) status = self._status([*found.values(), *upstream]) - stale = set() if settings is None else self._stale(settings) + stale = set() if settings is None else self._stale(settings, rows) return state.row_states( - self.rows, + rows, self.spec.params, self.columns, found, @@ -163,13 +169,17 @@ def state(self, settings: Mapping[str, Any] | None = None) -> dict[str, RowState self._tweaked.keys(), ) - def _results(self) -> dict[str, Record]: + def _results(self, keys: Sequence[str] | None = None) -> dict[str, Record]: + """The result of each row, or of the rows of ``keys`` if given.""" records = self.client.records(label=self.label) - return state.results(records, SpecId.of(self.spec), self.client.submitter) - - def _records_read(self) -> list[Record]: - """The records that cells reference, if the client knows them all.""" - ids = sorted(state.references(self.rows)) + if keys is not None: + chosen = set(keys) + records = [r for r in records if r.member in chosen] + return state.results(records, SpecId.of(self.spec)) + + def _records_read(self, rows: list[Row]) -> list[Record]: + """The records that ``rows`` reference, if the client knows them all.""" + ids = sorted(state.references(rows)) try: return self.client.records(ids=ids) if ids else [] except KeyError: # the workspace refuses such a cell at submission @@ -179,9 +189,9 @@ def _status(self, records: list[Record]) -> dict[str, Status]: statuses = self.client.status(records) return {r.id: s for r, s in zip(records, statuses, strict=True)} - def _stale(self, settings: Mapping[str, Any]) -> set[str]: + def _stale(self, settings: Mapping[str, Any], rows: list[Row]) -> set[str]: template = self.template(settings) - return {row.key for row in self.rows if self._differs(row, template)} + return {row.key for row in rows if self._differs(row, template)} def _differs(self, row: Row, template: Template) -> bool: if row.key in self._tweaked: @@ -202,7 +212,7 @@ def reduce( it by key and the cell by column. A reason about the settings is named once, without a key. """ - states = self.state() + states = self.state(keys=keys) rows = [self.rows[key] for key in keys] refused = [e for k in keys for e in (*states[k].problems, *_running(states[k]))] if refused: @@ -258,7 +268,7 @@ def tweak(self, key: str, params: Mapping[str, Any]) -> Record: only if the cells change after this. Raises ``SubmitError`` if the row is still running. """ - if running := _running(self.state()[key]): + if running := _running(self.state(keys=[key])[key]): raise SubmitError(running) request = Request(self.spec, params=params) record = self.client.submit( @@ -281,7 +291,7 @@ def tweak_params( def cancel(self, keys: Sequence[str] | None = None) -> None: """Cancel the results still running, of ``keys`` if given, or of any row.""" - chosen = [r for k, r in self._results().items() if keys is None or k in keys] + chosen = list(self._results(keys).values()) status = self._status(chosen) self.client.cancel([r for r in chosen if status[r.id] is Status.PENDING]) diff --git a/packages/essapps/src/ess/apps/batch/state.py b/packages/essapps/src/ess/apps/batch/state.py index 0d27648a..8adc10a2 100644 --- a/packages/essapps/src/ess/apps/batch/state.py +++ b/packages/essapps/src/ess/apps/batch/state.py @@ -88,19 +88,13 @@ def problems( return tuple(found) -def results( - records: Iterable[Record], spec: SpecId, submitter: str -) -> dict[str, Record]: +def results(records: Iterable[Record], spec: SpecId) -> dict[str, Record]: """ The result of each row by its key: the newest of ``records``, oldest - first, of ``spec`` and ``submitter`` with that member. So a rerun of a row - is its result, and a record of someone else under the same label is not. + first, of ``spec`` with that member, whoever submitted it. So a rerun of a + row is its result, and one folder or one service holds one history. """ - return { - r.member: r - for r in records - if r.member is not None and r.spec == spec and r.submitter == submitter - } + return {r.member: r for r in records if r.member is not None and r.spec == spec} def references(rows: Iterable[Row]) -> set[str]: diff --git a/packages/essapps/src/ess/apps/batch/table.py b/packages/essapps/src/ess/apps/batch/table.py index 4b9179be..db2c6994 100644 --- a/packages/essapps/src/ess/apps/batch/table.py +++ b/packages/essapps/src/ess/apps/batch/table.py @@ -67,16 +67,17 @@ class RowTable: @cached_property def _rows(self) -> dict[str, Row]: - by_hand = {row.key for row in self.added} - shown = [r for r in self.scan if r.key not in self.removed | by_hand] - return { - row.key: replace(row, values=self._with_edits(row)) - for row in (*shown, *self.added) - } - - def _with_edits(self, row: Row) -> dict[str, Any]: - cells = {**row.values, **self.edits.get(row.key, {})} - return {column: v for column, v in cells.items() if v is not EMPTY} + hidden = self.removed | {row.key for row in self.added} + shown = [r for r in self.scan if r.key not in hidden] + return {row.key: self._with_edits(row) for row in (*shown, *self.added)} + + def _with_edits(self, row: Row) -> Row: + if (edits := self.edits.get(row.key)) is None: + return row + cells = {**row.values, **edits} + return replace( + row, values={column: v for column, v in cells.items() if v is not EMPTY} + ) def __iter__(self) -> Iterator[Row]: return iter(self._rows.values()) diff --git a/packages/essapps/tests/batch/reduction_test.py b/packages/essapps/tests/batch/reduction_test.py index 0d19f9c8..f9d1f406 100644 --- a/packages/essapps/tests/batch/reduction_test.py +++ b/packages/essapps/tests/batch/reduction_test.py @@ -9,7 +9,7 @@ from ess.apps import Client, Status, SubmitError, file, local from ess.apps.batch import Batch, CorrectedFiles, FileInfo, Row, ready, remaining -from ess.apps.testing import FakeStore, StagedFunction +from ess.apps.testing import CountingClient, FakeStore, StagedFunction from ess.spec import function_binding from .toys import COLUMNS, FIND, PLAIN, STAGED, Gate, Measure, Outputs, pair @@ -44,6 +44,27 @@ def test_rows_are_reduced_and_found_by_their_key( assert state['1'].status is Status.COMPLETED +def test_the_state_of_chosen_rows_reads_only_their_records( + client: Client, sample: Measure +) -> None: + counting = CountingClient(client) + batch = Batch( + counting, # type: ignore[arg-type] + STAGED, + columns=COLUMNS, + label='scan', + source=pair, + ) + sample(1, 'water') + sample(2, 'sds') + batch.scan() + client.wait(batch.reduce(SETTINGS, ['1'])) + counting.reads.clear() + + assert list(batch.state(keys=['2'])) == ['2'] + assert counting.reads['status'] == 0 + + def test_columns_must_be_parameters(client: Client) -> None: with pytest.raises(KeyError, match='nope'): Batch(client, STAGED, columns=('nope',), label='scan') @@ -328,16 +349,15 @@ def test_a_cell_reading_a_failed_record_stops_only_its_row( assert ready(reduce.state()) == ['1'] -def test_results_of_another_submitter_are_not_a_rows_result( +def test_a_record_of_another_submitter_is_a_rows_result( scanned: Batch, client: Client, other: Client ) -> None: client.wait(scanned.reduce(SETTINGS, ['1'])) - mine = scanned.state()['1'].record row = {k: scanned.rows['1'].values[k] for k in ('sample', 'transmission')} - other.compute(STAGED, params={**row, **SETTINGS}, label='scan', member='1') + theirs = other.compute(STAGED, params={**row, **SETTINGS}, label='scan', member='1') - assert scanned.state()['1'].record == mine + assert scanned.state()['1'].record == theirs def test_persisting_goes_with_the_batch(files: CorrectedFiles, sample: Measure) -> None: From 5990656c0e783b96c84542b0e0081ed2a5cf99fa Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 13 Nov 2026 11:38:54 +0000 Subject: [PATCH 2/7] Automatic reduction is a batch reduced as its files arrive `ess.apps.batch.auto.Automatic(batch, at=, settings=)` steps one batch: it scans, and reduces each row whose cell at the trigger column `at` holds a file that no record under the batch's label with the row's key holds at `at`. So a failed record, a changed setting, or a new partner does not reduce a row again, and a new file in a list at `at` does. Which files the records hold is read from the proposal's changes, so a restart needs nothing on disk. A row that is refused does not hold back the others, and `refused` says why. A step looks at the rows whose cells or records changed, and at the refused rows when the settings change. `run(autos, interval)` steps a list of them in order, so that a batch whose source reads another's results comes after it. `ess.apps.rules` (`Rule`, `RuleStatus`, `TriggerLoop`) is removed: a technique writes its pairing once, as the source of a batch. The stories E and the test of a moved folder use the batch. The check that a file at a list parameter is refused moves to the tests of requests. Co-Authored-By: Claude Opus 5.5 --- packages/essapps/src/ess/apps/__init__.py | 4 - packages/essapps/src/ess/apps/batch/auto.py | 146 ++++ packages/essapps/src/ess/apps/rules.py | 165 ----- packages/essapps/tests/batch/auto_test.py | 234 +++++++ packages/essapps/tests/batch/conftest.py | 15 +- packages/essapps/tests/batch/toys.py | 13 + .../essapps/tests/client/requests_test.py | 11 + packages/essapps/tests/folder_test.py | 38 +- packages/essapps/tests/rules_test.py | 631 ------------------ .../essapps/tests/stories/e_automatic_test.py | 103 ++- 10 files changed, 483 insertions(+), 877 deletions(-) create mode 100644 packages/essapps/src/ess/apps/batch/auto.py delete mode 100644 packages/essapps/src/ess/apps/rules.py create mode 100644 packages/essapps/tests/batch/auto_test.py delete mode 100644 packages/essapps/tests/rules_test.py diff --git a/packages/essapps/src/ess/apps/__init__.py b/packages/essapps/src/ess/apps/__init__.py index 23113114..a0a27b91 100644 --- a/packages/essapps/src/ess/apps/__init__.py +++ b/packages/essapps/src/ess/apps/__init__.py @@ -17,7 +17,6 @@ Status, Template, ) -from .rules import Rule, RuleStatus, TriggerLoop from .workspace import Workspace, local __all__ = [ @@ -36,8 +35,6 @@ 'RecordFailed', 'RecordStatus', 'Request', - 'Rule', - 'RuleStatus', 'Selector', 'SpecId', 'Stage', @@ -45,7 +42,6 @@ 'Store', 'SubmitError', 'Template', - 'TriggerLoop', 'Workspace', 'WorkspaceProtocol', 'file', diff --git a/packages/essapps/src/ess/apps/batch/auto.py b/packages/essapps/src/ess/apps/batch/auto.py new file mode 100644 index 00000000..ae16c6e5 --- /dev/null +++ b/packages/essapps/src/ess/apps/batch/auto.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +""" +Automatic reduction: a batch reduced as its files arrive. The guide is the +section "Automatic reduction" of ``packages/essapps/docs/batch.md``. + +Which files the records of a row hold at the trigger column is read from the +proposal's changes (:meth:`~ess.apps.Client.changes`), from the start when an +:class:`Automatic` is made, so a restart needs nothing on disk. What a step +keeps besides, the rows and the settings of the last step, only spares it +work. A step looks at the rows whose cells changed, whose records changed, and, +if the settings changed, the rows that were refused. It costs the scan of all +files, and the state and submission of the rows it looks at, which +:meth:`Batch.state ` works out for those rows only. +""" + +from __future__ import annotations + +import copy +import time +from collections.abc import Callable, Mapping, Sequence +from typing import Any + +from ess.spec import FileRef +from ess.spec.refs import iter_refs + +from ..errors import SubmitError +from ..records import Record +from .reduction import Batch +from .table import Row + + +class Automatic: + """ + Reduces each row of ``batch`` that holds a new file at the column ``at``. + See ``packages/essapps/docs/batch.md``. + + Parameters + ---------- + batch: + The batch, with a source. It should persist what it submits + (``persist=True``), since nothing releases its records. + at: + The trigger column, of a file or a list of files. + settings: + Gives the settings. It is called at each step, so that a change + applies to the rows not reduced yet. + + Raises + ------ + KeyError + If ``at`` is not a column of ``batch``. + """ + + def __init__( + self, + batch: Batch, + *, + at: str, + settings: Callable[[], Mapping[str, Any]], + ) -> None: + if at not in batch.columns: + raise KeyError(f'{at!r} is not a column of the batch') + self.batch = batch + self.at = at + self.settings = settings + self.refused: dict[str, str] = {} + """Why each row with a new file at ``at`` did not run, by key.""" + self._cursor = 0 + self._held: dict[str, set[FileRef]] = {} + self._rows: dict[str, Row] = {} + self._settings: Mapping[str, Any] | None = None + + def step(self) -> dict[str, Record]: + """ + Scan, and reduce the rows with a new file at ``at``; their records by + key. A row that cannot run, or whose request is refused, does not hold + back the others, and :attr:`refused` says why. + """ + batch = self.batch + changes = batch.client.changes(after=self._cursor) + self._cursor = changes.cursor + touched: set[str] = set() + for record in batch.client.records(ids=changes.records): + if record.label == batch.label and record.member is not None: + held = self._held.setdefault(record.member, set()) + held |= _files_at(record.request.params, self.at) + touched.add(record.member) + batch.scan() + rows = {row.key: row for row in batch.rows} + settings = self.settings() + look_at = {k for k, row in rows.items() if self._rows.get(k) != row} + look_at |= touched & rows.keys() + if settings != self._settings: + look_at |= self.refused.keys() & rows.keys() + # A copy, so that settings changed in place count as changed. + self._rows, self._settings = rows, copy.deepcopy(settings) + self.refused = { + k: why for k, why in self.refused.items() if k in rows and k not in look_at + } + todo = [ + k + for k in rows # in the order of the rows + if k in look_at + and _files_at(rows[k].values, self.at) - self._held.get(k, set()) + ] + return self._reduce(settings, todo) + + def _reduce( + self, settings: Mapping[str, Any], keys: list[str] + ) -> dict[str, Record]: + """Submit ``keys``, less those a refusal names; none if it names none.""" + while keys: + try: + return self.batch.reduce(settings, keys) + except SubmitError as error: # noqa: PERF203 + named = set(keys).intersection(e.entry for e in error.errors) + if not named: + self.refused |= dict.fromkeys(keys, str(error)) + return {} + for key in named: + errors = [e for e in error.errors if e.entry == key] + self.refused[key] = str(SubmitError(errors)) + keys = [k for k in keys if k not in named] + return {} + + +def run(autos: Sequence[Automatic], interval: float = 1.0) -> None: + """ + Step each of ``autos`` in the order given, then wait ``interval`` seconds, + forever. A batch whose source reads the results of another comes after it. + """ + while True: + for auto in autos: + auto.step() + time.sleep(interval) + + +def _files_at(params: Mapping[str, Any], at: str) -> set[FileRef]: + """The files at the parameter ``at``: its value, or each element of its list.""" + return { + ref + for loc, ref in iter_refs({at: params.get(at)}) + if isinstance(ref, FileRef) + and (len(loc) == 1 or (len(loc) == 2 and isinstance(loc[1], int))) + } diff --git a/packages/essapps/src/ess/apps/rules.py b/packages/essapps/src/ess/apps/rules.py deleted file mode 100644 index cfd7e3d5..00000000 --- a/packages/essapps/src/ess/apps/rules.py +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) -""" -Rules and the trigger loop: automatic reduction of new files, as described in -``docs/developer/automatic-reduction.md``. - -What to run for a file belongs to the application, which knows the technique: how -to pair runs, and which requests one file needs. The trigger loop is a controller -of a client that never runs in the workspace. It finds the new files and hands each -to the application's function. Which files a rule has handled comes from the -records under the rule's label. The loop reads them from the proposal's changes -(:meth:`~ess.apps.Client.changes`), from the start when it starts, so it holds no -more than a position in them and the files it has seen. -""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from dataclasses import dataclass - -from ess.spec import FileRef -from ess.spec.refs import iter_refs - -from .client import Client -from .files import Selector -from .records import Record - - -@dataclass(frozen=True) -class Rule: - """ - Call ``submit`` for each new file the selector matches. - - ``submit(client, file)`` submits what the file needs, any number of requests, - and returns the record that settles the file: one under ``label`` that holds - the file at the parameter ``at``, or in a row of it if ``at`` is a list. From - then on the rule has handled the file, whatever becomes of the record. - - To leave a file for the next step, such as a sample whose transmission run has - not arrived, ``submit`` raises ``LookupError`` or ``ValueError``, as a refused - request does with ``SubmitError``. The rule's status names the file and the - reason. A step that stops between two requests of a file calls ``submit`` again, - so it submits the settling record last, and reuses what an earlier call submitted - and did not fail, as ``docs/developer/automatic-reduction.md`` shows. - - ``submit`` does not wait for a record or read an output: the loop calls one - function at a time, and a request can reference a pending record. It persists what - it submits: the loop never releases, so the workspace should keep nothing for it, - and a reference must stay readable after a restart. - """ - - name: str - submit: Callable[[Client, FileRef], Record] - selector: Selector - label: str - at: str - - -@dataclass -class RuleStatus: - """ - A rule's last step. - - ``reason`` says why the rule submitted nothing, or which files it - skipped and why; it is ``None`` if the rule settled each new file. - """ - - reason: str | None = None - - -class TriggerLoop: - """Steps the ``rules`` on ``client``. Raises ``ValueError`` if two share a name.""" - - def __init__(self, client: Client, rules: list[Rule]) -> None: - names = [rule.name for rule in rules] - if duplicates := {name for name in names if names.count(name) > 1}: - raise ValueError(f'two rules are named {sorted(duplicates)}') - self._client = client - self._rules = rules - self._cursor = 0 - self._handled: dict[str, set[FileRef]] = {rule.name: set() for rule in rules} - self._status = {rule.name: RuleStatus() for rule in rules} - - def step(self) -> list[Record]: - """ - Call each rule for what arrived since the last step; return the records - that settled a file. - - A file that its rule skips, or whose request is refused, stays new, so the - next step tries it again; the other files and rules go on. - - Raises - ------ - TypeError - If ``submit`` returned something that does not settle the file. The - next step would submit for the file again, so the loop stops. - """ - self._read() - settled: list[Record] = [] - for rule in self._rules: - handled = self._handled[rule.name] - reasons: list[str] = [] - new = [ - r for r in self._client.files.list(rule.selector) if r not in handled - ] - # A record may settle a later file of this list too, as a row. - for ref in new: - if ref in handled: - continue - try: - record = rule.submit(self._client, ref) - except (LookupError, ValueError) as error: # a SubmitError too - reasons.append(f'skipped {ref.file}: {error}') - continue - _check_settles(record, rule, ref) - self._take(record) - settled.append(record) - reason = '; '.join(reasons) or (None if new else 'no new file') - self._status[rule.name] = RuleStatus(reason) - return settled - - def run(self, interval: float = 1.0) -> None: - """Step forever.""" - while True: - self.step() - time.sleep(interval) - - def status(self, rule: Rule) -> RuleStatus: - return self._status[rule.name] - - def _read(self) -> None: - """Take the records that changed into each rule's handled files.""" - changes = self._client.changes(after=self._cursor) - self._cursor = changes.cursor - for record in self._client.records(ids=changes.records): - self._take(record) - - def _take(self, record: Record) -> None: - for rule in self._rules: - if record.label == rule.label: - self._handled[rule.name] |= _files_at(record, rule.at) - - -def _files_at(record: Record, at: str) -> set[FileRef]: - """The files that ``record`` holds at the parameter ``at``, or in a row of it.""" - return { - ref - for loc, ref in iter_refs(record.request.params) - if isinstance(ref, FileRef) - and loc[0] == at - and (len(loc) == 1 or (len(loc) == 2 and isinstance(loc[1], int))) - } - - -def _check_settles(record: object, rule: Rule, ref: FileRef) -> None: - if ( - not isinstance(record, Record) - or record.label != rule.label - or ref not in _files_at(record, rule.at) - ): - raise TypeError( - f'{rule.name}: submit for {ref.file} returned {record!r}, not a record ' - f'under the label {rule.label!r} that holds the file at {rule.at!r}' - ) diff --git a/packages/essapps/tests/batch/auto_test.py b/packages/essapps/tests/batch/auto_test.py new file mode 100644 index 00000000..73b4eb0b --- /dev/null +++ b/packages/essapps/tests/batch/auto_test.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +"""Automatic reduction: which rows a step reduces, and why a row did not run.""" + +from collections.abc import Sequence +from dataclasses import replace +from typing import Any + +import pytest + +from ess.apps import Client +from ess.apps.batch import Batch, FileInfo, Row +from ess.apps.batch.auto import Automatic +from ess.apps.testing import FakeFiles + +from .toys import COLUMNS, FIND, PLAIN, SERIES, STAGED, Gate, Measure, pair + +SETTINGS: dict[str, Any] = {'binning': {'bins': 2}} + + +@pytest.fixture +def auto(batch: Batch) -> Automatic: + return Automatic(batch, at='sample', settings=lambda: SETTINGS) + + +def _angles(files: Sequence[FileInfo]) -> list[Row]: + """A row per sample: all its runs so far, in run order.""" + runs: dict[str, list[Any]] = {} + for f in sorted(files, key=lambda f: f.metadata['run']): + runs.setdefault(f.metadata['sample'], []).append(f.ref) + return [Row(name, {'runs': refs}) for name, refs in runs.items()] + + +@pytest.fixture +def series(client: Client) -> Automatic: + batch = Batch(client, SERIES, columns=('runs',), label='series', source=_angles) + return Automatic(batch, at='runs', settings=dict) + + +def test_each_new_row_is_reduced_once( + auto: Automatic, client: Client, sample: Measure +) -> None: + sample(1, 'water') + sample(2, 'sds') + first = auto.step() + client.wait(first) + sample(3, 'water') + + assert list(first) == ['1', '2'] + assert list(auto.step()) == ['3'] + assert auto.step() == {} + + +def test_the_trigger_column_must_be_a_column(batch: Batch) -> None: + with pytest.raises(KeyError, match='offset'): + Automatic(batch, at='offset', settings=dict) + + +def test_a_row_whose_record_failed_is_not_reduced_again( + auto: Automatic, client: Client, sample: Measure, fake: FakeFiles +) -> None: + fake.corrupt(sample(1, 'water')) + assert client.wait(auto.step()) == {'1': 'failed'} + + assert auto.step() == {} + assert auto.refused == {} + + +def test_a_new_partner_does_not_reduce_a_row_again( + auto: Automatic, client: Client, sample: Measure, fake: FakeFiles +) -> None: + sample(1, 'water') + client.wait(auto.step()) + + later = fake.measure(1002, 2.0, role='transmission', sample='water') + + assert auto.batch.rows['1'].values['transmission'] != later # before the scan + assert auto.step() == {} + assert auto.batch.rows['1'].values['transmission'] == later + + +def test_a_file_that_arrives_again_is_not_reduced_again( + auto: Automatic, client: Client, sample: Measure +) -> None: + sample(1, 'water') + client.wait(auto.step()) + + sample(1, 'water') + + assert auto.step() == {} + + +def test_a_new_loop_of_any_submitter_reduces_only_what_no_record_holds( + auto: Automatic, client: Client, other: Client, sample: Measure +) -> None: + sample(1, 'water') + client.wait(auto.step()) + sample(2, 'sds') + + batch = Batch(other, STAGED, columns=COLUMNS, label='scan', source=pair) + restarted = Automatic(batch, at='sample', settings=lambda: SETTINGS) + + assert list(restarted.step()) == ['2'] + + +def test_a_refused_row_does_not_hold_back_the_others( + auto: Automatic, batch: Batch, sample: Measure +) -> None: + for run, name in [(1, 'water'), (2, 'sds'), (3, 'dmso')]: + sample(run, name) + batch.scan() + batch.rows = batch.rows.edit('2', {'thickness': -1.0}) + + assert list(auto.step()) == ['1', '3'] + assert auto.refused == { + '2': '2: thickness: Input should be greater than or equal to 0' + } + + +def test_a_waiting_row_runs_once_its_partner_arrives( + auto: Automatic, sample: Measure, fake: FakeFiles +) -> None: + sample(1, 'water', partner=False) + assert auto.step() == {} + assert auto.refused == {'1': '1: transmission: empty'} + + fake.measure(1001, 1.0, role='transmission', sample='water') + + assert list(auto.step()) == ['1'] + assert auto.refused == {} + + +def test_rows_refused_for_the_settings_run_once_the_settings_change( + batch: Batch, sample: Measure +) -> None: + settings = {'binning': {'bins': 0}} + auto = Automatic(batch, at='sample', settings=lambda: settings) + sample(1, 'water') + sample(2, 'sds') + assert auto.step() == {} + assert auto.refused == dict.fromkeys( + ['1', '2'], 'binning.bins: Input should be greater than or equal to 1' + ) + + settings = SETTINGS + + assert list(auto.step()) == ['1', '2'] + assert auto.refused == {} + + +def test_a_changed_setting_applies_to_new_rows_only( + batch: Batch, client: Client, sample: Measure +) -> None: + settings = SETTINGS + auto = Automatic(batch, at='sample', settings=lambda: settings) + sample(1, 'water') + client.wait(auto.step()) + + settings = {'binning': {'bins': 3}} + sample(2, 'sds') + + (record,) = auto.step().values() + assert record.member == '2' + assert record.request.params['binning'] == {'bins': 3} + + +def test_a_new_file_in_the_list_at_the_trigger_column_reduces_the_row_again( + series: Automatic, client: Client, fake: FakeFiles +) -> None: + runs = [fake.measure(1, 1.0, sample='si')] + client.wait(series.step()) + runs += [fake.measure(n, float(n), sample='si') for n in (3, 2)] + + (record,) = series.step().values() + client.wait(record) + + assert record.request.params['runs'] == [runs[0], runs[2], runs[1]] + assert client.output(record, 'total') == 6.0 + assert series.step() == {} + + +def test_a_running_row_with_a_new_file_runs_once_its_record_finishes( + series: Automatic, client: Client, fake: FakeFiles, gate: Gate +) -> None: + gate.close() + fake.measure(1, 1.0, sample='si') + first = series.step() + fake.measure(2, 2.0, sample='si') + assert series.step() == {} + assert series.refused == {'si': 'si: still running'} + + gate.open() + client.wait(first) + + (record,) = series.step().values() + assert len(record.request.params['runs']) == 2 + + +def _samples(files: Sequence[FileInfo]) -> list[Row]: + return [ + Row(str(f.metadata['run']), {'sample': f.ref}) + for f in files + if f.metadata.get('role') == 'sample' + ] + + +def test_a_row_reads_the_record_of_its_row_in_a_batch_stepped_before( + client: Client, sample: Measure +) -> None: + find = Batch(client, FIND, columns=('sample',), label='find', source=_samples) + + def with_offset(files: Sequence[FileInfo]) -> list[Row]: + found = {k: s.record for k, s in find.state().items() if s.record} + return [ + replace(row, values={**row.values, 'offset': found[row.key].ref('offset')}) + if row.key in found + else row + for row in pair(files) + ] + + reduce = Batch( + client, PLAIN, columns=(*COLUMNS, 'offset'), label='reduce', source=with_offset + ) + autos = [ + Automatic(find, at='sample', settings=dict), + Automatic(reduce, at='sample', settings=lambda: SETTINGS), + ] + sample(1, 'water') + + found, reduced = (auto.step() for auto in autos) + + assert reduced['1'].request.params['offset'] == found['1'].ref('offset') + assert client.wait(reduced['1']) == 'completed' + assert client.output(reduced['1'], 'total') == 2.0 diff --git a/packages/essapps/tests/batch/conftest.py b/packages/essapps/tests/batch/conftest.py index 005b4b39..9a00adbe 100644 --- a/packages/essapps/tests/batch/conftest.py +++ b/packages/essapps/tests/batch/conftest.py @@ -2,7 +2,7 @@ # Copyright (c) 2026 Scipp contributors (https://github.com/scipp) """The toy reduction of ``toys`` on a real in-process workspace.""" -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path import pandas as pd @@ -24,6 +24,14 @@ def _find(params: toys.FindParams) -> toys.FindOutputs: return toys.FindOutputs(offset=toys.Offset(value=value)) +def _series(gate: toys.Gate) -> Callable[[toys.SeriesParams], toys.Outputs]: + def series(params: toys.SeriesParams) -> toys.Outputs: + gate.wait() + return toys.Outputs(total=sum(read_file(run) for run in params.runs)) + + return series + + @pytest.fixture def gate() -> toys.Gate: return toys.Gate() @@ -53,12 +61,13 @@ def reduce(params: toys.Params) -> toys.Outputs: @pytest.fixture def workspace( - files: CorrectedFiles, staged_binding: StagedFunction + files: CorrectedFiles, staged_binding: StagedFunction, gate: toys.Gate ) -> Iterator[Workspace]: plain = function_binding(toys.PLAIN, staged_binding.function) find = function_binding(toys.FIND, _find) + series = function_binding(toys.SERIES, _series(gate)) workspace = Workspace( - proposal='p1', files=files, bind=[staged_binding, plain, find] + proposal='p1', files=files, bind=[staged_binding, plain, find, series] ) yield workspace workspace.close() diff --git a/packages/essapps/tests/batch/toys.py b/packages/essapps/tests/batch/toys.py index a95e6ec9..793a5f8a 100644 --- a/packages/essapps/tests/batch/toys.py +++ b/packages/essapps/tests/batch/toys.py @@ -42,6 +42,10 @@ class FindOutputs(BaseModel): offset: Offset +class SeriesParams(BaseModel): + runs: list[RunFile] = Field(min_length=1) + + STAGED = WorkflowSpec( name='staged', version=1, @@ -66,6 +70,15 @@ class FindOutputs(BaseModel): params=FindParams, outputs=FindOutputs, ) +SERIES = WorkflowSpec( + name='series', + version=1, + title='Series', + description='A toy that sums a list of runs.', + params=SeriesParams, + outputs=Outputs, + tables=('runs',), +) COLUMNS = ('sample', 'transmission', 'thickness') diff --git a/packages/essapps/tests/client/requests_test.py b/packages/essapps/tests/client/requests_test.py index d2379174..1e6227f1 100644 --- a/packages/essapps/tests/client/requests_test.py +++ b/packages/essapps/tests/client/requests_test.py @@ -244,6 +244,17 @@ def test_an_unknown_file_is_refused(client: Client) -> None: client.submit(toys.load_request(9)) +def test_a_file_at_a_list_parameter_is_refused( + client: Client, files: FakeFiles +) -> None: + run = files.measure(1, 2.0) + + with pytest.raises( + SubmitError, match=rf'^runs: {run.file} is a reference to a whole list; ' + ): + client.submit(toys.SUM, params={'runs': run}) + + def test_a_file_of_another_proposal_is_refused( client: Client, files: FakeFiles ) -> None: diff --git a/packages/essapps/tests/folder_test.py b/packages/essapps/tests/folder_test.py index 218e416e..50608ec9 100644 --- a/packages/essapps/tests/folder_test.py +++ b/packages/essapps/tests/folder_test.py @@ -10,8 +10,9 @@ import pytest import scipp as sc -from ess.apps import Client, Record, Rule, Selector, TriggerLoop, file +from ess.apps import Client, Selector, file from ess.apps.batch import Batch, FileInfo, Row +from ess.apps.batch.auto import Automatic from ess.apps.batch.frames import batch_table from ess.apps.folder import FolderStore, open_folder, scan_folder from ess.apps.testing import read_file @@ -152,25 +153,19 @@ def test_a_request_of_a_second_session_references_a_persisted_output( assert second.output(total, 'total') == 4.0 -def _find_each(client: Client, ref: FileRef) -> Record: - """Find the offset of a run, as the member of its run under ``found``.""" - (metadata,) = client.files.metadata([ref]) - return client.submit( - toys.FIND, - params={'sample': ref}, - label='found', - member=metadata['run'], - persist=True, - ) - - -def _found(client: Client) -> pd.DataFrame: - """The batch of a row per run, with its result under ``found``, as a table.""" +def _found(client: Client) -> Batch: + """A row per run, whose offset is found under the label ``found``.""" def rows(files: Sequence[FileInfo]) -> list[Row]: return [Row(f.metadata['run'], {'sample': f.ref}) for f in files] - batch = Batch(client, toys.FIND, columns=['sample'], label='found', source=rows) + return Batch( + client, toys.FIND, columns=['sample'], label='found', source=rows, persist=True + ) + + +def _table(client: Client) -> pd.DataFrame: + batch = _found(client) batch.scan() return batch_table(batch, batch.state()) @@ -178,11 +173,12 @@ def rows(files: Sequence[FileInfo]) -> list[Row]: def test_a_moved_folder_keeps_the_identity_of_its_files_and_what_was_done_with_them( folder: Path, bindings: list[Binding] ) -> None: - rule = Rule('find', _find_each, selector=Selector(), label='found', at='sample') with open_folder(folder, bindings) as first: - found = TriggerLoop(first, [rule]).step() + found = list( + Automatic(_found(first), at='sample', settings=dict).step().values() + ) first.wait(found) - table = _found(first) + table = _table(first) files = [r.request.params['sample'] for r in found] assert files == [ FileRef(file='path:run-1.json'), @@ -193,8 +189,8 @@ def test_a_moved_folder_keeps_the_identity_of_its_files_and_what_was_done_with_t with open_folder(folder.rename(folder.with_name('moved')), bindings) as second: assert second.files.list(Selector()) == files assert [second.output(r, 'offset').value for r in found] == [1, 2, 3] - assert TriggerLoop(second, [rule]).step() == [] - assert _found(second).equals(table) + assert Automatic(_found(second), at='sample', settings=dict).step() == {} + assert _table(second).equals(table) again = second.submit(found[0].request) second.wait([again]) assert second.output(again, 'offset').value == 1 diff --git a/packages/essapps/tests/rules_test.py b/packages/essapps/tests/rules_test.py deleted file mode 100644 index 235ede1a..00000000 --- a/packages/essapps/tests/rules_test.py +++ /dev/null @@ -1,631 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) -"""The trigger loop: what one step submits, and why a rule submits nothing.""" - -import re -from collections.abc import Iterator -from pathlib import Path -from typing import Any - -import pytest -from pydantic import BaseModel, Field - -from ess.apps import ( - Client, - Record, - Rule, - Selector, - Status, - SubmitError, - Template, - TriggerLoop, - Workspace, - file, -) -from ess.apps.testing import CountingClient, FakeFiles, FakeStore, read_file -from ess.spec import ( - FileRef, - MaskFile, - Referenced, - RunFile, - WorkflowSpec, - function_binding, -) - - -class LoadParams(BaseModel): - sample: RunFile - - -class LoadMaskParams(BaseModel): - mask: MaskFile - - -class NormalizeParams(BaseModel): - sample: RunFile - reference: RunFile - - -class SumRunsParams(BaseModel): - runs: list[RunFile] = Field(min_length=1) - - -class Group(BaseModel): - run: RunFile - - -class LoadGroupParams(BaseModel): - group: Group - - -class Centre(BaseModel): - x: float - - -class CentreOutputs(BaseModel): - centre: Centre - - -class ReduceParams(BaseModel): - sample: RunFile - transmission: RunFile - centre: Referenced[Centre] - - -class ValueOutputs(BaseModel): - value: float - - -def _spec(name: str, params: type[BaseModel], **kw: Any) -> WorkflowSpec: - kw.setdefault('outputs', ValueOutputs) - return WorkflowSpec( - name=name, version=1, title=name, description=name, params=params, **kw - ) - - -LOAD = _spec('load', LoadParams) -LOAD_GROUP = _spec('load-group', LoadGroupParams) -LOAD_MASK = _spec('load-mask', LoadMaskParams) -NORMALIZE = _spec('normalize', NormalizeParams) -SUM_RUNS = _spec('sum-runs', SumRunsParams, tables=('runs',)) -FIND_CENTRE = _spec('find-centre', LoadParams, outputs=CentreOutputs) -REDUCE = _spec('reduce', ReduceParams) - - -def load(params: LoadParams) -> ValueOutputs: - return ValueOutputs(value=read_file(params.sample)) - - -def load_group(params: LoadGroupParams) -> ValueOutputs: - return ValueOutputs(value=read_file(params.group.run)) - - -def load_mask(params: LoadMaskParams) -> ValueOutputs: - return ValueOutputs(value=read_file(params.mask)) - - -def sum_runs(params: SumRunsParams) -> ValueOutputs: - return ValueOutputs(value=sum(read_file(run) for run in params.runs)) - - -def normalize(params: NormalizeParams) -> ValueOutputs: - return ValueOutputs(value=read_file(params.sample) / read_file(params.reference)) - - -def find_centre(params: LoadParams) -> CentreOutputs: - return CentreOutputs(centre=Centre(x=read_file(params.sample) / 10)) - - -def reduce(params: ReduceParams) -> ValueOutputs: - transmission = read_file(params.transmission) - return ValueOutputs(value=read_file(params.sample) / transmission + params.centre.x) - - -@pytest.fixture -def files(tmp_path: Path) -> FakeFiles: - return FakeFiles(proposal='p1', directory=tmp_path) - - -@pytest.fixture -def client(files: FakeFiles) -> Iterator[Client]: - workspace = Workspace( - proposal='p1', - files=files, - bind=[ - function_binding(LOAD, load), - function_binding(LOAD_GROUP, load_group), - function_binding(LOAD_MASK, load_mask), - function_binding(NORMALIZE, normalize), - function_binding(SUM_RUNS, sum_runs), - function_binding(FIND_CENTRE, find_centre), - function_binding(REDUCE, reduce), - ], - store=FakeStore(), - ) - yield Client(workspace, submitter='anna') - workspace.close() - - -def _rule(template: Template, at: str, *, name: str = 'rule', **kw: Any) -> Rule: - """A rule that fills the template's one blank ``at`` with each file.""" - - def submit(client: Client, ref: FileRef) -> Record: - member = str(client.files.metadata(ref)['run']) - request = template.fill(params={at: ref}) - return client.submit(request, label=label, member=member, persist=True) - - label = kw.setdefault('label', name) - kw.setdefault('selector', Selector(role='sample')) - return Rule(name, submit, at=at, **kw) - - -@pytest.fixture -def loading() -> Rule: - return _rule(Template(LOAD, blanks=('sample',)), 'sample', label='loaded') - - -# What one file needs, as an application decides. The metadata stand in for what the -# files will say of their use: `run_label` is shared by the runs of one sample, and -# `run_type` says what each measured. -def _transmission_of(client: Client, sample: FileRef) -> FileRef: - label = client.files.metadata(sample)['run_label'] - found = client.files.list(Selector(run_type='trans', run_label=label)) - if not found: - raise LookupError(f'no transmission run of {label} yet') - return found[-1] - - -def _centre_of(client: Client, sample: FileRef) -> Record: - """The sample's centre: one that an earlier call submitted, or a new one.""" - member = str(client.files.metadata(sample)['run']) - for record in reversed(client.records(label='centres')): - if record.member == member and client.status(record) not in ( - Status.FAILED, - Status.CANCELLED, - ): - return record - return client.submit( - FIND_CENTRE, - params={'sample': sample}, - label='centres', - member=member, - persist=True, - ) - - -def _reduce(client: Client, sample: FileRef, transmission: Any) -> Record: - return client.submit( - REDUCE, - params={ - 'sample': sample, - 'transmission': transmission, - 'centre': _centre_of(client, sample).ref('centre'), - }, - label='reduced', - member=str(client.files.metadata(sample)['run']), - persist=True, - ) - - -def _reduce_with_its_own_centre(client: Client, sample: FileRef) -> Record: - return _reduce(client, sample, _transmission_of(client, sample)) - - -@pytest.fixture -def reducing() -> Rule: - return Rule( - 'reduce', - _reduce_with_its_own_centre, - selector=Selector(run_type='sans'), - label='reduced', - at='sample', - ) - - -def test_one_file_may_submit_a_chain_of_requests( - client: Client, files: FakeFiles, reducing: Rule -) -> None: - sample = files.measure(1, 8.0, run_type='sans', run_label='water') - files.measure(2, 2.0, run_type='trans', run_label='water') - - (reduced,) = TriggerLoop(client, rules=[reducing]).step() - - assert reduced.request.params['sample'] == sample - assert client.output(reduced, 'value') == 8.0 / 2.0 + 0.8 - assert len(client.records(label='centres')) == 1 - - -def test_a_file_the_function_skips_is_tried_again( - client: Client, files: FakeFiles, reducing: Rule -) -> None: - loop = TriggerLoop(client, rules=[reducing]) - sample = files.measure(1, 8.0, run_type='sans', run_label='water') - - assert loop.step() == [] - assert loop.status(reducing).reason == ( - f'skipped {sample.file}: no transmission run of water yet' - ) - - files.measure(2, 2.0, run_type='trans', run_label='water') # arrives after it - (reduced,) = loop.step() - - assert reduced.request.params['sample'] == sample - assert loop.status(reducing).reason is None - assert loop.step() == [] - - -def test_a_file_that_cannot_go_on_does_not_hold_back_the_others( - client: Client, files: FakeFiles, reducing: Rule -) -> None: - waiting = files.measure(1, 8.0, run_type='sans', run_label='water') - other = files.measure(2, 6.0, run_type='sans', run_label='sds') - files.measure(3, 3.0, run_type='trans', run_label='sds') - loop = TriggerLoop(client, rules=[reducing]) - - (reduced,) = loop.step() - - assert reduced.request.params['sample'] == other - assert waiting.file in loop.status(reducing).reason - - -def test_two_runs_of_one_sample_are_two_files( - client: Client, files: FakeFiles, reducing: Rule -) -> None: - runs = [files.measure(n, 8.0, run_type='sans', run_label='water') for n in (1, 2)] - files.measure(3, 2.0, run_type='trans', run_label='water') - - reduced = TriggerLoop(client, rules=[reducing]).step() - - assert [r.request.params['sample'] for r in reduced] == runs - - -def test_a_call_after_a_refusal_reuses_what_it_submitted( - client: Client, files: FakeFiles -) -> None: - rule = Rule( - 'reduce', - lambda client, sample: _reduce(client, sample, file(run=2)), - selector=Selector(run_type='sans'), - label='reduced', - at='sample', - ) - files.measure(1, 8.0, run_type='sans') # run 2 is not known yet - loop = TriggerLoop(client, rules=[rule]) - - assert loop.step() == [] - assert 'unknown file run:2' in loop.status(rule).reason - assert loop.step() == [] - - files.measure(2, 2.0) - (reduced,) = TriggerLoop(client, rules=[rule]).step() # as after a restart - - assert client.output(reduced, 'value') == 8.0 / 2.0 + 0.8 - assert len(client.records(label='centres')) == 1 - - -def test_a_file_whose_centre_failed_settles_with_a_failed_record( - client: Client, files: FakeFiles, reducing: Rule -) -> None: - sample = files.measure(1, 8.0, run_type='sans', run_label='water') - files.measure(2, 2.0, run_type='trans', run_label='water') - files.corrupt(sample) - loop = TriggerLoop(client, rules=[reducing]) - failed_centre = client.submit( - FIND_CENTRE, params={'sample': sample}, label='centres', member='1' - ) - assert client.wait(failed_centre) == 'failed' - - # A new centre may fail before the reduction references it: then the next step - # tries again. - settled: list[Record] = [] - for _ in range(100): - if settled := loop.step(): - break - client.wait(client.records(label='centres')) - - (reduced,) = settled - assert client.wait(reduced) == 'failed' - assert loop.step() == [] - - -def test_a_function_that_raises_value_error_skips_the_file( - client: Client, files: FakeFiles -) -> None: - def submit(client: Client, ref: FileRef) -> Record: - raise ValueError('not this one') - - rule = Rule('rule', submit, selector=Selector(), label='rule', at='sample') - run = files.measure(1, 2.0) - loop = TriggerLoop(client, rules=[rule]) - - assert loop.step() == [] - assert loop.status(rule).reason == f'skipped {run.file}: not this one' - - -def test_any_other_exception_of_a_function_stops_the_loop( - client: Client, files: FakeFiles -) -> None: - def submit(client: Client, ref: FileRef) -> Record: - raise RuntimeError('a bug') - - rule = Rule('rule', submit, selector=Selector(), label='rule', at='sample') - files.measure(1, 2.0) - - with pytest.raises(RuntimeError, match='a bug'): - TriggerLoop(client, rules=[rule]).step() - - -def test_rules_of_two_labels_each_handle_a_file( - client: Client, files: FakeFiles -) -> None: - template = Template(LOAD, blanks=('sample',)) - rules = [_rule(template, 'sample', name=name) for name in ('first', 'second')] - files.measure(1, 2.0, role='sample') - - settled = TriggerLoop(client, rules=rules).step() - - assert [r.label for r in settled] == ['first', 'second'] - - -def test_a_file_in_a_group_at_the_parameter_does_not_count_as_handled( - client: Client, files: FakeFiles -) -> None: - def submit(client: Client, ref: FileRef) -> Record: - raise ValueError('called') - - rule = Rule('rule', submit, selector=Selector(), label='x', at='group') - run = files.measure(1, 2.0) - client.submit(LOAD_GROUP, params={'group': {'run': run}}, label='x') - loop = TriggerLoop(client, rules=[rule]) - - loop.step() - - assert loop.status(rule).reason == f'skipped {run.file}: called' - - -def test_two_rules_of_one_name_are_refused(client: Client, loading: Rule) -> None: - with pytest.raises(ValueError, match=r"two rules are named \['rule'\]"): - TriggerLoop(client, rules=[loading, loading]) - - -@pytest.mark.parametrize( - 'returned', - ['nothing', 'another-label', 'another-file'], -) -def test_a_function_whose_result_does_not_settle_the_file_stops_the_loop( - client: Client, files: FakeFiles, returned: str -) -> None: - other = files.measure(9, 1.0) - - def submit(client: Client, ref: FileRef) -> Any: - if returned == 'nothing': - return None - sample = other if returned == 'another-file' else ref - label = 'elsewhere' if returned == 'another-label' else 'rule' - return client.submit(LOAD, params={'sample': sample}, label=label, persist=True) - - rule = Rule( - 'rule', submit, selector=Selector(role='sample'), label='rule', at='sample' - ) - run = files.measure(1, 2.0, role='sample') - - with pytest.raises(TypeError, match=f'rule: submit for {re.escape(run.file)}'): - TriggerLoop(client, rules=[rule]).step() - - -def test_a_rule_whose_requests_are_refused_does_not_stop_the_others( - client: Client, files: FakeFiles, loading: Rule -) -> None: - unknown = _rule( - Template(NORMALIZE, params={'reference': file(run=9)}, blanks=('sample',)), - 'sample', - name='unknown', - label='normalized', - ) - loop = TriggerLoop(client, rules=[unknown, loading]) - run = files.measure(2, 5.0, role='sample') - - (loaded,) = loop.step() - - assert loaded.label == 'loaded' - assert loop.status(unknown).reason == ( - f'skipped {run.file}: reference: unknown file run:9' - ) - assert loop.status(loading).reason is None - - -def test_a_rule_says_when_there_is_no_new_file(client: Client, loading: Rule) -> None: - loop = TriggerLoop(client, rules=[loading]) - - assert loop.step() == [] - assert loop.status(loading).reason == 'no new file' - - -def test_a_rule_submits_each_new_file_though_one_cannot_be_read( - client: Client, files: FakeFiles -) -> None: - rule = _rule(Template(LOAD, blanks=('sample',)), 'sample', selector=Selector()) - loop = TriggerLoop(client, rules=[rule]) - runs = [files.measure(n, float(n)) for n in (1, 2, 3)] - files.corrupt(runs[0]) - - submitted = loop.step() - - assert [r.request.params['sample'] for r in submitted] == runs - assert client.wait(submitted) == ['failed', 'completed', 'completed'] - assert loop.status(rule).reason is None - assert loop.step() == [] # a failed record settles its file - - -def test_a_rule_selects_any_kind_of_file(client: Client, files: FakeFiles) -> None: - rule = _rule( - Template(LOAD_MASK, blanks=('mask',)), - 'mask', - selector=Selector(kind='mask', instrument='loki'), - ) - files.measure(1, 5.0, role='sample') - files.measure(2, 5.0, kind='mask', instrument='dream') - loki = files.measure(3, 3.0, kind='mask', instrument='loki') - - (loaded,) = TriggerLoop(client, rules=[rule]).step() - - assert loaded.request.params == {'mask': loki} - assert client.output(loaded, 'value') == 3.0 - - -def test_a_rule_has_handled_a_file_only_at_the_place_it_fills( - client: Client, files: FakeFiles -) -> None: - first = files.measure(1, 6.0, role='sample') - second = files.measure(2, 2.0, role='sample') - template = Template(NORMALIZE, params={'reference': second}, blanks=('sample',)) - rule = _rule(template, 'sample', label='normalized') - # holds the first at the place, and the second elsewhere - client.submit(template.fill(params={'sample': first}), label='normalized') - - (submitted,) = TriggerLoop(client, rules=[rule]).step() - - assert submitted.member == '2' - assert submitted.request.params['sample'] == second - - -def _sum_all_so_far(client: Client, ref: FileRef) -> Record: - runs = client.files.list(Selector(role='sample')) - template = Template(SUM_RUNS, blanks=('runs',)) - return client.submit( - template.fill(params={'runs': runs}), label='summed', persist=True - ) - - -@pytest.fixture -def summed() -> Rule: - return Rule( - 'summed', - _sum_all_so_far, - selector=Selector(role='sample'), - label='summed', - at='runs', - ) - - -def test_a_record_settles_each_file_in_a_row_of_a_list( - client: Client, files: FakeFiles, summed: Rule -) -> None: - loop = TriggerLoop(client, rules=[summed]) - first = files.measure(1, 2.0, role='sample') - (one,) = loop.step() - second = files.measure(2, 3.0, role='sample') - third = files.measure(3, 4.0, role='sample') - (two,) = loop.step() # one call: its record settles both new files - - assert one.request.params['runs'] == [first] - assert two.request.params['runs'] == [first, second, third] - assert client.output(two, 'value') == 9.0 - assert loop.step() == [] - - -def test_a_loop_started_on_a_history_has_handled_the_files_in_rows( - client: Client, files: FakeFiles, summed: Rule -) -> None: - files.measure(1, 2.0, role='sample') - files.measure(2, 3.0, role='sample') - client.wait(TriggerLoop(client, rules=[summed]).step()) - - assert TriggerLoop(client, rules=[summed]).step() == [] - - -def test_an_idle_step_reads_no_records( - client: Client, files: FakeFiles, loading: Rule -) -> None: - counting = CountingClient(client) - loop = TriggerLoop(counting, rules=[loading]) # type: ignore[arg-type] - files.measure(1, 2.0, role='sample') - client.wait(loop.step()) - loop.step() # reads the record the first step submitted - counting.reads.clear() - - assert loop.step() == [] - assert counting.reads['records'] == 0 - - -def test_a_step_after_a_new_record_reads_only_that_record( - client: Client, files: FakeFiles, loading: Rule -) -> None: - counting = CountingClient(client) - loop = TriggerLoop(counting, rules=[loading]) # type: ignore[arg-type] - files.measure(1, 2.0, role='sample') - client.wait(loop.step()) - loop.step() - measured = files.measure(2, 3.0) - template = Template(LOAD, blanks=('sample',)) - other = client.compute(template.fill(params={'sample': measured}), label='other') - counting.reads.clear() - - assert loop.step() == [] - assert counting.reads['records'] == 1 - assert client.output(other, 'value') == 3.0 - - -def test_a_loop_started_on_a_history_does_not_submit_the_handled_files_again( - client: Client, files: FakeFiles, loading: Rule -) -> None: - files.measure(1, 2.0, role='sample') - files.measure(2, 3.0, role='sample') - client.wait(TriggerLoop(client, rules=[loading]).step()) - counting = CountingClient(client) - - late = TriggerLoop(counting, rules=[loading]) # type: ignore[arg-type] - - assert late.step() == [] - assert counting.reads['records'] == 2 - assert len(client.records(label='loaded')) == 2 - - -def test_two_rules_with_one_label_in_one_step_submit_each_file_once( - client: Client, files: FakeFiles, loading: Rule -) -> None: - again = Rule( - 'again', - loading.submit, - selector=loading.selector, - label=loading.label, - at=loading.at, - ) - loop = TriggerLoop(client, rules=[loading, again]) - files.measure(1, 2.0, role='sample') - - assert len(loop.step()) == 1 - assert loop.status(again).reason == 'no new file' - assert loop.step() == [] - assert len(client.records(label='loaded')) == 1 - - -def test_a_file_at_a_list_parameter_is_refused_by_the_workspace( - client: Client, files: FakeFiles -) -> None: - sample = files.measure(1, 2.0, role='sample') - template = Template(SUM_RUNS, blanks=('runs',)) - - summed = template.fill(params={'runs': sample}) - - with pytest.raises( - SubmitError, - match=r'^runs: uuid:run-1 is a reference to a whole list; a reference stands', - ): - client.submit(summed) - - -def test_a_batch_with_a_file_that_cannot_be_read_fails_only_its_record( - client: Client, files: FakeFiles -) -> None: - runs = [files.measure(n, float(n)) for n in (1, 2, 3)] - files.corrupt(runs[1]) - - template = Template(LOAD, blanks=('sample',)) - requests = { - str(client.files.metadata(run)['run']): template.fill(params={'sample': run}) - for run in runs - } - scan = client.compute(requests, label='scan') - - assert client.wait(scan) == {'1': 'completed', '2': 'failed', '3': 'completed'} diff --git a/packages/essapps/tests/stories/e_automatic_test.py b/packages/essapps/tests/stories/e_automatic_test.py index e648c23b..08dd48f5 100644 --- a/packages/essapps/tests/stories/e_automatic_test.py +++ b/packages/essapps/tests/stories/e_automatic_test.py @@ -4,11 +4,13 @@ # ruff: noqa: F821 -from collections.abc import Callable +from collections.abc import Callable, Sequence import pytest -from ess.apps import Client, Record, Rule, Selector, Template, TriggerLoop +from ess.apps import Client +from ess.apps.batch import Batch, FileInfo, Row +from ess.apps.batch.auto import Automatic from ess.apps.testing import FakeFiles from ess.spec import FileRef @@ -17,48 +19,51 @@ NO_PUBLISH = 'publication is not implemented' -def iofq_rule(template: Template, selector: Selector) -> Rule: - def submit(client: Client, run: FileRef) -> Record: - member = str(client.files.metadata(run)['run']) - request = template.fill(params={'sample': run}) - return client.submit(request, label='iofq', member=member, persist=True) +def per_run(files: Sequence[FileInfo]) -> list[Row]: + """A row per file the batch's selector matches: every raw file.""" + return [Row(str(f.metadata['run']), {'sample': f.ref}) for f in files] - return Rule('auto-iofq', submit, selector=selector, label='iofq', at='sample') + +def iofq_batch(client: Client) -> Batch: + return Batch( + client, + toys.IOFQ, + columns=('sample',), + label='iofq', + source=per_run, + persist=True, + ) def test_e1_series_grows_reduction_follows( client: Client, connect: Callable[..., Client], measure: toys.Measure ) -> None: reference = measure(1, [1.0, 1.0], role='reference') - template = Template(toys.STITCH, params={'reference': reference}, blanks=('runs',)) - - def stitch(client: Client, run: FileRef) -> Record: - """Stitch all angles of the run's sample so far, in run order.""" - sample = client.files.metadata(run)['sample'] - angles = client.files.list(Selector(role='sample', sample=sample)) - angles.sort(key=lambda ref: client.files.metadata(ref)['run']) - return client.submit( - template.fill(params={'runs': angles}), - label='reflectivity', - member=sample, - persist=True, - ) - - rule = Rule( - 'reflectivity', - stitch, - selector=Selector(role='sample'), + + def angles(files: Sequence[FileInfo]) -> list[Row]: + """A row per sample: all its angles so far, in run order.""" + runs: dict[str, list[FileRef]] = {} + for f in sorted(files, key=lambda f: f.metadata['run']): + if f.metadata.get('role') == 'sample': + runs.setdefault(f.metadata['sample'], []).append(f.ref) + return [Row(sample, {'runs': refs}) for sample, refs in runs.items()] + + batch = Batch( + client, + toys.STITCH, + columns=('runs',), label='reflectivity', - at='runs', + source=angles, + persist=True, ) - loop = TriggerLoop(client, rules=[rule]) + auto = Automatic(batch, at='runs', settings=lambda: {'reference': reference}) r2 = measure(2, [1.0, 2.0], role='sample', sample='si') - loop.step() + client.wait(auto.step()) r4 = measure(4, [4.0, 8.0], role='sample', sample='si') r3 = measure(3, [1.0, 1.0], role='sample', sample='si') measure(2, [1.0, 2.0], role='sample', sample='si') - loop.step() + auto.step() user = connect() # any client of the proposal latest = user.latest('reflectivity', member='si') @@ -78,50 +83,42 @@ def stitch(client: Client, run: FileRef) -> Record: def test_e2_automatic_reduction_goes_quiet( upgrade: Callable[..., Client], measure: toys.Measure ) -> None: - rule = iofq_rule( - Template(toys.IOFQ, params={'normalization': None}, blanks=('sample',)), - Selector(role='sample'), - ) upgraded = upgrade(specs=[toys.IOFQ_V2]) # a workspace without version 1 - loop = TriggerLoop(upgraded, rules=[rule]) - run = measure(1, [1.0, 1.0], role='sample') + auto = Automatic( + iofq_batch(upgraded), at='sample', settings=lambda: {'normalization': None} + ) + measure(1, [1.0, 1.0], role='sample') - assert loop.step() == [] - assert loop.status(rule).reason == f'skipped {run.file}: unknown spec sans-iofq/v1' + assert auto.step() == {} + assert auto.refused == {'1': 'unknown spec sans-iofq/v1'} @pytest.mark.xfail(reason=NO_PUBLISH) def test_e3_reduction_of_our_own_output( client: Client, measure: toys.Measure, files: FakeFiles ) -> None: - rule = iofq_rule( - Template(toys.IOFQ, params={'normalization': None}, blanks=('sample',)), - Selector(), + auto = Automatic( + iofq_batch(client), at='sample', settings=lambda: {'normalization': None} ) - loop = TriggerLoop(client, rules=[rule]) measure(1, [1.0, 2.0, 3.0, 4.0]) - (reduced,) = loop.step() + (reduced,) = auto.step().values() pid = client.publish(reduced.ref('iofq'), 'scicat') files.add_published(scicat.entries[pid]) - assert loop.step() == [] + assert auto.step() == {} def test_e4_template_improved_during_a_beamtime( client: Client, connect: Callable[..., Client], measure: toys.Measure ) -> None: - samples = Selector(role='sample') - template = Template(toys.IOFQ, params={'normalization': None}, blanks=('sample',)) + settings = {'normalization': None} + auto = Automatic(iofq_batch(client), at='sample', settings=lambda: settings) first = measure(1, [1.0, 2.0, 3.0, 4.0], role='sample') - (before,) = TriggerLoop(client, rules=[iofq_rule(template, samples)]).step() + (before,) = auto.step().values() - improved = Template( - toys.IOFQ, - params={'reduction': toys.Reduction(threshold=1.5), 'normalization': None}, - blanks=('sample',), - ) + settings = {'reduction': toys.Reduction(threshold=1.5), 'normalization': None} second = measure(2, [1.0, 2.0, 3.0, 4.0], role='sample') - (after,) = TriggerLoop(client, rules=[iofq_rule(improved, samples)]).step() + (after,) = auto.step().values() user = connect() # any client of the proposal assert after.request.files() == [second] From 2f8f67f077b38bb9ad680d9471829542602e3ec5 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 13 Nov 2026 11:45:02 +0000 Subject: [PATCH 3/7] docs: automatic reduction is a batch reduced as its files arrive automatic-reduction.md describes the loop over batches: which rows run and why, what the loop keeps, refusals, chains across batches, the cost of a step, and what is open (new files since a position, a group per file, a range in Selector, old rows reduced again for a new spec version). The batch guide gains a section on the automatic use, with the table of which rows run for "reduce remaining", `essapps-batch reduce --remaining`, and the loop, and says that a source may read another batch's state. A row's result no longer depends on the submitter. Stories E, the README, deployments, the ADRs, the prior-art notes, the client guide, and the tours name the batch where they named rules and the trigger loop. Co-Authored-By: Claude Opus 5.5 --- docs/developer/README.md | 21 ++- ...-a-value-lives-while-something-keeps-it.md | 4 +- .../adr/0004-history-is-append-only-lists.md | 8 +- .../0007-packages-split-by-dependencies.md | 2 +- ...-identified-by-its-catalogue-identifier.md | 4 +- docs/developer/automatic-reduction.md | 175 ++++++++++++------ docs/developer/deployments.md | 4 +- docs/developer/prior-art/aiida.md | 4 +- docs/developer/prior-art/ewoks.md | 2 +- docs/developer/prior-art/git.md | 14 +- docs/developer/prior-art/mantid.md | 10 +- docs/developer/prior-art/snakemake.md | 12 +- docs/developer/roles/client-authors.ipynb | 4 +- docs/developer/system-stories.md | 2 +- docs/developer/three-ways-to-run-a-spec.html | 8 +- docs/developer/tours/layers.html | 55 +++--- docs/developer/user-stories.md | 86 +++++---- packages/essapps/docs/batch.md | 53 +++++- .../essapps/src/ess/apps/batch/corrections.py | 4 +- packages/essapps/src/ess/apps/protocol.py | 4 +- 20 files changed, 292 insertions(+), 184 deletions(-) diff --git a/docs/developer/README.md b/docs/developer/README.md index d755c52c..5c896163 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -60,7 +60,7 @@ What the framework adds to the plain loop: ## Components ``` - controller: a notebook, an application, the trigger loop decides what to run + controller: a notebook, an application, automatic reduction decides what to run │ client: submit, push, output, release, ... ▼ workspace ─── file source: files and their metadata @@ -128,7 +128,7 @@ An output of a record of the bound spec, one the workspace runs with the same th An output of a record of another spec, such as an older version or the same version with changed models, is read as stored, an array as it is and anything else as a `DataGroup`, and a new request may not reference it ([ADR 0009](adr/0009-a-record-of-another-spec-stays-readable.md)). **Controller.** The workspace does not decide what to run. -A *controller* is client code that decides over time what to submit, push, and release, such as the cells of a notebook, the callbacks of an application's screen, or the trigger loop of automatic reduction. +A *controller* is client code that decides over time what to submit, push, and release, such as the cells of a notebook, the callbacks of an application's screen, or the loop of automatic reduction. A controller is never part of the workspace, and calls it only through a client: ```python @@ -265,19 +265,20 @@ total = client.freeze(volume) # the final sum, as ## Automatic reduction -A *rule* calls an application's function for each new file as it arrives; the function pairs runs as the technique needs and submits what the file needs ([automatic-reduction.md](automatic-reduction.md)): +Automatic reduction is a batch of `ess.apps.batch` reduced as its files arrive ([automatic-reduction.md](automatic-reduction.md)). +The application's source pairs runs as the technique needs, and a row runs once for each new file at its trigger column: ```python -def reduce_transmission(client: Client, run: FileRef) -> Record: - return client.submit(TRANSMISSION, params={'run': run, 'empty_beam': file(run=60336)}, - label='transmission', persist=True) +def transmission_runs(files: Sequence[FileInfo]) -> list[Row]: + return [Row(str(f.metadata['run']), {'run': f.ref}) + for f in files if f.metadata.get('role') == 'transmission'] -rule = Rule('transmission', reduce_transmission, selector=Selector(role='transmission'), - label='transmission', at='run') -TriggerLoop(client, rules=[rule]).run() +batch = Batch(client, TRANSMISSION, columns=('run',), label='transmission', + source=transmission_runs, persist=True) +run([Automatic(batch, at='run', settings=lambda: {'empty_beam': file(run=60336)})]) ``` -The loop reads which files a rule has handled from the changes, so a step reads only the records that changed since the last one. +The loop reads from the changes which files the records of each row hold, so a restarted loop reduces no row again. ## Provenance and publication diff --git a/docs/developer/adr/0002-a-value-lives-while-something-keeps-it.md b/docs/developer/adr/0002-a-value-lives-while-something-keeps-it.md index 76028411..6b648a01 100644 --- a/docs/developer/adr/0002-a-value-lives-while-something-keeps-it.md +++ b/docs/developer/adr/0002-a-value-lives-while-something-keeps-it.md @@ -10,7 +10,7 @@ History holds records, not values ([ADR 0004](0004-history-is-append-only-lists. Values take memory, so something must decide when they go, and which are written. - **Two deployments.** Interactive work runs in the user's process (`local()`), on a VISA desktop or a laptop, and ending the process frees everything. The service runs batch and automatic reduction, and work spread over a cluster, for hours or days. The service must bound leaks and peak memory by design, not rely on users freeing memory ([users](../../requirements/users.md)). -- **Unattended work.** The trigger loop submits through one client and never releases anything. A batch client closes, and users read its outputs the next morning. +- **Unattended work.** The automatic reduction submits through one client and never releases anything. A batch client closes, and users read its outputs the next morning. - **Several connections.** A user works on one proposal through several clients at once, such as a notebook next to an app, or a web page and the same page reloaded. - **Writing costs.** A batch of 500 would write story S3's masked counts, an output meant for inspecting a reduction, once per record, 500 times. A volume that a web UI only shows cuts of (B4) is hundreds of GB. The framework cannot tell intermediate from final outputs. @@ -43,7 +43,7 @@ So whether it is refused depends only on what the workspace keeps, not on how fa **Nothing is written unless persisted**, since the framework cannot tell which outputs are worth writing (Context). A client chooses per output, with `persist=` at submission or at `client.freeze`. -The store is then the only keeper, as batch reduction and the trigger loop need. +The store is then the only keeper, as batch reduction and automatic reduction need. A failed write fails the record, so a controller sees it like any other failed record. The store keys a value by record ID and output name, so a record holds no file name and a write changes no record. A persisted output stays readable after an upgrade removes its record's spec version, since the record stays in history, and is read as plain scipp data ([ADR 0009](0009-a-record-of-another-spec-stays-readable.md)). diff --git a/docs/developer/adr/0004-history-is-append-only-lists.md b/docs/developer/adr/0004-history-is-append-only-lists.md index db640972..c8b5105e 100644 --- a/docs/developer/adr/0004-history-is-append-only-lists.md +++ b/docs/developer/adr/0004-history-is-append-only-lists.md @@ -13,7 +13,7 @@ So history only grows until it is dropped. The stories need four things from history: -- Records outlive the process of the workspace (B6, H2, H3). The trigger loop knows which files a rule has handled only from the records under the rule's label. +- Records outlive the process of the workspace (B6, H2, H3). The automatic reduction knows which rows it has reduced only from the records under the batch's label. - A submission is stored whole or not at all. - The order of the records under a label, and of the pushes into an accumulator, is kept. - Persisted values are found again. @@ -52,15 +52,15 @@ History is five lists, each only appended to: - **History defined as one event log, with records and labels as views**, indexes rebuilt from the log. The lists hold the same history. The log adds one order across all lists, and outside storage only the change feed needs an order, of the records alone. It would tie a hosted workspace to a log. - **A history held without a person kept as long as the proposal.** That may be forever, a second record of what ran next to SciCat. -- **Each record dropped by age, keeping the older records that a kept record reads.** The trigger loop would reduce again every file whose records were dropped. It would need a memory of its own, such as a set of handled files per rule. -- **A proposal's history dropped a fixed time after its last record.** A trigger loop that sees no new file for that long loses its rule's records and then reduces every file again. A client still in use would lose the records it reads. +- **Each record dropped by age, keeping the older records that a kept record reads.** The automatic reduction would reduce again every row whose records were dropped. It would need a memory of its own, such as a set of reduced files per batch. +- **A proposal's history dropped a fixed time after its last record.** An automatic reduction that sees no new file for that long loses its records and then reduces every row again. A client still in use would lose the records it reads. - **The record of a freeze stored with every row, in storage that shares repeated rows.** Sharing saves bytes, but every layer above the store still handles each row twice, once in its push and once in the record. - **Output values kept with their records.** This stores one output per state, and makes a catalogue of outputs next to SciCat, which is a non-goal. ## Consequences - Dropping a proposal's history leaves no dangling reference. No other proposal reads its records (system story G5), and an idle proposal gets no call and has no pending record. -- A running trigger loop calls its client while it waits for changes, so its proposal is not idle. A loop started for a proposal whose history was dropped reduces its files again. +- A running automatic reduction calls its client at each step, so its proposal is not idle. A loop started for a proposal whose history was dropped reduces its rows again. - A proposal that is never idle, such as one whose automatic reduction runs all year, keeps all its history. - A client whose process ended makes no call, so it does not keep its proposal from being idle. - A workspace that restarts does not know when its proposal was last called, so it counts idle time from its start. diff --git a/docs/developer/adr/0007-packages-split-by-dependencies.md b/docs/developer/adr/0007-packages-split-by-dependencies.md index b954ecf1..d25d41f9 100644 --- a/docs/developer/adr/0007-packages-split-by-dependencies.md +++ b/docs/developer/adr/0007-packages-split-by-dependencies.md @@ -77,7 +77,7 @@ essspec <- essapps <- frontends (over HTTP) - **Names follow the ess monorepo**: distribution `essX`, package `ess.X`, one level under `ess`. A second distribution cannot install into a regular package (one with an `__init__.py`) such as `ess.apps`, and a namespace package (one without, which may span distributions) could not export `local` and the rest. So `ess.apps.panel` exists only as part of the `essapps` distribution. - **`essspec` (`ess.spec`) is everything a workflow author writes against.** The vocabulary (`WorkflowSpec`, references, the functions that read field types, parameter models), the binding protocol (`Binding`, `StageBinding`, `AccumulatorBinding`, `Driver`), and the bindings `DriverBinding` and `function_binding` need only pydantic. A client imports `ess.spec` for the specs, and the bindings cost it nothing more. `ess.spec.conversions` needs scipp (extra `[scipp]`), and `ess.spec.pipeline` with `PipelineBinding` needs sciline (extra `[sciline]`). It is developed in this repository while it changes, and nothing outside this repository depends on a version of it. Once a second consumer, such as esslivedata or essnmx, is to adopt it, it moves to scipp/ess as a distribution of its own and is published from there. -- **`essapps` (`ess.apps`) is what notebooks and apps use**: records, the client, the workspace and its log, files, the controllers (rules, the trigger loop), the batch toolbox `ess.apps.batch`, and fakes for tests. The core depends on essspec and pydantic, not on essreduce or sciline. The subpackages behind extras change in step with the core, so they are released with it. Importing one without its extra raises an `ImportError` that names the extra. A local workspace serves a web frontend on VISA or a laptop through `ess.apps.http.server`, as the service does. The trigger loop is in the core, and the function of a rule in the application that knows the technique; `ess.apps.service` runs the loop as a daemon from a configuration file. +- **`essapps` (`ess.apps`) is what notebooks and apps use**: records, the client, the workspace and its log, files, the batch toolbox `ess.apps.batch` with its loop of automatic reduction, and fakes for tests. The core depends on essspec and pydantic, not on essreduce or sciline. The subpackages behind extras change in step with the core, so they are released with it. Importing one without its extra raises an `ImportError` that names the extra. A local workspace serves a web frontend on VISA or a laptop through `ess.apps.http.server`, as the service does. The loop of automatic reduction is in `ess.apps.batch`, and the source of rows in the application that knows the technique; `ess.apps.service` runs the loop as a daemon from a configuration file. - **`essinstruments` (`ess.instruments`) holds one subpackage per instrument, and is released on its own schedule.** A fix to LoKI's specs ships without whatever changed in the core since its last release, and a core release ships without unreleased instrument changes. It depends on essspec, not on essapps. Each instrument has two modules: `specs.py` with the specs, and `bindings.py` with their bindings, which need the instrument's workflow package, behind an extra such as `[loki]`, so that an install for LoKI does not install the workflow packages of every other instrument. The two are separate because a spec may move into its workflow package without its binding, and a type checker checks the binding against the spec. `ess.instruments.loki` holds the specs and bindings of LoKI's reductions; `ess.loki`, from esssans, holds the workflow they bind to. A spec can move into its workflow package only after essspec has moved to scipp/ess, and only once its binding is a plain `PipelineBinding`, with no facility defaults or adapters. Deployment configuration, such as which automatic-reduction rules run and data paths, is not here; it belongs to the service's configuration. - **A Python client that imports spec modules installs the instrument's workflow package.** A local workspace, a notebook, and a Panel app on VISA or a laptop run the bindings, so they have it anyway. A notebook against the service wants scipp and plotting for the outputs, so it runs in an ESS environment that has the workflow packages too. A client without workflow packages, such as a web frontend or the command-line tool with `--url`, reads `SerializedWorkflowSpec`: the JSON form of a spec, which names each data type as a string. A spec module imports scipp for the types of data: an array is a field of a group, typed by its scipp class, such as `intensity: sc.DataArray` of the group `IofQ`, so that a type checker checks a binding against the spec, and the workspace checks by the group's class that an output fits a field typed `Referenced[IofQ]`. - **`essapps` finds bindings through the entry-point group `ess.apps.workflows`.** An entry point is a name that a distribution registers under a group in its `pyproject.toml` (see the example below); `importlib.metadata.entry_points(group=...)` lists the entries of all installed distributions. Each entry names a function that returns a list of bindings, each of which names its spec. `local()` without `bind=`, the command-line tool, and the service read the group, so a distribution registers its workflows without importing the framework. Listing the specs this way imports the bindings; a client of the service gets the list from the service instead. An entry registers whether or not its extra is installed: an entry whose workflow package is missing is skipped, and `essapps specs` lists its specs as unavailable, naming the extra; any other import error fails the load. Two entries that register the same spec name and version fail the load, naming both entries, since picking one would leave records ambiguous about what ran. This is decided and not implemented. diff --git a/docs/developer/adr/0010-a-file-is-identified-by-its-catalogue-identifier.md b/docs/developer/adr/0010-a-file-is-identified-by-its-catalogue-identifier.md index 16de0328..4d9f6113 100644 --- a/docs/developer/adr/0010-a-file-is-identified-by-its-catalogue-identifier.md +++ b/docs/developer/adr/0010-a-file-is-identified-by-its-catalogue-identifier.md @@ -9,7 +9,7 @@ A person names a file as `file(run=60339)`, and the file source resolves the name to the file's *identity*, which the record keeps and provenance lists. With paths as identities: -- A moved folder changes every identity, so a trigger loop reduces its runs again. +- A moved folder changes every identity, so the automatic reduction reduces its runs again. - The same run seen through two file sources has two identities. The catalogue is SciCat. @@ -40,7 +40,7 @@ A person still writes `file(run=60339)`. - A moved folder, and a second file source, give a raw file the same identity. - Provenance lists raw files as the catalogue names them. - A file source reads the identifier from a raw file, or gets it from the catalogue. -- A file first listed without its catalogue identifier and later with it changes identity: a trigger loop reduces it again, and the records made before name an identity the file source no longer knows. +- A file first listed without its catalogue identifier and later with it changes identity: the automatic reduction reduces it again, and the records made before name an identity the file source no longer knows. - Identities are in the log, so changing them changes the log's format ([ADR 0004](0004-history-is-append-only-lists.md#decision)). ## Open diff --git a/docs/developer/automatic-reduction.md b/docs/developer/automatic-reduction.md index 3a3a86a3..67806b23 100644 --- a/docs/developer/automatic-reduction.md +++ b/docs/developer/automatic-reduction.md @@ -1,84 +1,145 @@ # Automatic reduction -This sub-design builds on the core of [README.md](README.md). -From the core it needs only that records show their label and member, and `client.files`. -Its stories are E1 to E4 in [user-stories.md](user-stories.md). +This sub-design builds on the core of [README.md](README.md) and on the batch toolbox, `ess.apps.batch` ([its guide](../../packages/essapps/docs/batch.md)). +Its stories are E1 to E4 in [user-stories.md](user-stories.md), and [deployments.md](deployments.md#automatic-reduction) places it. -## Rules +## A batch reduced as its files arrive -The framework knows no technique: which runs belong together, and which requests one new file needs, is for the application to say. -A rule says it with a function: +The framework knows no technique: which runs make one reduction is the application's to say. +It says so once, as the *source* of a batch, a function from files to rows. +A person reducing a batch and the automatic reduction use the same source. + +The batch is a chain of tables, of which only the files grow on their own: ```python -def reduce_sample(client: Client, sample: FileRef) -> Record: - fields = client.files.metadata(sample) - found = client.files.list(Selector(run_type='trans', run_label=fields['run_label'])) - if not found: - raise LookupError('no transmission run yet') # try again at the next step - member = str(fields['run']) - centre = next((r for r in reversed(client.records(label='centre')) - if r.member == member and client.status(r) not in (Status.FAILED, Status.CANCELLED)), - None) # from a call that stopped half-way - if centre is None: - centre = client.submit(BEAM_CENTRE, params={'sample': sample, ...}, - label='centre', member=member, persist=True) - return client.submit(IOFQ, params={'sample': sample, 'transmission': found[-1], - 'beam_centre': centre.ref('centre'), ...}, - label='iofq', member=member, persist=True) - -rule = Rule('iofq', reduce_sample, selector=Selector(run_type='sans'), label='iofq', at='sample') +files = client.files.list(selector) # with their metadata +rows = source(files) # Batch.scan: plain data in and out +results = records under the batch's label, by member # the history, read as a table +state = rows joined with results on key == member # Batch.state +todo = [key for key in state if runs(key)] # the predicate of the next section +submit(todo) # appends to the history, so results change ``` -`run_type` and `run_label` stand in for what ESS files will say about a run, which is not settled ([data](../requirements/data.md)). A selector takes any metadata field, so only the rule's code and the file source name them. +The automatic reduction is this chain stepped by a loop, `ess.apps.batch.auto.Automatic`: -The function returns the record that *settles* the file: one under the rule's label that holds the file at the parameter `at`, or in a row of it if `at` is a list. -The loop checks this and raises `TypeError` otherwise, since it would call the function for that file again at every step. +```python +batch = Batch(client, IOFQ, columns=('sample', 'sample_transmission'), label='auto/iofq', + source=pair_by_sample, staged=True, persist=True) +auto = Automatic(batch, at='sample', settings=lambda: load_settings('iofq.yaml')) +run([auto]) # steps forever: scan, then reduce the rows with a new file at `sample` +``` -The function persists what it submits. The loop never releases, so the workspace should keep nothing for it. A reference must also stay readable once the loop's process has restarted. +`settings` is called at each step, so a setting changed during a beamtime applies to the rows not reduced yet. +The loop is a controller: it runs in a process of its own or in a notebook, never in the workspace. -A function may run again for a file: when a step stopped before the settling record, or when that record was refused. So it submits the settling record last, and reuses what an earlier call submitted, as with the centre above. -It does not reuse a failed or cancelled record, since a reference to one is refused, and the file would never settle. With a new centre, the reduction is accepted while the centre is pending, then fails with it, and that failed record settles the file. +## Which rows run -The function does not wait for a record or read an output. The loop calls one function at a time, so a wait holds up every rule, and a request can reference a pending record instead. +A person and the loop differ in one predicate, the `runs` above ([batch.md](../../packages/essapps/docs/batch.md#automatic-reduction) tables both). +For the loop, a row runs if its cell at the *trigger column* `at` holds a file that no record under the batch's label, with the row's key as its member, holds at `at`: -Rules are frozen dataclasses. The core keeps no store of them, and records do not name them: a record's request says everything that determines its result. +```python +def runs(row, held): # held: the files at `at` of the records of the row's key + return bool(files_at(row.values, at) - held) and ready(row) +``` -## The trigger loop +`ready` is the batch's: the row is complete, and its result is not running. +A file at `at` is the cell's value, or an element of its list, as a series of angles gives it; a file deeper inside the value does not count. -The trigger loop is the controller for rules. It runs in a server of its own or in a notebook, never in the workspace. -It finds new files through `client.files`, and calls each rule's function for each of them. +| What happened to a row | Runs | +|---|---| +| no record holds its file at `at` | yes | +| its record failed or was cancelled | no | +| a setting changed, or a cell at another column | no | +| a new file joined its list at `at` | yes | -```python -loop = TriggerLoop(client, rules=[rule]) -loop.step() # calls the rules for what arrived since the last step, returns the settling records -loop.status(rule).reason # why the rule submitted nothing or skipped a file -loop.run() # steps forever -``` +Each line answers a story or a failure mode: + +- A failed record counts, so that a file that fails is not retried at every step. Running it again is a person's decision (D2). +- A changed setting does not reduce old rows again (E4). Their records keep the settings they ran with, and a person who wants them reduced again makes a batch with its own label. +- A later partner, such as a transmission run that the source pairs with rows made before, changes a cell at another column, and does not reduce those rows again. +- A series grows (E1): a new angle in a row's list reduces the row again with every angle so far, and an angle whose file arrives again keeps its identity and is not new. + +The records are read whatever their spec version or submitter. So a new version of the spec, or a restart under another name, reduces no old row again. +A label is what a consumer group is to a stream: the records under it say which files are consumed, and nothing else does. +A new label starts from the first file, and a person's batch under its own label does not stop the loop from reducing a row; a record that another client submits under the loop's label, with a row's key as its member, does. -The loop keeps nothing that the records do not hold: it reads them from `client.changes`, from the start when it starts, and keeps the files it has seen. A rule has handled a file when a record under the rule's label holds it at the parameter `at`, or in a row of it, so a restarted or replaced loop does not reduce a file again, and a file that arrives again keeps its identity and is not reduced twice. -A file that the record stores at another field, such as an empty-beam run, does not count. +A row that is running waits: the loop submits it again, with its new file, once its record has finished. + +## What the loop keeps + +Which files the records of each key hold is read from the proposal's changes (`client.changes`), from the start when the loop starts. +So a restarted loop reduces no row again, and keeps nothing on disk of its own. This holds while the proposal's history is kept. A running loop keeps it, since its calls keep the proposal from being idle ([`ess.apps.protocol`](../../packages/essapps/src/ess/apps/protocol.py)). -A record can settle more files than the one it was submitted for. A function that stitches all angles of a sample so far settles every new angle of that sample in one call: +Besides, the loop keeps the rows and the settings of its last step, which spare it work only. +A step looks at the rows whose cells changed, those with a record that changed, and, if the settings changed, those that were refused: + +| A row waits because | It is looked at again when | +|---|---| +| a cell is empty, such as a missing transmission run | its cells change | +| its result is running | its record changes | +| its request was refused, such as an invalid value | its cells or the settings change | + +The batch works out the state of those rows only (`Batch.state(settings, keys)`). + +## Refusals + +`Batch.reduce` is all or none, as a person needs it: one invalid row refuses the call, and the error names it. +The loop instead submits the rows, drops those the `SubmitError` names (`FieldError.entry`), and submits the rest, so a refused row does not hold back the others. +A reason that names no row is about the settings, such as a spec the workspace no longer runs (E2), and refuses every row. +`Automatic.refused` holds the reason of each row that has a new file and did not run, by key, so that an application can show why. Why a row waits is also its `RowState.problems`. + +## Chains + +A row may reference the output of a row of another batch, as a sample reads the transmission computed from its transmission run. +The second batch's source reads the first batch's state, and the loop steps the first batch before the second: ```python -def stitch(client: Client, run: FileRef) -> Record: - sample = client.files.metadata(run)['sample'] - angles = client.files.list(Selector(role='sample', sample=sample)) - angles.sort(key=lambda ref: client.files.metadata(ref)['run']) - return client.submit(template.fill(params={'runs': angles}), - label='reflectivity', member=sample, persist=True) +transmissions = Batch(client, TRANSMISSION, columns=('run',), label='auto/transmission', ...) + +def samples(files): + done = {transmissions.rows[k].info['sample']: s.record + for k, s in transmissions.state().items() if s.record} + return [Row(key, {'sample': run, 'transmission': done[name].ref('transmission')}) ...] + +run([Automatic(transmissions, at='run', settings=...), + Automatic(Batch(client, IOFQ, ..., source=samples), at='sample', settings=...)]) ``` -The label belongs to the rule: any record under it counts, whoever submitted it and whether or not it failed. -A record that another client of the workspace submits under a rule's label stops the rule from reducing the files it names, so manual work uses labels of its own. -A failed record counts so that the loop does not retry a failed file, which could fail again at every step; running it again is the user's decision (D2). +A sample that arrives before its transmission run waits with an empty cell. +In the step in which the transmission is submitted, the sample's row references the pending record, so both run in one step. +If the transmission's record has failed by the time the sample's row is made, the row waits: a cell reads a failed record. +If it fails after, the sample's record fails with it and holds the sample's file, so the row does not run again. +A source is recomputed at every step, so a chain survives a restart, which a reference set by hand in a cell (an edit) does not. + +## Cost + +A step lists every file, runs the source over all of them, and compares the rows with those of the last step. +That is linear in the number of files, about 4 µs per file and 3 µs more to list it in the fake file source. +The state and the submission cost only the rows looked at. +A loop that starts looks at every row once. -A file whose function raises `LookupError` or `ValueError`, or whose request is refused with `SubmitError`, is skipped: the rule goes on with its other new files, and its status names each skipped file and why. -No record names the skipped file, so the next step tries it again, for example once its transmission run has arrived. -Any other exception stops the loop, as does `RecordFailed` from a function that reads an output. +| Runs reduced | 3 000 | 30 000 | 100 000 | +|---|---|---|---| +| a step with one new file, best of ten | 12 ms | 0.19 s | 0.74 s | +| of which listing the files | 5 ms | 75 ms | 0.33 s | +| mean of ten | 25 ms | 0.39 s | 1.4 s | + +Measured with a toy workflow, the fake file source, and a history in memory. +The mean is about twice the best because of Python's garbage collector: each step makes a row for each file, and the collector then walks every object of the process, the records among them. ## Open -- Adding, replacing, and listing rules in a running trigger loop. -- A rule that pushes into an accumulator, for a sum that grows with each file (D7 as a rule). The accumulator would be the only thing the workspace keeps for the loop, which would freeze it with `persist=` once the sum is complete. +- **New files since a position.** `FileSource.list` returns every file, so each step lists the whole proposal. A catalogue cannot be listed whole at each step; the file source needs a call that returns what is new since a position, as `client.changes` does for records. +- **A step below linear.** The source runs over all files because nothing says which rows a new file can touch. An optional group per file would say it: a step would then make the rows of the groups that got a file, from those files and the files shared by all groups. + + ```python + Batch(..., source=pair_by_sample, + group=lambda f: None if f.metadata['role'] == 'can' else f.metadata['sample']) # None: shared + ``` + + A source that pairs by time, such as "the can measured before", has no such group and stays linear. +- **Starting a new label at a given run.** A new label consumes from the first file. Starting it at a run needs a range in `Selector`, which matches equal values only. +- **Old rows reduced again for a new spec version.** One more term of the predicate, from the spec of each row's result (ADR 0009): `runs(row, held) or result.spec != spec`. +- **A loop that pushes into an accumulator**, for a sum that grows with each file (D7). The accumulator would be the only thing the workspace keeps for the loop, which would freeze it with `persist=` once the sum is complete. +- **The application around the loop**: one process per instrument, a workspace per proposal, the configuration files, and a page that shows the batches. The page and the loop would use one `Batch` from two threads, and `Batch` has no lock. diff --git a/docs/developer/deployments.md b/docs/developer/deployments.md index a07e8180..41d16b86 100644 --- a/docs/developer/deployments.md +++ b/docs/developer/deployments.md @@ -59,7 +59,7 @@ The web page (`ess.apps.batch.web`) is experimental and runs as a demo. ``` DMSC server, one process per instrument ┌──────────────────────────────────────────────────────────────────┐ - │ loop: for each new file, submit what the file needs │ + │ loop: scan each batch, reduce the rows with a new file │ │ page to configure the reduction │ │ configuration: per instrument, adjusted per proposal │ │ a Client and a Workspace per proposal: history and store on disk │ @@ -74,7 +74,7 @@ The process holds a workspace for each proposal that has new files. It runs with no page open, and users find its results through SciCat, not by connecting to it. For the first release, the same process may run on a VISA machine ([requirements](../requirements/users.md)). A scientist who needs a variant of the automatic reduction runs a batch. -Of this deployment, only the loop exists (`TriggerLoop`, [automatic-reduction.md](automatic-reduction.md)). +Of this deployment, only the loop exists (`ess.apps.batch.auto`, [automatic-reduction.md](automatic-reduction.md)). ## One proposal, several histories diff --git a/docs/developer/prior-art/aiida.md b/docs/developer/prior-art/aiida.md index cc83f707..5e0d0dc2 100644 --- a/docs/developer/prior-art/aiida.md +++ b/docs/developer/prior-art/aiida.md @@ -59,14 +59,14 @@ AiiDA's export default is what `client.provenance` and `client.publish` do: ever `BaseRestartWorkChain` keys handlers on exit codes and retries up to `max_iterations` (default 5), then aborts or pauses for a person ([workchains_restart.rst L76-115][restart]). Our records end `completed`, `failed`, or `cancelled`, and `client.failure` returns a message ([README, Requests and records](../README.md#requests-and-records)). `failed` covers a workflow that raised, an unreadable file, and a failed write. -If rules move into applications (scipp/essapps#33), the application decides retries, and today it can only match on a message. +The automatic reduction does not retry a failed record; an application that retries one (scipp/essapps#33) can today only match on a message. AiiDA's split is the minimum it would need: the workflow reported a failure, the workflow code raised, or the framework could not run it. **Transient infrastructure failures pause instead of failing.** A transport task (upload, submit, poll, retrieve) that raises is rescheduled at growing intervals. After 5 consecutive failures the job is paused, and `verdi process play` resumes it from the failed task ([calculations/concepts.rst L176-200][backoff]). Each computer also sets a minimum interval between scheduler polls ([computers.py L73-74][poll]). For us, a slow or missing `/ess` mount is the likely transient failure on the service. -Today it fails the record, and since a failed record counts as handled ([automatic-reduction.md](../automatic-reduction.md), The trigger loop), automatic reduction never reduces that run. +Today it fails the record, and since a failed record counts as a row reduced ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)), automatic reduction never reduces that run again. Whether the workspace retries before it fails a record, or marks the failure so a controller may resubmit, belongs to the service design (scipp/essapps#27). **Checkpointed workflow state does not survive upgrades.** A WorkChain resumes after a daemon restart because its state is serialised between steps. diff --git a/docs/developer/prior-art/ewoks.md b/docs/developer/prior-art/ewoks.md index 16508ffe..bc290506 100644 --- a/docs/developer/prior-art/ewoks.md +++ b/docs/developer/prior-art/ewoks.md @@ -52,7 +52,7 @@ The listener (`app/workflow_server`, 46 lines) starts at the newest scan (`get_l This bears on the one question that blocks our design, how files say what role a run plays ([requirements](../../requirements/README.md#questions-that-block-the-design)): ESRF writes the reduction request itself into the raw data. At ESS that would need NICOS to write it. It agrees with [#33][i33]: the configuration per beamline lives in `blissoda`, beside the acquisition system, not in ewoks. -Our trigger loop instead counts a file as handled when a record under the rule's label names it, so downtime loses nothing ([automatic-reduction.md](../automatic-reduction.md#the-trigger-loop)). +Our automatic reduction instead counts a row's file as reduced when a record under the batch's label holds it, so downtime loses nothing ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). **Calibrations are chosen from the state at trigger time.** ID11 takes the last file in a directory by name (`sorted(glob(pattern))[-1]`, `id11/xrpd_processor.py` L113), ID31 reads the live energy motor (`id31/xrpd_processor.py` L151), and the streamline scanner takes the calibration scan nearest the current motor position (`streamline/scanner.py` L445). diff --git a/docs/developer/prior-art/git.md b/docs/developer/prior-art/git.md index 86269934..1658cfba 100644 --- a/docs/developer/prior-art/git.md +++ b/docs/developer/prior-art/git.md @@ -63,9 +63,9 @@ The catalogue entry survives the proposal's history for the same reason a pushed **Names need a namespace.** Git keeps branches, tags, and remote branches under separate prefixes (`refs/heads/`, `refs/tags/`, `refs/remotes/`), resolves a short name by a fixed order of rules, and warns when it is ambiguous ([revisions L27-L33][revname], `core.warnAmbiguousRefs`, true by default, [config/core L407-L409][warn]). Here, a label is one free string per proposal. -A rule owns its label by convention only: a record a notebook submits under it stops the rule from reducing the files that record names ([automatic-reduction.md](../automatic-reduction.md#the-trigger-loop)). +The automatic reduction owns its label by convention only: a record a notebook submits under it, with a row's key as its member, stops the loop from reducing that row's file ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). Labels are tentative ([open question 3](../README.md#open-questions)). -If they stay, separate the names of rules from the names people choose before the first users, since renaming names that records already carry is costly. +If they stay, separate the labels of the automatic reduction from the labels people choose before the first users, since renaming names that records already carry is costly. **The interface leaked the model.** Perez De Rosso and Jackson trace git's usability problems to its concepts, such as the staging area and detached HEAD, not to its commands ([Onward! 2013][wwg]). @@ -76,9 +76,9 @@ The first-release application ([requirements, Goal](../../requirements/README.md **Automation belongs next to the versioned content, and its status on a server.** Git hooks live in `$GIT_DIR/hooks`, outside the tracked tree ([gitrepository-layout L170-L178][hooks]), so a clone does not carry them. CI services read their configuration from a tracked file, show the status of each run to anyone with access, and leave re-running failed jobs to a person, with the same commit as the first run ([GitHub docs][rerun]). -The trigger loop matches the last point: a failed record counts as handled, and running it again is the user's decision ([automatic-reduction.md](../automatic-reduction.md#the-trigger-loop)). -It differs on the second: a rule's status lives in the `TriggerLoop` object, which an operator in another process cannot read ([#24][i24]). -A rule is an application's function ([#33][i33]), so such a server lists the names and statuses of its rules, not what they submit. +The automatic reduction matches the last point: a failed record counts as a row reduced, and running it again is the user's decision ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). +It differs on the second: why a row did not run lives in the `Automatic` object, which an operator in another process cannot read ([#24][i24]). +A source of rows is an application's function ([#33][i33]), so such a server lists the batches and the states of their rows, not how the rows are made. **Where git's model does not apply.** Git merges histories by text. @@ -89,9 +89,9 @@ Git is distributed with no single writer; here one process writes a proposal's h - Order under a label by the order logged, or by a link to the previous record; never by time. - References to identities, never to names that move. -- Separate namespaces for rules' labels and people's labels, if labels stay. +- Separate namespaces for the labels of the automatic reduction and people's labels, if labels stay. - A hash and size per persisted output, when the store's layout is designed. -- Rule status kept where other processes can read it. +- Why a row of the automatic reduction did not run, kept where other processes can read it. ## Do not borrow diff --git a/docs/developer/prior-art/mantid.md b/docs/developer/prior-art/mantid.md index 236acd0c..dd58d707 100644 --- a/docs/developer/prior-art/mantid.md +++ b/docs/developer/prior-art/mantid.md @@ -33,7 +33,7 @@ The interface's documentation says so ([ISIS Reflectometry.rst L646-L650](https: In 2024, on ISIS's analysis machines (IDAaaS), the journal began to list runs seconds before their files reached the data cache, so autoprocessing failed those rows and never came back to them ([mantid#36860](https://github.com/mantidproject/mantid/issues/36860)). Retrying failed rows at each poll was considered and rejected as confusing for users. The fix lists only runs found in the data cache's index ([mantid#37449](https://github.com/mantidproject/mantid/pull/37449)), together with slower polling ([mantid#36895](https://github.com/mantidproject/mantid/pull/36895)). -For us: the trigger loop makes the same choice, a failed record counts as handled ([automatic-reduction.md](../automatic-reduction.md#the-trigger-loop)). +For us: the automatic reduction makes the same choice, a failed record counts as a row reduced ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). So the file source must list a run's file only once it is complete; otherwise the first reduction fails and the file is never reduced again without a person. This bears on the Open question of how long after the file writer finishes the catalogue lists a run ([data](../../requirements/data.md#open)). @@ -65,7 +65,7 @@ Either change resets the group, so it is stitched again ([Group.cpp L195-L205](h FIA's INTER rule attaches every file in the run's directory with the same title prefix to the new run's job ([inter_rules.py L32-L44](https://github.com/fiaisis/run-detection/blob/881dacdc3c07d5ee1cbf12d49843d5a8fe7fe397/rundetection/rules/inter_rules.py#L32-L44)), but FIA's INTER script only raises "script not implemented" ([INTER/reduce.py](https://github.com/fiaisis/autoreduction-scripts/blob/de9e1bfd96e67a6b216d7ecd8e8e3f0b868a7b5b/INTER/reduce.py)). For TOSCA, OSIRIS and IRIS, a rule walks back over consecutive run numbers with similar titles and adds a summed job per arrival ([common_rules.py L69-L127](https://github.com/fiaisis/run-detection/blob/881dacdc3c07d5ee1cbf12d49843d5a8fe7fe397/rundetection/rules/common_rules.py#L69-L127)). For SANS, a scatter run looks for its transmission run by title and a transmission run for its scatter run, so whichever arrives last starts the job ([sans_rules.py L275-L315](https://github.com/fiaisis/run-detection/blob/881dacdc3c07d5ee1cbf12d49843d5a8fe7fe397/rundetection/rules/sans_rules.py#L275-L315)). -For us: a rule's function can reduce every run of the series at each arrival, as FIA's INTER rule would ([automatic-reduction.md](../automatic-reduction.md#the-trigger-loop)). +For us: a row whose trigger column is a list holds every run of the series so far, and is reduced again at each arrival, as FIA's INTER rule would ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). Mantid's cost per arrival is one row reduction plus one stitch; with our core that is one record per angle and a stitch request that references the persisted per-angle outputs. Whether ESS automatic reduction should stitch as angles arrive is Open ([reflectometry](../../requirements/reflectometry.md#open)). @@ -79,7 +79,7 @@ For us: a record never changes, and a changed template affects only requests mad **Exclusions carry a reason.** A user can type an exclude reason next to a run in the search results; excluded runs are not transferred to the table, and the reasons save with the batch ([QtSearchModel.cpp L180-L181](https://github.com/mantidproject/mantid/blob/0a137707b24563d2e5d690951651fcee052ce635/qt/scientific_interfaces/ISISReflectometry/GUI/Runs/QtSearchModel.cpp#L180-L181), [RunsPresenter.cpp L492](https://github.com/mantidproject/mantid/blob/0a137707b24563d2e5d690951651fcee052ce635/qt/scientific_interfaces/ISISReflectometry/GUI/Runs/RunsPresenter.cpp#L492)). -For us: the trigger loop has no exclusions; under scipp/essapps#33 they would be configuration of the application's rule. +For us: a person marks a file `exclude=True` (`CorrectedFiles`), and a source of rows that reads it leaves the file out; no reason is kept. **Provenance is a script, not a structured request.** The reflectometry interface removed its notebook export in Mantid 4.1 "as this was not used and not useful" ([release notes](https://github.com/mantidproject/mantid/blob/0a137707b24563d2e5d690951651fcee052ce635/docs/source/release/v4.1.0/reflectometry.rst?plain=1#L164)); since 6.10 the ORSO output file embeds the reduction script ([release notes](https://github.com/mantidproject/mantid/blob/0a137707b24563d2e5d690951651fcee052ce635/docs/source/release/v6.10.0/reflectometry.rst?plain=1#L13)). @@ -89,14 +89,14 @@ For us: the record holds the request; the container image is a precedent for sof **A rule edited in place rewrote history.** In ISIS autoreduction, a run pointed at the database row of values that filled it ([records.py L170-L188](https://github.com/autoreduction/queue-processor/blob/4eab9dea17df1fe599a5fcf23a77ea1e2554de2e/autoreduce_qp/model/database/records.py#L170-L188)), and editing the values "from run N onward" overwrote that row ([configure_new_runs.py L49-L60](https://github.com/autoreduction/frontend/blob/917b665c06c91d9bfc3733c89fcbf74fa0937413/autoreduce_frontend/reduction_viewer/views/configure_new_runs.py#L49-L60)), so old runs then showed values they never used. -For us: a record holds every parameter value and does not name the rule or template it came from ([automatic-reduction.md](../automatic-reduction.md#rules)). +For us: a record holds every parameter value and does not name the batch or template it came from ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). **FIA keeps rules as code and their configuration as data.** Each instrument has a JSON *specification* in FIA's database; each key names a Python rule class, and its value configures it ([specifications.py](https://github.com/fiaisis/run-detection/blob/881dacdc3c07d5ee1cbf12d49843d5a8fe7fe397/rundetection/specifications.py), [models.py L136](https://github.com/fiaisis/FIA-API/blob/9707aa094c4e4a9c96f31142e8777f2e0c430273/fia_api/core/models.py#L136)). An API call replaces the specification ([instrument_specs.py L38-L46](https://github.com/fiaisis/FIA-API/blob/9707aa094c4e4a9c96f31142e8777f2e0c430273/fia_api/routers/instrument_specs.py#L38-L46)), and run detection fetches it from the API. FIA has working scripts for ENGIN-X, GEM, IMAT, INES, IRIS, LOQ, MARI, OSIRIS, SANS2D, TOSCA and VESUVIO; those for the reflectometers INTER, POLREF and SURF are placeholders ([autoreduction-scripts](https://github.com/fiaisis/autoreduction-scripts/tree/de9e1bfd96e67a6b216d7ecd8e8e3f0b868a7b5b)). It learns of new runs from each instrument's last-run file, not from the catalogue, and after a restart it submits the runs between the last one it recorded and the current one ([lastrun_file_monitor.py L263](https://github.com/fiaisis/file-watcher/blob/bb9101322fc485b15cc16e448e705ec42f614ac0/file_watcher/lastrun_file_monitor.py#L263)). -For us: this is the shape scipp/essapps#33 proposes, rule functions in the application with their configuration as data, and one answer to its Open question: FIA stores the configuration behind an API so that it changes without a deployment. +For us: this is the shape scipp/essapps#33 proposes, the source of rows in the application with its settings as data, and one answer to its Open question: FIA stores the configuration behind an API so that it changes without a deployment. ## Borrow diff --git a/docs/developer/prior-art/snakemake.md b/docs/developer/prior-art/snakemake.md index e10f7931..bcabc72a 100644 --- a/docs/developer/prior-art/snakemake.md +++ b/docs/developer/prior-art/snakemake.md @@ -26,9 +26,9 @@ A bug made the environment trigger fire on unchanged environments; 7.8.1 fixed i The maintainer kept the new default and pointed users to `--rerun-triggers mtime`, which shipped with 7.8.0, and to `--cleanup-metadata `, which existed before, to silence a trigger. He reported as many requests for the new behaviour as complaints about it. Every planned job prints the reason it runs ([FAQ](https://github.com/snakemake/snakemake/blob/4523356973a83aa237a121717ee81dd2e09175da/docs/project_info/faq.rst#L526-L531)). -For us: the trigger loop counts a file as handled once a record under the rule's label names it, and no record names the rule ([automatic-reduction.md](../automatic-reduction.md#the-trigger-loop)). -So changing what a rule's function submits reduces no handled file again, and whether a stored result is out of date stays the user's decision. -If a story asks for it, borrow `--list-changes params` ([cli.py](https://github.com/snakemake/snakemake/blob/4523356973a83aa237a121717ee81dd2e09175da/src/snakemake/cli.py#L1230-L1236)): a query listing the records under a label whose request differs from what the rule would submit now, for the user to resubmit. +For us: the automatic reduction counts a row's file as reduced once a record under the batch's label holds it, whatever settings and source made the record ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). +So changing the settings or the source reduces no row again, and whether a stored result is out of date stays the user's decision. +If a story asks for it, borrow `--list-changes params` ([cli.py](https://github.com/snakemake/snakemake/blob/4523356973a83aa237a121717ee81dd2e09175da/src/snakemake/cli.py#L1230-L1236)): a query listing the records under a label whose request differs from what the batch would submit now, for the user to resubmit. **What a record of a run holds, and at what grain.** For each output, Snakemake stores the rule, input paths, parameter values, the code, checksums of small inputs, a hash of the software environment, and a list of package names and versions ([MetadataRecord](https://github.com/snakemake/snakemake/blob/4523356973a83aa237a121717ee81dd2e09175da/src/snakemake/persistence/__init__.py#L32-L49), [filled](https://github.com/snakemake/snakemake/blob/4523356973a83aa237a121717ee81dd2e09175da/src/snakemake/persistence/__init__.py#L428-L450)). @@ -45,7 +45,7 @@ On Slurm, failures such as `OUT_OF_MEMORY`, `TIMEOUT`, and `NODE_FAIL` come from For us: `ess.apps.Workspace` records the message of the workflow's Python exception. A segfault in a C extension, or a process killed for memory, raises none. On the service (scipp/essapps#27), a failed record needs the workflow's console output and the batch system's reason, captured by the framework rather than by each binding. -Because its outputs are gone, Snakemake retries a failed job at the next invocation; our trigger loop counts a failed record as handled, and running it again is the user's decision ([automatic-reduction.md](../automatic-reduction.md#the-trigger-loop)). +Because its outputs are gone, Snakemake retries a failed job at the next invocation; our automatic reduction counts a failed record as a row reduced, and running it again is the user's decision ([automatic-reduction.md](../automatic-reduction.md#which-rows-run)). **A shared file system shows files late, and the batch system reports late.** After a cluster job ends, Snakemake waits up to `--latency-wait` seconds, 5 by default, for its outputs to appear ([cli.py](https://github.com/snakemake/snakemake/blob/4523356973a83aa237a121717ee81dd2e09175da/src/snakemake/cli.py#L1389-L1397)); open issues report slow scheduling on high-latency file systems ([#2920](https://github.com/snakemake/snakemake/issues/2920), [#4240](https://github.com/snakemake/snakemake/issues/4240)). @@ -74,7 +74,7 @@ For us: a workspace started on history that another workspace still writes is re **Temporary and protected outputs.** `temp()` deletes an output once every consumer has run, and `protected()` write-protects an expensive one ([rules.rst](https://github.com/snakemake/snakemake/blob/4523356973a83aa237a121717ee81dd2e09175da/docs/snakefiles/rules.rst#L2132-L2160)). The workflow author marks them, since the author knows which outputs are throwaway. -For us: nothing is written unless persisted, and a rule's function says what it persists ([ADR 0002](../adr/0002-a-value-lives-while-something-keeps-it.md), [automatic-reduction.md](../automatic-reduction.md#rules)). +For us: nothing is written unless persisted, and the batch of the automatic reduction says what it persists ([ADR 0002](../adr/0002-a-value-lives-while-something-keeps-it.md), [batch.md](../../../packages/essapps/docs/batch.md#automatic-reduction)). If a story asks, an author's mark on outputs meant only for inspection could set that default. **Between-workflow caching is opt-in and narrow.** @@ -101,7 +101,7 @@ For us: the service's executor is undecided (scipp/essapps#27); its interface is - The workflow's console output and the batch system's failure reason on a failed record, captured by the framework. - Completion reported by the worker after its writes are closed, never inferred from a file appearing. - Memory declared by the binding's author, overridable by the site, with a default from input size. -- If asked for: a query for records whose request differs from what a rule would submit now. +- If asked for: a query for records whose request differs from what a batch would submit now. ## Do not borrow diff --git a/docs/developer/roles/client-authors.ipynb b/docs/developer/roles/client-authors.ipynb index e9ca5f74..cd619017 100644 --- a/docs/developer/roles/client-authors.ipynb +++ b/docs/developer/roles/client-authors.ipynb @@ -765,7 +765,7 @@ "| notebook | `local(...)` | `submit`, `output`, stages, accumulators | releasing large values; not modifying values that `output` returned (see below) |\n", "| local app (not a notebook) | `local(...)` | the same, behind a form | releasing what it no longer shows; choosing the rows of each table |\n", "| app against the service | `connect(url, proposal=...)` (not implemented) | `submit(..., persist=True)`, labels, `latest` | releasing what it no longer shows (how the service bounds a workspace's memory is open); reading results from the store |\n", - "| automatic reduction | either | `Rule`, `TriggerLoop` | choosing what to persist |\n", + "| automatic reduction | either | `ess.apps.batch.auto` | choosing what to persist |\n", "| an app for many samples | `local(...)` | `ess.apps.batch` | see its [guide](../../../packages/essapps/docs/batch.md) |\n", "| program, CLI, agent | `connect(...)` (not implemented) | `SpecId`, `SerializedWorkflowSpec` | importing no workflow package |\n", "\n", @@ -804,7 +804,7 @@ "id": "48", "metadata": {}, "source": [ - "A `Rule` is a name, a function, a selector, a label, and the parameter `at` that holds the file. The `TriggerLoop` calls the function for each new file that the selector matches; the function pairs runs as the technique needs, submits what the file needs, and returns the record that settles the file. See [automatic-reduction.md](../automatic-reduction.md).\n", + "Automatic reduction makes such a batch from the files as they arrive, with `ess.apps.batch`. The application's source pairs runs as the technique needs, and `Automatic(batch, at=..., settings=...)` reduces each row that holds a new file at the column `at`. See [automatic-reduction.md](../automatic-reduction.md).\n", "\n", "**Values in the user's process are not copied.**\n", "In a notebook, `client.output` of a record returns the value the framework holds, not a copy.\n", diff --git a/docs/developer/system-stories.md b/docs/developer/system-stories.md index aad374f3..750c0ff9 100644 --- a/docs/developer/system-stories.md +++ b/docs/developer/system-stories.md @@ -115,4 +115,4 @@ Property: every record pending at the upgrade with a persist request finishes wi ### H3. History ends when the work does Actor: user. Goal: the records of a proposal are there while the user works with them, and for days to weeks after; what was published lasts. -Property: once a proposal has been idle for the retention period, with no call and no record pending, its history is dropped as a whole, never in part. So a kept record's provenance is complete. A running trigger loop keeps its proposal from being idle, so it never reduces a handled file again. A published entry still answers what produced it after the history behind it is dropped. +Property: once a proposal has been idle for the retention period, with no call and no record pending, its history is dropped as a whole, never in part. So a kept record's provenance is complete. A running automatic reduction keeps its proposal from being idle, so it never reduces a row again for a file it has reduced. A published entry still answers what produced it after the history behind it is dropped. diff --git a/docs/developer/three-ways-to-run-a-spec.html b/docs/developer/three-ways-to-run-a-spec.html index f47b348c..a60d75de 100644 --- a/docs/developer/three-ways-to-run-a-spec.html +++ b/docs/developer/three-ways-to-run-a-spec.html @@ -153,7 +153,7 @@

Three ways to run a spec

state n
An accumulator after its first n pushes. Not a record: client.output copies it, and client.freeze makes the logged state a record of its plain request.
logged state
State n for the n pushes logged so far, whether or not they have been added. A read is of the logged state when it is called, a freeze once no other freeze of the accumulator is being checked.
driver
The object behind an accumulator, with push(table, rows) and outputs(). The workflow package writes it, e.g. to sum a numerator and a denominator.
-
controller
Client code that decides what to submit, push, and release: a notebook, an app, the trigger loop.
+
controller
Client code that decides what to submit, push, and release: a notebook, an app, the loop of automatic reduction.
@@ -604,8 +604,8 @@

What else a full picture needs, and what it can leave out

Needed to read the grid
  • Records and references. Every connection between requests is a reference: record.ref('centre') or file(run=611). A reference never names an accumulator, only the record of its freeze. A reference to a pending record is valid. The workspace waits for it.
    • -
  • Template is the shared building block: stage, accumulator, batch, and rules all start from one.
    • -
  • Controllers. The workspace never decides what to submit or push. A notebook, an app, or the TriggerLoop of rules does.
    • +
  • Template is the shared building block: stage, accumulator, and batch, the automatic reduction among them, all start from one.
    • +
  • Controllers. The workspace never decides what to submit or push. A notebook, an app, or the loop of automatic reduction (ess.apps.batch.auto) does.
  • Where it runs. Both: values in memory while something keeps them, written only when persisted. The service (designed, not built) runs one workspace for each proposal. Whether the users of a proposal share its values in memory, or the service separates or bounds them per user, is open. So are how it bounds a workspace's memory, and where it holds an accumulator's driver. Placement says the results do not differ.
@@ -615,7 +615,7 @@

What else a full picture needs, and what it can leave out

  • Files and the file source (client.files, selectors, watch)
  • Labels and members (tentative)
  • Status, failure, cancel, as_completed
    • -
  • Rules
    • +
  • Automatic reduction
  • Provenance and publication
  • Release and lifetimes of values
    • diff --git a/docs/developer/tours/layers.html b/docs/developer/tours/layers.html index bd5c237a..bf85c439 100644 --- a/docs/developer/tours/layers.html +++ b/docs/developer/tours/layers.html @@ -21,7 +21,7 @@

    essapps, layer by layer

    - +
    @@ -107,7 +107,7 @@

    essapps, layer by layer

    # a dropdown labels kept[entry] by log[entry]` }, { level: 7, title: 'That is a workspace', - body: `

    Keep going, and the form needs more. Drop entries that nobody needs before memory runs out. Show the same entries in a notebook, to a colleague, and to the trigger loop. Reference an output whose call has not finished. Run the same way at DMSC.

    + body: `

    Keep going, and the form needs more. Drop entries that nobody needs before memory runs out. Show the same entries in a notebook, to a colleague, and to the automatic reduction. Reference an output whose call has not finished. Run the same way at DMSC.

    A log of requests, the values kept while something holds them, the file source, and the bindings: that is a workspace, built once for every caller. local() needs nothing that the script does not: pydantic and the standard library.

    It also resolves each file when a request is submitted, so the record names the file itself, and checks that the proposal may read it.

    `, code: `client = local(proposal='p1', @@ -171,25 +171,24 @@

    essapps, layer by layer

    This is why the slides are a walk through a small grid, not a single line: see the panel on the right.

    `, code: `client = connect('https://reduce.example', proposal='p1') app = App(client)` }, - { level: 11, caller: 'trigger', placement: 'service', planned: SERVICE, title: 'A trigger loop instead of a person', - body: `

    Automatic reduction is one more controller. For each new file that a rule's selector matches, it calls the application's function, which submits what the file needs, persisted.

    + { level: 11, caller: 'trigger', placement: 'service', planned: SERVICE, title: 'A loop instead of a person', + body: `

    Automatic reduction is one more controller. It scans a batch as files arrive, and reduces each row that holds a new file at its trigger column, persisted. The application's source says which files make a row.

    It runs on a server at DMSC, outside the workspace. The workspace never decides what to run.

    `, - code: `def stitch(client, run): - sample = client.files.metadata(run)['sample'] - angles = client.files.list( - Selector(role='sample', sample=sample)) - return client.submit( - STITCH, params={'runs': angles, 'reference': file(run=500)}, - label='reflectivity', member=sample, persist=True) - -rule = Rule('reflectivity', stitch, - selector=Selector(role='sample'), - at='runs', label='reflectivity') -TriggerLoop(client, rules=[rule]).run()` }, + code: `def angles(files): + runs = {} + for f in sorted(files, key=lambda f: f.metadata['run']): + runs.setdefault(f.metadata['sample'], []).append(f.ref) + return [Row(s, {'runs': refs}) for s, refs in runs.items()] + +batch = Batch(client, STITCH, columns=('runs',), + label='reflectivity', source=angles, + selector=Selector(role='sample'), persist=True) +run([Automatic(batch, at='runs', + settings=lambda: {'reference': file(run=500)})])` }, { level: 11, caller: 'all', placement: 'service', planned: SERVICE, title: 'Many controllers, one history', - body: `

    All clients of a proposal share its history. A notebook can reference a transmission that the trigger loop computed, and the app lists that record in its pickers.

    -

    Every client of a workspace reads the same values. The trigger loop persists its outputs, so they outlive the workspace. A proposal has one workspace. Whether its users share the values in memory, or the service separates or bounds them per user, is open. Provenance follows each reference back to the files.

    `, - code: `# computed by the trigger loop + body: `

    All clients of a proposal share its history. A notebook can reference a transmission that the automatic reduction computed, and the app lists that record in its pickers.

    +

    Every client of a workspace reads the same values. The automatic reduction persists its outputs, so they outlive the workspace. A proposal has one workspace. Whether its users share the values in memory, or the service separates or bounds them per user, is open. Provenance follows each reference back to the files.

    `, + code: `# computed by the automatic reduction t = client.latest('transmission', member='250K') record = client.submit(IOFQ, params={ ..., 'transmission': t.ref('transmission')}) @@ -217,7 +216,7 @@

    essapps, layer by layer

    const GROUPS = [['Workflow', 0, 3], ['Without a workspace', 3, 7], ['Workspace', 7, 12], ['Swap caller or placement', 12, 17], ['Other bindings', 17, 19]]; const NAMES = { - caller: { notebook: 'Notebook', app: 'Panel app', trigger: 'Trigger loop', all: 'All three' }, + caller: { notebook: 'Notebook', app: 'Panel app', trigger: 'Automatic reduction', all: 'All three' }, placement: { local: 'local()', service: 'service' }, binding: { pipeline: 'Pipeline', driver: 'Driver', function: 'Function' }, }; @@ -282,15 +281,15 @@

    essapps, layer by layer

    sample run 60339 ▾
    transmission record ▾
    Submit
    `), - trigger: h('Trigger loop', 'automatic reduction', `
    for each new file:
-  fill the template
-  submit(label=...,
-         persist=True)
    `), + trigger: h('Automatic reduction', 'a batch, stepped', `
    at each step:
+  scan the files
+  reduce each row with
+  a new file
    `), }; const CALLER_SMALL = [ h('Jupyter notebook', '', '
    client.submit(IOFQ, ...)
    '), h('Panel app', '', '
    App(client)
    '), - h('Trigger loop', '', '
    TriggerLoop(client, ...)
    '), + h('Automatic reduction', '', '
    run([Automatic(batch, ...)])
    '), ]; function layout(cfg, { box, arrow }) { @@ -386,7 +385,7 @@

    essapps, layer by layer

    caller: { notebook: 'A notebook submits requests and reads outputs through the client.', app: 'The Panel app makes a form per spec from its params schema and submits through the client.', - trigger: 'The trigger loop fills a template for each new file a rule matches, and submits it under the rule\'s label.', + trigger: 'The loop of automatic reduction scans a batch, and submits each row with a new file at its trigger column under the batch\'s label.', all: 'Several controllers share one proposal\'s history, each through its own client.', }, placement: { @@ -405,8 +404,8 @@

    essapps, layer by layer

    caller: { notebook: `record = client.submit(SPEC, params={...})\nclient.output(record, ...)`, app: 'app = App(client)', - trigger: 'TriggerLoop(client, rules=[rule]).run()', - all: '# each in its own process, with its own client\nclient.submit(SPEC, params={...})\nApp(client)\nTriggerLoop(client, rules=[rule]).run()', + trigger: 'run([Automatic(batch, at=..., settings=...)])', + all: '# each in its own process, with its own client\nclient.submit(SPEC, params={...})\nApp(client)\nrun([Automatic(batch, ...)])', }, }; diff --git a/docs/developer/user-stories.md b/docs/developer/user-stories.md index d69910be..4f11deb9 100644 --- a/docs/developer/user-stories.md +++ b/docs/developer/user-stories.md @@ -108,12 +108,14 @@ Besides the calls in README.md, the stories use this one: record.request.files() # the files the request names directly ``` -The trigger loop belongs to the sub-design for automatic reduction. -The stories use two of its calls: +Automatic reduction belongs to its sub-design ([automatic-reduction.md](automatic-reduction.md)), and is a batch of `ess.apps.batch` ([batch.md](../../packages/essapps/docs/batch.md)). +The stories use these calls: ```python -loop.step() # handles what arrived since the last step; returns the records it made -loop.status(rule).reason # why a rule submitted nothing or skipped a file +batch = Batch(client, SPEC, columns=(...), label=..., source=rows_of_files, persist=True) +auto = Automatic(batch, at='sample', settings=lambda: settings) +auto.step() # scans, and reduces the rows with a new file at `at`; their records by key +auto.refused # why each such row did not run, by key ``` ## S. Small stories @@ -637,29 +639,39 @@ That a cut is ready within seconds of each run, and how history grows with the r ## E. Automatic reduction +The stories E2 to E4 reduce a row per run: + +```python +def per_run(files): # every raw file + return [Row(str(f.metadata['run']), {'sample': f.ref}) for f in files] + +def iofq_batch(client): + return Batch(client, IOFQ, columns=('sample',), label='iofq', source=per_run, persist=True) +``` + ### E1. Series grows, reduction follows Actor: reflectometry user during a beamtime. Goal: after each angle, the stitched curve grows by one angle; one curve per sample, not one per arrival. ```python reference = measure(1, [1.0, 1.0], role='reference') -template = Template(STITCH, params={'reference': reference}, blanks=('runs',)) -def stitch(client, run): # all angles of the sample so far, in run order - sample = client.files.metadata(run)['sample'] - angles = client.files.list(Selector(role='sample', sample=sample)) - angles.sort(key=lambda ref: client.files.metadata(ref)['run']) - return client.submit(template.fill(params={'runs': angles}), label='reflectivity', member=sample, persist=True) +def angles(files): # a row per sample: all its angles so far, in run order + runs = {} + for f in sorted(files, key=lambda f: f.metadata['run']): + if f.metadata.get('role') == 'sample': + runs.setdefault(f.metadata['sample'], []).append(f.ref) + return [Row(sample, {'runs': refs}) for sample, refs in runs.items()] -rule = Rule('reflectivity', stitch, selector=Selector(role='sample'), at='runs', label='reflectivity') -loop = TriggerLoop(client, rules=[rule]) +batch = Batch(client, STITCH, columns=('runs',), label='reflectivity', source=angles, persist=True) +auto = Automatic(batch, at='runs', settings=lambda: {'reference': reference}) r2 = measure(2, [1.0, 2.0], role='sample', sample='si') -loop.step() +client.wait(auto.step()) # a row still running waits for its record r4 = measure(4, [4.0, 8.0], role='sample', sample='si') r3 = measure(3, [1.0, 1.0], role='sample', sample='si') measure(2, [1.0, 2.0], role='sample', sample='si') # arrives again -loop.step() +auto.step() user = connect() # any client of the proposal latest = user.latest('reflectivity', member='si') @@ -668,60 +680,52 @@ assert latest.request.params['runs'] == [r2, r3, r4] # run order, not arrival assert user.output(latest, 'stitched').curve.values.tolist() == [1.0, 2.0, 2.0, 2.0, 2.0, 4.0] ``` -Each record is a plain request that stitches every angle so far, one row per angle. An accumulator over `STITCH` would give the same curves (C4); whether a rule pushes into an accumulator is open ([automatic-reduction.md](automatic-reduction.md)). +Each record is a plain request that stitches every angle so far, one row per angle. An accumulator over `STITCH` would give the same curves (C4); whether the automatic reduction pushes into an accumulator is open ([automatic-reduction.md](automatic-reduction.md)). -The function persists what it submits, so any client of the proposal reads each curve ([`ess.apps.protocol`](../../packages/essapps/src/ess/apps/protocol.py)). +The batch persists what it submits, so any client of the proposal reads each curve ([`ess.apps.protocol`](../../packages/essapps/src/ess/apps/protocol.py)). ### E2. Automatic reduction goes quiet -Actor: instrument scientist. Goal: after an upgrade removed the template's spec version, see why nothing is reduced. +Actor: instrument scientist. Goal: after an upgrade removed the spec version of the settings, see why nothing is reduced. ```python -def iofq_rule(template, selector): - def submit(client, run): - member = str(client.files.metadata(run)['run']) - return client.submit(template.fill(params={'sample': run}), label='iofq', member=member, persist=True) - return Rule('auto-iofq', submit, selector=selector, at='sample', label='iofq') - -rule = iofq_rule(Template(IOFQ, params={'normalization': None}, blanks=('sample',)), Selector(role='sample')) upgraded = upgrade(specs=[IOFQ_V2]) # version 1 is gone -loop = TriggerLoop(upgraded, rules=[rule]) -run = measure(1, [1.0, 1.0], role='sample') +auto = Automatic(iofq_batch(upgraded), at='sample', settings=lambda: {'normalization': None}) +measure(1, [1.0, 1.0], role='sample') -assert loop.step() == [] -assert loop.status(rule).reason == f'skipped {run.file}: unknown spec sans-iofq/v1' +assert auto.step() == {} +assert auto.refused == {'1': 'unknown spec sans-iofq/v1'} ``` ### E3. Reduction of our own output -Actor: none; a failure mode. Goal: a published result that SciCat lists as a dataset does not trigger the rule. +Actor: none; a failure mode. Goal: a published result that SciCat lists as a dataset does not make a row. ```python -rule = iofq_rule(Template(IOFQ, params={'normalization': None}, blanks=('sample',)), Selector()) # every raw file -loop = TriggerLoop(client, rules=[rule]) +auto = Automatic(iofq_batch(client), at='sample', settings=lambda: {'normalization': None}) measure(1, [1.0, 2.0, 3.0, 4.0]) -(reduced,) = loop.step() +(reduced,) = auto.step().values() pid = client.publish(reduced.ref('iofq'), 'scicat') files.add_published(scicat.entries[pid]) -assert loop.step() == [] +assert auto.step() == {} ``` -A published entry is a file of kind `'derived'`, and a selector matches only raw files unless it names another kind. +A published entry is a file of kind `'derived'`, and a batch's source is given only raw files unless its selector names another kind. ### E4. Template improved during a beamtime -Actor: instrument scientist. Goal: new runs use the improved template; earlier results stay; the results made before the improvement can be found. +Actor: instrument scientist. Goal: new runs use the improved settings; earlier results stay; the results made before the improvement can be found. ```python -samples = Selector(role='sample') -template = Template(IOFQ, params={'normalization': None}, blanks=('sample',)) +settings = {'normalization': None} +auto = Automatic(iofq_batch(client), at='sample', settings=lambda: settings) first = measure(1, [1.0, 2.0, 3.0, 4.0], role='sample') -(before,) = TriggerLoop(client, rules=[iofq_rule(template, samples)]).step() +(before,) = auto.step().values() -improved = Template(IOFQ, params={'reduction': Reduction(threshold=1.5), 'normalization': None}, blanks=('sample',)) +settings = {'reduction': Reduction(threshold=1.5), 'normalization': None} # read at the next step second = measure(2, [1.0, 2.0, 3.0, 4.0], role='sample') -(after,) = TriggerLoop(client, rules=[iofq_rule(improved, samples)]).step() +(after,) = auto.step().values() user = connect() # any client of the proposal assert after.request.files() == [second] # run 1 is not reduced again @@ -731,7 +735,7 @@ stale = [r for r in user.records(label='iofq') if r.request.params['reduction'][ assert [r.request.files() for r in stale] == [[first]] # to reprocess, if the user wants ``` -The second loop stands in for a running trigger loop whose rule is replaced. It learns that run 1 is handled from the records under `iofq`. +A row runs only for a file at `sample` that no record under `iofq` holds, so a changed setting does not reduce run 1 again. A restarted loop learns the same from the records. ## F. Publication and provenance diff --git a/packages/essapps/docs/batch.md b/packages/essapps/docs/batch.md index 7f52c333..075ebd89 100644 --- a/packages/essapps/docs/batch.md +++ b/packages/essapps/docs/batch.md @@ -58,7 +58,7 @@ batch.cancel() - A **row** has a **key**, a value per column, and read-only **info** to show with it. A cell can hold anything a request can: a file, a reference to an output, a number, a list of runs. - A **source** is your function from files to rows. A **scan** calls it. - An **edit** is a cell set by hand, by a person or by your code. -- A row's **result** is the newest record of the client's submitter, with the batch's spec and label, and the row's key as its member. So two `Batch` objects with the same spec and label are one batch: give each batch its own label. +- A row's **result** is the newest record with the batch's spec and label, and the row's key as its member, whoever submitted it. So two `Batch` objects with the same spec and label are one batch: give each batch its own label. Choose a key that does not change from one scan to the next, such as the run number. Results and edits are found by key. Put the name of the sample in `info`, under a name that is not a column. @@ -95,7 +95,7 @@ A row added by hand stays. An edited row that a scan no longer gives is not show ## Writing a source -A source takes `FileInfo(ref, metadata)` for each file, and returns `Row`s. It calls nothing, so test it on plain data: +A source takes `FileInfo(ref, metadata)` for each file, and returns `Row`s. It calls nothing but, in a chain, the `state` of another batch ([Chains](#chains)), so test it on plain data: ```python def test_a_sample_run_pairs_with_its_transmission_run(): @@ -110,7 +110,7 @@ Pairing is yours to write. The framework knows no technique. If a source raises, ## When the metadata is wrong or missing -Wrap the file source, and give the wrapper to the workspace. Forms, rules, and sources then all see a file as corrected: +Wrap the file source, and give the wrapper to the workspace. Forms and sources then all see a file as corrected: ```python files = CorrectedFiles(FolderFiles('/data', table)) @@ -140,6 +140,49 @@ for key, record in centres.items(): After a centre failed, this reruns only what is left, and points those rows at the new centre. The field must be typed `Referenced[...]` in the spec. +An edit is lost when the session ends. A source that reads the other batch's state makes the same cells at every scan, which the automatic reduction needs: + +```python +def with_centre(files: Sequence[FileInfo]) -> list[Row]: + found = {k: s.record for k, s in find.state().items() if s.record} + return [replace(row, values={**row.values, 'beam_centre': found[row.key].ref('centre')}) + if row.key in found else row + for row in pair_by_sample(files)] +``` + +Scan `find` before a batch with this source, so that its rows see the newest centres. + +## Automatic reduction + +`Automatic` reduces a batch as its files arrive. It needs a source, and the batch should persist what it submits, since nothing releases its records: + +```python +from ess.apps.batch.auto import Automatic, run + +batch = Batch(client, IOFQ, columns=('sample', 'sample_transmission'), label='auto/iofq', + source=pair_by_sample, staged=True, persist=True) +auto = Automatic(batch, at='sample', settings=lambda: settings) +auto.step() # scans, and reduces the rows with a new file at `sample`; their records by key +auto.refused # why each such row did not run, by key: {'60339': '60339: sample_transmission: empty'} +run([auto]) # steps forever, once a second +``` + +`at` is the *trigger column*, of a file or a list of files. `settings` is called at each step, so a change applies to the rows not reduced yet. + +**A row runs if its cell at `at` holds a file that no record under the label, with the row's key, holds at `at`.** Any record counts, failed, cancelled, of any version of the spec, and of any submitter. So the automatic reduction differs from a person's reduce: + +| A row runs if | no result | failed or cancelled | a setting or another cell changed | a new file at the trigger column | +|---|---|---|---|---| +| the page, "reduce remaining" | yes | yes | yes (stale) | yes | +| `essapps-batch reduce --remaining` | yes | yes | no | no | +| automatic | yes | no | no | yes | + +A row that cannot run, such as one without its transmission run, or whose request is refused, does not hold back the others, and `refused` says why. A reason that names no row, about the settings, refuses every row. A row that waits is looked at again when its cells change, a row still running when its record finishes, and a refused row also when the settings change. + +The loop reads the records from the proposal's changes, from the start when an `Automatic` is made, so a restart needs nothing on disk and reduces no row again. Give the automatic reduction its own label: a person's record under it counts as a row reduced. + +In a chain, step the batch that is read first: `run([Automatic(find, ...), Automatic(iofq, ...)])`. [automatic-reduction.md](../../../docs/developer/automatic-reduction.md) holds the design and what is open. + ## From files, and from the command line A batch can be given as two files: a template, and a table with a row for each sample. @@ -249,7 +292,7 @@ run,role,sample ,direct_beam, <- with a column file: a file that is no run ``` -`read_metadata` is a function of the instrument package, from the path of a file to its metadata, or `None` for a file that is not the instrument's (`ess.apps.folder.scan_folder`). The runs come first, by their number, since the order of the rows is the order of measurement. A file that cannot be read is left out, with a warning. Two files of one run are refused. Running `files` again after files gained a `pid` changes their identity: a trigger loop reduces them again, and earlier records name files the folder no longer knows. +`read_metadata` is a function of the instrument package, from the path of a file to its metadata, or `None` for a file that is not the instrument's (`ess.apps.folder.scan_folder`). The runs come first, by their number, since the order of the rows is the order of measurement. A file that cannot be read is left out, with a warning. Two files of one run are refused. Running `files` again after files gained a `pid` changes their identity: the automatic reduction reduces them again, and earlier records name files the folder no longer knows. A file says its kind, its run, and perhaps its sample, but not what a run is for. A row source that pairs runs reads the `role`, and a field such as the direct beam requires it. The run table gives it (`frames.described`): a row describes the file of its `run`, or the file at its `file` if it has no run. A run of the table that has no file in the folder is ignored, so the table may cover more than the folder holds. A file that no row describes keeps what it says of itself, and where both have a column, a cell of the run table that is not empty wins. Two rows for one run are refused. Without `--with`, the roles are typed into `files.csv`, so `files` then refuses to write over a table that exists. @@ -288,7 +331,7 @@ $B reduce settings.yaml rows.csv --label demo -o done.csv python -m ess.apps.batch.panel.demo ws # the rows show as completed ``` -The command finds the results of the app, and the app those of the command, if both use the batch's label and the same submitter (`open_folder` gives `user`), and the outputs are persisted (`Batch(..., persist=True)`). +The command finds the results of the app, and the app those of the command, if both use the batch's label, and the outputs are persisted (`Batch(..., persist=True)`). | Button | Same as | |---|---| diff --git a/packages/essapps/src/ess/apps/batch/corrections.py b/packages/essapps/src/ess/apps/batch/corrections.py index 7114bee8..22f0ac34 100644 --- a/packages/essapps/src/ess/apps/batch/corrections.py +++ b/packages/essapps/src/ess/apps/batch/corrections.py @@ -26,8 +26,8 @@ class CorrectedFiles: """ A file source over ``source``, with corrections a person makes. - The workspace gets this source, so forms, rules, and sources of rows all see - a file as corrected, as :class:`~ess.apps.protocol.FileSource` requires:: + The workspace gets this source, so forms and sources of rows all see a file + as corrected, as :class:`~ess.apps.protocol.FileSource` requires:: files = CorrectedFiles(FolderFiles('/data', table)) client = local(proposal='p1', files=files, bind=BINDINGS) diff --git a/packages/essapps/src/ess/apps/protocol.py b/packages/essapps/src/ess/apps/protocol.py index 0749ca9a..d0da0f06 100644 --- a/packages/essapps/src/ess/apps/protocol.py +++ b/packages/essapps/src/ess/apps/protocol.py @@ -478,8 +478,8 @@ def path(self, ref: FileRef) -> Path: validators of a group that holds a file see its path. So ``path`` neither stages nor downloads the file, and does not check that it can be read: a file that cannot be read fails the workflow that reads it, - so that one unreadable file fails its own record, not a batch or a - rule's submission that holds it. + so that one unreadable file fails its own record, not a batch that + holds it. """ ... From f54693849107224826f8276f7617af7fa2527733 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 13 Nov 2026 12:03:36 +0000 Subject: [PATCH 4/7] fix: a step of the automatic reduction that raises loses no row A step worked out which rows to look at from what changed since the last step, and stored that before it submitted, so an exception in the settings, the source, or a submission left a row unreduced until it changed again. A step now checks every row's cell at the trigger column against the files the records hold, which come from the records only. The rows refused are kept as they were then, and are tried again once their cells, their records, or the settings change. The check reads the cell directly, a file or a list of files, which costs less than comparing the rows with those of the last step. `Automatic` refuses a batch without a source. `Batch.state(keys=...)` gives the same state of a row as `Batch.state()` when a cell references an unknown record: only the rows that reference it lose the status of the records they read. A row given with `EMPTY` in its values has that cell empty. Co-Authored-By: Claude Opus 5.5 --- packages/essapps/src/ess/apps/batch/auto.py | 84 +++++--- .../essapps/src/ess/apps/batch/reduction.py | 17 +- packages/essapps/src/ess/apps/batch/table.py | 5 +- packages/essapps/tests/batch/auto_test.py | 181 +++++++++++++++++- packages/essapps/tests/batch/conftest.py | 7 +- .../essapps/tests/batch/reduction_test.py | 21 +- packages/essapps/tests/batch/table_test.py | 7 + packages/essapps/tests/batch/toys.py | 16 ++ 8 files changed, 294 insertions(+), 44 deletions(-) diff --git a/packages/essapps/src/ess/apps/batch/auto.py b/packages/essapps/src/ess/apps/batch/auto.py index ae16c6e5..6c2d3fca 100644 --- a/packages/essapps/src/ess/apps/batch/auto.py +++ b/packages/essapps/src/ess/apps/batch/auto.py @@ -4,14 +4,18 @@ Automatic reduction: a batch reduced as its files arrive. The guide is the section "Automatic reduction" of ``packages/essapps/docs/batch.md``. -Which files the records of a row hold at the trigger column is read from the -proposal's changes (:meth:`~ess.apps.Client.changes`), from the start when an -:class:`Automatic` is made, so a restart needs nothing on disk. What a step -keeps besides, the rows and the settings of the last step, only spares it -work. A step looks at the rows whose cells changed, whose records changed, and, -if the settings changed, the rows that were refused. It costs the scan of all -files, and the state and submission of the rows it looks at, which -:meth:`Batch.state ` works out for those rows only. +Which files the records of each key hold at the trigger column is read from +the proposal's changes (:meth:`~ess.apps.Client.changes`), from the start when +an :class:`Automatic` is made, so a restart needs nothing on disk. Every step +works out from these files which rows run, so a step that raises, in the +changes, the settings, the source, or a submission, loses nothing: the next +step tries again. Besides, a step keeps only the rows that were refused, as +they were then, and the settings: a refused row is tried again once its cells, +its records, or the settings change. + +A step costs the scan of all files and a check of each row's cell at the +trigger column. The state of the rows it submits reads every record under the +batch's label, though it works out the status of those rows only. """ from __future__ import annotations @@ -61,16 +65,21 @@ def __init__( ) -> None: if at not in batch.columns: raise KeyError(f'{at!r} is not a column of the batch') + if batch.source is None: + raise ValueError('this batch has no source of rows') self.batch = batch self.at = at self.settings = settings - self.refused: dict[str, str] = {} - """Why each row with a new file at ``at`` did not run, by key.""" self._cursor = 0 self._held: dict[str, set[FileRef]] = {} - self._rows: dict[str, Row] = {} + self._refused: dict[str, tuple[Row, str]] = {} self._settings: Mapping[str, Any] | None = None + @property + def refused(self) -> dict[str, str]: + """Why each row with a new file at ``at`` did not run, by key.""" + return {key: why for key, (_, why) in self._refused.items()} + def step(self) -> dict[str, Record]: """ Scan, and reduce the rows with a new file at ``at``; their records by @@ -79,30 +88,29 @@ def step(self) -> dict[str, Record]: """ batch = self.batch changes = batch.client.changes(after=self._cursor) - self._cursor = changes.cursor - touched: set[str] = set() for record in batch.client.records(ids=changes.records): if record.label == batch.label and record.member is not None: held = self._held.setdefault(record.member, set()) held |= _files_at(record.request.params, self.at) - touched.add(record.member) - batch.scan() - rows = {row.key: row for row in batch.rows} + self._refused.pop(record.member, None) + self._cursor = changes.cursor settings = self.settings() - look_at = {k for k, row in rows.items() if self._rows.get(k) != row} - look_at |= touched & rows.keys() if settings != self._settings: - look_at |= self.refused.keys() & rows.keys() - # A copy, so that settings changed in place count as changed. - self._rows, self._settings = rows, copy.deepcopy(settings) - self.refused = { - k: why for k, why in self.refused.items() if k in rows and k not in look_at + self._refused.clear() + # A copy, so that settings changed in place count as changed. + self._settings = copy.deepcopy(settings) + batch.scan() + rows = batch.rows + self._refused = { + key: (row, why) + for key, (row, why) in self._refused.items() + if key in rows and rows[key] == row } todo = [ - k - for k in rows # in the order of the rows - if k in look_at - and _files_at(rows[k].values, self.at) - self._held.get(k, set()) + row.key + for row in rows + if row.key not in self._refused + and _new(row.values.get(self.at), self._held.get(row.key, set())) ] return self._reduce(settings, todo) @@ -116,14 +124,18 @@ def _reduce( except SubmitError as error: # noqa: PERF203 named = set(keys).intersection(e.entry for e in error.errors) if not named: - self.refused |= dict.fromkeys(keys, str(error)) + self._refuse(keys, str(error)) return {} for key in named: errors = [e for e in error.errors if e.entry == key] - self.refused[key] = str(SubmitError(errors)) + self._refuse([key], str(SubmitError(errors))) keys = [k for k in keys if k not in named] return {} + def _refuse(self, keys: list[str], why: str) -> None: + rows = self.batch.rows + self._refused |= {key: (rows[key], why) for key in keys} + def run(autos: Sequence[Automatic], interval: float = 1.0) -> None: """ @@ -136,8 +148,20 @@ def run(autos: Sequence[Automatic], interval: float = 1.0) -> None: time.sleep(interval) +def _new(cell: Any, held: set[FileRef]) -> bool: + """Whether ``cell`` is a file, or a list with a file, that ``held`` lacks.""" + if isinstance(cell, FileRef): + return cell not in held + if isinstance(cell, list): + return any(isinstance(f, FileRef) and f not in held for f in cell) + return False + + def _files_at(params: Mapping[str, Any], at: str) -> set[FileRef]: - """The files at the parameter ``at``: its value, or each element of its list.""" + """ + The files at the parameter ``at`` of a request: its value, or each element + of its list, as a file or its plain form. + """ return { ref for loc, ref in iter_refs({at: params.get(at)}) diff --git a/packages/essapps/src/ess/apps/batch/reduction.py b/packages/essapps/src/ess/apps/batch/reduction.py index d50bd408..1a11fdd0 100644 --- a/packages/essapps/src/ess/apps/batch/reduction.py +++ b/packages/essapps/src/ess/apps/batch/reduction.py @@ -156,7 +156,7 @@ def state( """ rows = list(self.rows) if keys is None else [self.rows[k] for k in keys] found = self._results(keys) - upstream = self._records_read(rows) + upstream = self._known(sorted(state.references(rows))) status = self._status([*found.values(), *upstream]) stale = set() if settings is None else self._stale(settings, rows) return state.row_states( @@ -177,13 +177,18 @@ def _results(self, keys: Sequence[str] | None = None) -> dict[str, Record]: records = [r for r in records if r.member in chosen] return state.results(records, SpecId.of(self.spec)) - def _records_read(self, rows: list[Row]) -> list[Record]: - """The records that ``rows`` reference, if the client knows them all.""" - ids = sorted(state.references(rows)) + def _known(self, ids: list[str]) -> list[Record]: + """ + The records of ``ids`` that the client knows. If one is unknown, the + others are read one by one, so that it touches only the rows that + reference it, whose submission it refuses. + """ try: return self.client.records(ids=ids) if ids else [] - except KeyError: # the workspace refuses such a cell at submission - return [] + except KeyError: + if len(ids) == 1: + return [] + return [r for i in ids for r in self._known([i])] def _status(self, records: list[Record]) -> dict[str, Status]: statuses = self.client.status(records) diff --git a/packages/essapps/src/ess/apps/batch/table.py b/packages/essapps/src/ess/apps/batch/table.py index db2c6994..4fdb9248 100644 --- a/packages/essapps/src/ess/apps/batch/table.py +++ b/packages/essapps/src/ess/apps/batch/table.py @@ -72,9 +72,10 @@ def _rows(self) -> dict[str, Row]: return {row.key: self._with_edits(row) for row in (*shown, *self.added)} def _with_edits(self, row: Row) -> Row: - if (edits := self.edits.get(row.key)) is None: + edits = self.edits.get(row.key) + if edits is None and not any(v is EMPTY for v in row.values.values()): return row - cells = {**row.values, **edits} + cells = {**row.values, **(edits or {})} return replace( row, values={column: v for column, v in cells.items() if v is not EMPTY} ) diff --git a/packages/essapps/tests/batch/auto_test.py b/packages/essapps/tests/batch/auto_test.py index 73b4eb0b..01d640bd 100644 --- a/packages/essapps/tests/batch/auto_test.py +++ b/packages/essapps/tests/batch/auto_test.py @@ -11,9 +11,19 @@ from ess.apps import Client from ess.apps.batch import Batch, FileInfo, Row from ess.apps.batch.auto import Automatic -from ess.apps.testing import FakeFiles - -from .toys import COLUMNS, FIND, PLAIN, SERIES, STAGED, Gate, Measure, pair +from ess.apps.testing import CountingClient, FakeFiles + +from .toys import ( + COLUMNS, + FIND, + GROUPED, + PLAIN, + SERIES, + STAGED, + Gate, + Measure, + pair, +) SETTINGS: dict[str, Any] = {'binning': {'bins': 2}} @@ -56,6 +66,13 @@ def test_the_trigger_column_must_be_a_column(batch: Batch) -> None: Automatic(batch, at='offset', settings=dict) +def test_the_batch_must_have_a_source(client: Client) -> None: + batch = Batch(client, PLAIN, columns=COLUMNS, label='scan') + + with pytest.raises(ValueError, match='no source'): + Automatic(batch, at='sample', settings=dict) + + def test_a_row_whose_record_failed_is_not_reduced_again( auto: Automatic, client: Client, sample: Measure, fake: FakeFiles ) -> None: @@ -232,3 +249,161 @@ def with_offset(files: Sequence[FileInfo]) -> list[Row]: assert reduced['1'].request.params['offset'] == found['1'].ref('offset') assert client.wait(reduced['1']) == 'completed' assert client.output(reduced['1'], 'total') == 2.0 + + +class Flaky: + """A client whose next ``fail`` submissions raise, as a lost connection does.""" + + def __init__(self, client: Client) -> None: + self._client = client + self.fail = 0 + + def __getattr__(self, name: str) -> Any: + return getattr(self._client, name) + + def submit(self, *args: Any, **kwargs: Any) -> Any: + if self.fail: + self.fail -= 1 + raise ConnectionError('lost') + return self._client.submit(*args, **kwargs) + + +def test_a_row_whose_submission_raised_runs_at_the_next_step( + client: Client, sample: Measure +) -> None: + flaky = Flaky(client) + batch = Batch( + flaky, # type: ignore[arg-type] + PLAIN, + columns=COLUMNS, + label='scan', + source=pair, + ) + auto = Automatic(batch, at='sample', settings=lambda: SETTINGS) + sample(1, 'water') + flaky.fail = 1 + with pytest.raises(ConnectionError): + auto.step() + + assert list(auto.step()) == ['1'] + assert auto.step() == {} + + +class Raising: + """Calls ``function``, or raises while ``on`` is set.""" + + def __init__(self, function: Any) -> None: + self.function = function + self.on = False + + def __call__(self, *args: Any) -> Any: + if self.on: + raise RuntimeError('unreadable') + return self.function(*args) + + +@pytest.mark.parametrize('raising', ['settings', 'source']) +def test_a_step_that_raises_as_a_running_record_finishes_loses_nothing( + client: Client, fake: FakeFiles, gate: Gate, raising: str +) -> None: + settings, source = Raising(dict), Raising(_angles) + batch = Batch(client, SERIES, columns=('runs',), label='series', source=source) + auto = Automatic(batch, at='runs', settings=settings) + gate.close() + fake.measure(1, 1.0, sample='si') + first = auto.step() + fake.measure(2, 2.0, sample='si') + assert auto.step() == {} # still running + gate.open() + client.wait(first) + + (settings if raising == 'settings' else source).on = True + with pytest.raises(RuntimeError): + auto.step() + settings.on = source.on = False + + (record,) = auto.step().values() + assert len(record.request.params['runs']) == 2 + + +def test_a_record_holding_the_file_at_another_parameter_does_not_count( + auto: Automatic, client: Client, sample: Measure, fake: FakeFiles +) -> None: + run = sample(1, 'water') + other = fake.measure(5000, 1.0, role='transmission', sample='other') + params = {'sample': other, 'transmission': run, **SETTINGS} + client.wait(client.submit(PLAIN, params=params, label='scan', member='1')) + + assert list(auto.step()) == ['1'] + + +def test_a_file_inside_a_group_at_the_trigger_column_does_not_count( + client: Client, fake: FakeFiles +) -> None: + def grouped(files: Sequence[FileInfo]) -> list[Row]: + return [Row(str(f.metadata['run']), {'group': {'run': f.ref}}) for f in files] + + batch = Batch(client, GROUPED, columns=('group',), label='g', source=grouped) + auto = Automatic(batch, at='group', settings=dict) + fake.measure(1, 1.0) + + assert auto.step() == {} + assert auto.refused == {} + + +def test_a_record_of_another_spec_with_the_rows_key_counts( + auto: Automatic, client: Client, sample: Measure +) -> None: + run = sample(1, 'water') + client.wait(client.submit(FIND, params={'sample': run}, label='scan', member='1')) + + assert auto.step() == {} + + +def test_settings_changed_in_place_count_as_changed( + batch: Batch, sample: Measure +) -> None: + settings = {'binning': {'bins': 0}} + auto = Automatic(batch, at='sample', settings=lambda: settings) + sample(1, 'water') + assert auto.step() == {} + + settings['binning']['bins'] = 2 + + assert list(auto.step()) == ['1'] + + +def test_an_idle_step_reads_no_records(client: Client, sample: Measure) -> None: + counting = CountingClient(client) + batch = Batch( + counting, # type: ignore[arg-type] + PLAIN, + columns=COLUMNS, + label='scan', + source=pair, + ) + auto = Automatic(batch, at='sample', settings=lambda: SETTINGS) + sample(1, 'water') + client.wait(auto.step()) + auto.step() # reads the record that finished + counting.reads.clear() + + assert auto.step() == {} + assert sum(counting.reads.values()) == 0 + + +def test_two_loops_on_one_label_submit_a_file_once( + client: Client, sample: Measure +) -> None: + first, second = ( + Automatic( + Batch(client, PLAIN, columns=COLUMNS, label='scan', source=pair), + at='sample', + settings=lambda: SETTINGS, + ) + for _ in range(2) + ) + sample(1, 'water') + + assert list(first.step()) == ['1'] + assert second.step() == {} diff --git a/packages/essapps/tests/batch/conftest.py b/packages/essapps/tests/batch/conftest.py index 9a00adbe..2ed61067 100644 --- a/packages/essapps/tests/batch/conftest.py +++ b/packages/essapps/tests/batch/conftest.py @@ -66,8 +66,13 @@ def workspace( plain = function_binding(toys.PLAIN, staged_binding.function) find = function_binding(toys.FIND, _find) series = function_binding(toys.SERIES, _series(gate)) + grouped = function_binding( + toys.GROUPED, lambda p: toys.Outputs(total=read_file(p.group.run)) + ) workspace = Workspace( - proposal='p1', files=files, bind=[staged_binding, plain, find, series] + proposal='p1', + files=files, + bind=[staged_binding, plain, find, series, grouped], ) yield workspace workspace.close() diff --git a/packages/essapps/tests/batch/reduction_test.py b/packages/essapps/tests/batch/reduction_test.py index f9d1f406..b4446e79 100644 --- a/packages/essapps/tests/batch/reduction_test.py +++ b/packages/essapps/tests/batch/reduction_test.py @@ -10,7 +10,7 @@ from ess.apps import Client, Status, SubmitError, file, local from ess.apps.batch import Batch, CorrectedFiles, FileInfo, Row, ready, remaining from ess.apps.testing import CountingClient, FakeStore, StagedFunction -from ess.spec import function_binding +from ess.spec import OutputRef, function_binding from .toys import COLUMNS, FIND, PLAIN, STAGED, Gate, Measure, Outputs, pair @@ -44,7 +44,7 @@ def test_rows_are_reduced_and_found_by_their_key( assert state['1'].status is Status.COMPLETED -def test_the_state_of_chosen_rows_reads_only_their_records( +def test_the_state_of_chosen_rows_asks_the_status_of_their_records_only( client: Client, sample: Measure ) -> None: counting = CountingClient(client) @@ -349,6 +349,23 @@ def test_a_cell_reading_a_failed_record_stops_only_its_row( assert ready(reduce.state()) == ['1'] +def test_an_unknown_record_in_a_cell_touches_only_its_row( + chain: tuple[Batch, Batch], client: Client +) -> None: + _, reduce = chain + failed = reduce.reduce({'binning': {'bins': 1}}, ['1']) + client.wait(failed) + reduce.rows = reduce.rows.edit( + '1', {'offset': OutputRef(record='nope', output='x')} + ) + reduce.rows = reduce.rows.edit('2', {'offset': failed['1'].ref('total')}) + + whole, chosen = reduce.state()['2'], reduce.state(keys=['2'])['2'] + + assert [e.msg for e in whole.problems] == ['the record it reads is failed'] + assert chosen.problems == whole.problems + + def test_a_record_of_another_submitter_is_a_rows_result( scanned: Batch, client: Client, other: Client ) -> None: diff --git a/packages/essapps/tests/batch/table_test.py b/packages/essapps/tests/batch/table_test.py index d5cdfd61..1273a5a5 100644 --- a/packages/essapps/tests/batch/table_test.py +++ b/packages/essapps/tests/batch/table_test.py @@ -135,3 +135,10 @@ def test_a_row_added_takes_no_edit_left_from_a_row_gone() -> None: orphan = RowTable().rescan([Row('a', {'x': 1})]).edit('a', {'x': 9}).rescan([]) assert orphan.add(Row('a', {'x': 5, 'y': 7}))['a'].values == {'x': 5, 'y': 7} + + +def test_a_row_given_with_empty_has_that_cell_empty() -> None: + t = RowTable().rescan([Row('a', {'x': EMPTY, 'y': 1})]).add(Row('b', {'x': EMPTY})) + + assert t['a'].values == {'y': 1} + assert t['b'].values == {} diff --git a/packages/essapps/tests/batch/toys.py b/packages/essapps/tests/batch/toys.py index 793a5f8a..4a7ed385 100644 --- a/packages/essapps/tests/batch/toys.py +++ b/packages/essapps/tests/batch/toys.py @@ -46,6 +46,14 @@ class SeriesParams(BaseModel): runs: list[RunFile] = Field(min_length=1) +class Group(BaseModel): + run: RunFile + + +class GroupedParams(BaseModel): + group: Group + + STAGED = WorkflowSpec( name='staged', version=1, @@ -79,6 +87,14 @@ class SeriesParams(BaseModel): outputs=Outputs, tables=('runs',), ) +GROUPED = WorkflowSpec( + name='grouped', + version=1, + title='Grouped', + description='A toy whose run is inside a group.', + params=GroupedParams, + outputs=Outputs, +) COLUMNS = ('sample', 'transmission', 'thickness') From 3fe3610bb228b6faec14b950902b2cbc86d2589c Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 13 Nov 2026 12:07:27 +0000 Subject: [PATCH 5/7] perf: the loop compares files by their identity strings Two file references are equal only when their identity strings are, so the files held at the trigger column are kept as those strings. Checking a row then hashes a string instead of a pydantic model. Co-Authored-By: Claude Opus 5.5 --- packages/essapps/src/ess/apps/batch/auto.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/essapps/src/ess/apps/batch/auto.py b/packages/essapps/src/ess/apps/batch/auto.py index 6c2d3fca..7b83945a 100644 --- a/packages/essapps/src/ess/apps/batch/auto.py +++ b/packages/essapps/src/ess/apps/batch/auto.py @@ -71,7 +71,8 @@ def __init__( self.at = at self.settings = settings self._cursor = 0 - self._held: dict[str, set[FileRef]] = {} + # The identity strings, which are equal only if their files are. + self._held: dict[str, set[str]] = {} self._refused: dict[str, tuple[Row, str]] = {} self._settings: Mapping[str, Any] | None = None @@ -148,22 +149,22 @@ def run(autos: Sequence[Automatic], interval: float = 1.0) -> None: time.sleep(interval) -def _new(cell: Any, held: set[FileRef]) -> bool: +def _new(cell: Any, held: set[str]) -> bool: """Whether ``cell`` is a file, or a list with a file, that ``held`` lacks.""" if isinstance(cell, FileRef): - return cell not in held + return cell.file not in held if isinstance(cell, list): - return any(isinstance(f, FileRef) and f not in held for f in cell) + return any(isinstance(f, FileRef) and f.file not in held for f in cell) return False -def _files_at(params: Mapping[str, Any], at: str) -> set[FileRef]: +def _files_at(params: Mapping[str, Any], at: str) -> set[str]: """ - The files at the parameter ``at`` of a request: its value, or each element - of its list, as a file or its plain form. + The identities of the files at the parameter ``at`` of a request: its + value, or each element of its list, as a file or its plain form. """ return { - ref + ref.file for loc, ref in iter_refs({at: params.get(at)}) if isinstance(ref, FileRef) and (len(loc) == 1 or (len(loc) == 2 and isinstance(loc[1], int))) From 76399f907f75b60d44e5c5259a48365e34f8650b Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 13 Nov 2026 12:09:50 +0000 Subject: [PATCH 6/7] docs: the cost of a step, and what is open in automatic reduction automatic-reduction.md states the scale that the cost is measured against, how the interval of `run` sets the load, why the history is not cut off by age or count, and what a step keeps now that it checks every row. It no longer says that the state of chosen rows reads only their records. Open gains: reading the records of chosen members, a source that makes a row from one file by queries on an index of the files, the files held at the trigger column for a page, and a trigger cell that names a file by run number. The batch guide calls an `Automatic` the loop, keeps "automatic reduction" for the activity, and says that which rows are reduced is defined by the records under the label only, so a new label reduces from the first file and a row whose key changes is reduced again. Co-Authored-By: Claude Opus 5.5 --- .../0007-packages-split-by-dependencies.md | 2 +- docs/developer/automatic-reduction.md | 96 ++++++++++++------- packages/essapps/docs/batch.md | 28 +++--- 3 files changed, 80 insertions(+), 46 deletions(-) diff --git a/docs/developer/adr/0007-packages-split-by-dependencies.md b/docs/developer/adr/0007-packages-split-by-dependencies.md index d25d41f9..92faed14 100644 --- a/docs/developer/adr/0007-packages-split-by-dependencies.md +++ b/docs/developer/adr/0007-packages-split-by-dependencies.md @@ -78,7 +78,7 @@ essspec <- essapps <- frontends (over HTTP) - **Names follow the ess monorepo**: distribution `essX`, package `ess.X`, one level under `ess`. A second distribution cannot install into a regular package (one with an `__init__.py`) such as `ess.apps`, and a namespace package (one without, which may span distributions) could not export `local` and the rest. So `ess.apps.panel` exists only as part of the `essapps` distribution. - **`essspec` (`ess.spec`) is everything a workflow author writes against.** The vocabulary (`WorkflowSpec`, references, the functions that read field types, parameter models), the binding protocol (`Binding`, `StageBinding`, `AccumulatorBinding`, `Driver`), and the bindings `DriverBinding` and `function_binding` need only pydantic. A client imports `ess.spec` for the specs, and the bindings cost it nothing more. `ess.spec.conversions` needs scipp (extra `[scipp]`), and `ess.spec.pipeline` with `PipelineBinding` needs sciline (extra `[sciline]`). It is developed in this repository while it changes, and nothing outside this repository depends on a version of it. Once a second consumer, such as esslivedata or essnmx, is to adopt it, it moves to scipp/ess as a distribution of its own and is published from there. - **`essapps` (`ess.apps`) is what notebooks and apps use**: records, the client, the workspace and its log, files, the batch toolbox `ess.apps.batch` with its loop of automatic reduction, and fakes for tests. The core depends on essspec and pydantic, not on essreduce or sciline. The subpackages behind extras change in step with the core, so they are released with it. Importing one without its extra raises an `ImportError` that names the extra. A local workspace serves a web frontend on VISA or a laptop through `ess.apps.http.server`, as the service does. The loop of automatic reduction is in `ess.apps.batch`, and the source of rows in the application that knows the technique; `ess.apps.service` runs the loop as a daemon from a configuration file. -- **`essinstruments` (`ess.instruments`) holds one subpackage per instrument, and is released on its own schedule.** A fix to LoKI's specs ships without whatever changed in the core since its last release, and a core release ships without unreleased instrument changes. It depends on essspec, not on essapps. Each instrument has two modules: `specs.py` with the specs, and `bindings.py` with their bindings, which need the instrument's workflow package, behind an extra such as `[loki]`, so that an install for LoKI does not install the workflow packages of every other instrument. The two are separate because a spec may move into its workflow package without its binding, and a type checker checks the binding against the spec. `ess.instruments.loki` holds the specs and bindings of LoKI's reductions; `ess.loki`, from esssans, holds the workflow they bind to. A spec can move into its workflow package only after essspec has moved to scipp/ess, and only once its binding is a plain `PipelineBinding`, with no facility defaults or adapters. Deployment configuration, such as which automatic-reduction rules run and data paths, is not here; it belongs to the service's configuration. +- **`essinstruments` (`ess.instruments`) holds one subpackage per instrument, and is released on its own schedule.** A fix to LoKI's specs ships without whatever changed in the core since its last release, and a core release ships without unreleased instrument changes. It depends on essspec, not on essapps. Each instrument has two modules: `specs.py` with the specs, and `bindings.py` with their bindings, which need the instrument's workflow package, behind an extra such as `[loki]`, so that an install for LoKI does not install the workflow packages of every other instrument. The two are separate because a spec may move into its workflow package without its binding, and a type checker checks the binding against the spec. `ess.instruments.loki` holds the specs and bindings of LoKI's reductions; `ess.loki`, from esssans, holds the workflow they bind to. A spec can move into its workflow package only after essspec has moved to scipp/ess, and only once its binding is a plain `PipelineBinding`, with no facility defaults or adapters. Deployment configuration, such as which batches automatic reduction runs and data paths, is not here; it belongs to the service's configuration. - **A Python client that imports spec modules installs the instrument's workflow package.** A local workspace, a notebook, and a Panel app on VISA or a laptop run the bindings, so they have it anyway. A notebook against the service wants scipp and plotting for the outputs, so it runs in an ESS environment that has the workflow packages too. A client without workflow packages, such as a web frontend or the command-line tool with `--url`, reads `SerializedWorkflowSpec`: the JSON form of a spec, which names each data type as a string. A spec module imports scipp for the types of data: an array is a field of a group, typed by its scipp class, such as `intensity: sc.DataArray` of the group `IofQ`, so that a type checker checks a binding against the spec, and the workspace checks by the group's class that an output fits a field typed `Referenced[IofQ]`. - **`essapps` finds bindings through the entry-point group `ess.apps.workflows`.** An entry point is a name that a distribution registers under a group in its `pyproject.toml` (see the example below); `importlib.metadata.entry_points(group=...)` lists the entries of all installed distributions. Each entry names a function that returns a list of bindings, each of which names its spec. `local()` without `bind=`, the command-line tool, and the service read the group, so a distribution registers its workflows without importing the framework. Listing the specs this way imports the bindings; a client of the service gets the list from the service instead. An entry registers whether or not its extra is installed: an entry whose workflow package is missing is skipped, and `essapps specs` lists its specs as unavailable, naming the extra; any other import error fails the load. Two entries that register the same spec name and version fail the load, naming both entries, since picking one would leave records ambiguous about what ran. This is decided and not implemented. - **The command-line tool is a client in `essapps`**: `ess.apps.cli`, with the script `essapps`. It builds its options from `SerializedWorkflowSpec`, so it runs any registered spec in its own process, and with `--url` any spec of the service without importing a workflow package. Each parameter is one option, `--`. A file field takes a file's identity or a path, `--sample run:loki/4711`. A group takes its fields as JSON typed from the params schema. A union of groups takes one alternative's fields, with its tag, as JSON typed from the params schema. A parameter `Referenced[T]` takes a record's output, such as `--centre r17.centre`. A union with a member `Referenced[...]` takes either, told apart by the argument: a JSON object, a file, or a record's output. A list parameter that is not a table takes a JSON list of its elements, typed from the params schema. A table takes a JSON or CSV file with one row per line. The tool needs only argparse, and `--url` needs `[remote]`. Writing outputs to files uses `ess.apps.writers`, which the service uses too ([ADR 0002](0002-a-value-lives-while-something-keeps-it.md)). A tool with instrument-specific arguments, such as essnmx's reducer, stays in its workflow package. diff --git a/docs/developer/automatic-reduction.md b/docs/developer/automatic-reduction.md index 67806b23..13630172 100644 --- a/docs/developer/automatic-reduction.md +++ b/docs/developer/automatic-reduction.md @@ -7,7 +7,7 @@ Its stories are E1 to E4 in [user-stories.md](user-stories.md), and [deployments The framework knows no technique: which runs make one reduction is the application's to say. It says so once, as the *source* of a batch, a function from files to rows. -A person reducing a batch and the automatic reduction use the same source. +A person reducing a batch and automatic reduction use the same source. The batch is a chain of tables, of which only the files grow on their own: @@ -20,13 +20,13 @@ todo = [key for key in state if runs(key)] # the predicate of the next s submit(todo) # appends to the history, so results change ``` -The automatic reduction is this chain stepped by a loop, `ess.apps.batch.auto.Automatic`: +Automatic reduction steps this chain with a *loop*, an `ess.apps.batch.auto.Automatic`: ```python batch = Batch(client, IOFQ, columns=('sample', 'sample_transmission'), label='auto/iofq', source=pair_by_sample, staged=True, persist=True) -auto = Automatic(batch, at='sample', settings=lambda: load_settings('iofq.yaml')) -run([auto]) # steps forever: scan, then reduce the rows with a new file at `sample` +loop = Automatic(batch, at='sample', settings=lambda: load_settings('iofq.yaml')) +run([loop]) # steps forever: scan, then reduce the rows with a new file at `sample` ``` `settings` is called at each step, so a setting changed during a beamtime applies to the rows not reduced yet. @@ -59,28 +59,26 @@ Each line answers a story or a failure mode: - A later partner, such as a transmission run that the source pairs with rows made before, changes a cell at another column, and does not reduce those rows again. - A series grows (E1): a new angle in a row's list reduces the row again with every angle so far, and an angle whose file arrives again keeps its identity and is not new. -The records are read whatever their spec version or submitter. So a new version of the spec, or a restart under another name, reduces no old row again. +The records are read whatever their spec or submitter. So a new version of the spec, or a restart under another name, reduces no old row again. A label is what a consumer group is to a stream: the records under it say which files are consumed, and nothing else does. -A new label starts from the first file, and a person's batch under its own label does not stop the loop from reducing a row; a record that another client submits under the loop's label, with a row's key as its member, does. +A loop on a new label starts from the first file. A person's batch under its own label does not stop the loop from reducing a row; a record that another client submits under the loop's label, with a row's key as its member, does. A row that is running waits: the loop submits it again, with its new file, once its record has finished. ## What the loop keeps -Which files the records of each key hold is read from the proposal's changes (`client.changes`), from the start when the loop starts. +Which files the records of each key hold, `held` above, is read from the proposal's changes (`client.changes`), from the start when the loop is made. So a restarted loop reduces no row again, and keeps nothing on disk of its own. This holds while the proposal's history is kept. A running loop keeps it, since its calls keep the proposal from being idle ([`ess.apps.protocol`](../../packages/essapps/src/ess/apps/protocol.py)). -Besides, the loop keeps the rows and the settings of its last step, which spare it work only. -A step looks at the rows whose cells changed, those with a record that changed, and, if the settings changed, those that were refused: +Each step checks every row against `held`, so a step that raises, in the changes, the settings, the source, or a submission, loses nothing: the next step tries again. +Besides `held`, the loop keeps the settings of the last step and the rows it could not submit, each as it was then, so that it does not try them at every step: -| A row waits because | It is looked at again when | +| A row did not run because | It is tried again when | |---|---| | a cell is empty, such as a missing transmission run | its cells change | | its result is running | its record changes | -| its request was refused, such as an invalid value | its cells or the settings change | - -The batch works out the state of those rows only (`Batch.state(settings, keys)`). +| its request was refused, such as an invalid value | its cells, its records, or the settings change | ## Refusals @@ -92,7 +90,7 @@ A reason that names no row is about the settings, such as a spec the workspace n ## Chains A row may reference the output of a row of another batch, as a sample reads the transmission computed from its transmission run. -The second batch's source reads the first batch's state, and the loop steps the first batch before the second: +The second batch's source reads the first batch's state, and `run` steps the loop of the first batch before that of the second: ```python transmissions = Batch(client, TRANSMISSION, columns=('run',), label='auto/transmission', ...) @@ -111,35 +109,69 @@ In the step in which the transmission is submitted, the sample's row references If the transmission's record has failed by the time the sample's row is made, the row waits: a cell reads a failed record. If it fails after, the sample's record fails with it and holds the sample's file, so the row does not run again. A source is recomputed at every step, so a chain survives a restart, which a reference set by hand in a cell (an edit) does not. +The source reads the state of every row of the first batch at every step. ## Cost -A step lists every file, runs the source over all of them, and compares the rows with those of the last step. -That is linear in the number of files, about 4 µs per file and 3 µs more to list it in the fake file source. -The state and the submission cost only the rows looked at. -A loop that starts looks at every row once. +Some instruments in full operation are expected to write 10 000 to 100 000 files per proposal, and a file every few seconds. +The numbers below are measured against that scale. + +A step lists every file, runs the source over all of them, and checks each row's cell at `at` against `held`. +That is linear in the number of files. +To submit the rows that run, the batch reads every record under the label to find the result of each of those rows, and asks for the status of those rows only. +A loop that is made reads every record of the proposal once. -| Runs reduced | 3 000 | 30 000 | 100 000 | +| Files reduced | 3 000 | 30 000 | 100 000 | |---|---|---|---| -| a step with one new file, best of ten | 12 ms | 0.19 s | 0.74 s | -| of which listing the files | 5 ms | 75 ms | 0.33 s | -| mean of ten | 25 ms | 0.39 s | 1.4 s | +| a step with one new file, best of ten | 15 ms | 0.21 s | 0.82 s | +| of which listing the files | 5 ms | 82 ms | 0.32 s | +| mean of ten | 30 ms | 0.40 s | 1.4 s | -Measured with a toy workflow, the fake file source, and a history in memory. +Measured with a toy workflow, the fake file source, and a history in memory, on a machine shared with other work. The mean is about twice the best because of Python's garbage collector: each step makes a row for each file, and the collector then walks every object of the process, the records among them. -## Open +A step costs the same for one new file or fifty, so the interval of `run` sets the load on one core, `step / (step + interval)`. +With a step of about 1 s at 100 000 files, that is 50 % at an interval of 1 s, 9 % at 10 s, and 3 % at 30 s. +The price is that a new file waits up to one interval. +An interval that follows the step time, such as `max(1 s, 10 × the last step)`, would keep the load near 10 % at any number of files; it is not built. -- **New files since a position.** `FileSource.list` returns every file, so each step lists the whole proposal. A catalogue cannot be listed whole at each step; the file source needs a call that returns what is new since a position, as `client.changes` does for records. -- **A step below linear.** The source runs over all files because nothing says which rows a new file can touch. An optional group per file would say it: a step would then make the rows of the groups that got a file, from those files and the files shared by all groups. +## Why the history is not cut off + +A cut-off by age or by count, of the files given to the source, would bound the cost of a step, but it changes the results: - ```python - Batch(..., source=pair_by_sample, - group=lambda f: None if f.metadata['role'] == 'can' else f.metadata['sample']) # None: shared - ``` +- A partner older than the cut-off, such as a can measured at the start, is gone, and its row waits for ever. +- A sample measured again after the cut-off gets a row with its later runs only, which is reduced with those, and nothing shows it. - A source that pairs by time, such as "the can measured before", has no such group and stays linear. -- **Starting a new label at a given run.** A new label consumes from the first file. Starting it at a run needs a range in `Selector`, which matches equal values only. +A cut-off that a person sets on purpose, "only the files since Tuesday", is a selector on the files given to the source. + +## Open + +- **New files since a position.** `FileSource.list` returns every file, so each step lists the whole proposal. A catalogue cannot be listed whole at each step; the file source needs a call that returns what is new since a position, as `client.changes` does for records. +- **Reading the records of chosen members.** The state of the rows that run reads every record under the label (`client.records(label=...)`) and keeps those of the rows' keys. A remote client cannot read 100 000 records at each step. +- **A step below linear.** The source runs over all files because nothing says which rows a new file can touch. Two ways would say it; neither is prototyped, and a spike on LoKI's pairing and on a pairing by time would show which is better: + - *A group per file*, declared by the author. A step makes the rows of the groups that got a file, from those files and the files shared by all groups. A source that pairs by time, such as "the can measured before", has no such group and stays linear. + + ```python + Batch(..., source=pair_by_sample, + group=lambda f: None if f.metadata['role'] == 'can' else f.metadata['sample']) # None: shared + ``` + + - *A row from one file, by queries on an index of the files.* The framework records what each row asked for. It needs no declaration by the author, and changes the signature of every source. + + ```python + def row_of(file: FileInfo, files: FileIndex) -> Row | None: + if file.metadata.get('role') != 'sample': + return None + partners = files.list(Selector(role='transmission', sample=file.metadata['sample'])) + ... + # the framework records that this row asked for (role=transmission, sample=water): + # a new file with that metadata makes this row again, and no other + ``` + + A class with state that the author writes, `add(file) -> changed rows`, is not offered. It would have to give the same rows for any order of arrival, a role corrected later, a file that arrives again, and a restart, which a function over the files gives without effort. +- **A cell at `at` that names a file.** Files are compared by identity, as `FileInfo.ref` gives it. A source that writes `file(run=1)` at `at` makes a row that no record holds, since the record holds the identity, so the row runs at every step. Comparing them needs the file source's `resolve`, which a client cannot reach. +- **Starting a new label at a given run.** A loop on a new label consumes from the first file. Starting it at a run needs a range in `Selector`, which matches equal values only. - **Old rows reduced again for a new spec version.** One more term of the predicate, from the spec of each row's result (ADR 0009): `runs(row, held) or result.spec != spec`. +- **The files held at `at`, by key, for a page.** A page reads `Batch.state`, which counts the results of the batch's spec only. After a new version of the spec, a page shows the old rows as not reduced, while the loop holds them. - **A loop that pushes into an accumulator**, for a sum that grows with each file (D7). The accumulator would be the only thing the workspace keeps for the loop, which would freeze it with `persist=` once the sum is complete. - **The application around the loop**: one process per instrument, a workspace per proposal, the configuration files, and a page that shows the batches. The page and the loop would use one `Batch` from two threads, and `Batch` has no lock. diff --git a/packages/essapps/docs/batch.md b/packages/essapps/docs/batch.md index 075ebd89..700900e4 100644 --- a/packages/essapps/docs/batch.md +++ b/packages/essapps/docs/batch.md @@ -140,7 +140,7 @@ for key, record in centres.items(): After a centre failed, this reruns only what is left, and points those rows at the new centre. The field must be typed `Referenced[...]` in the spec. -An edit is lost when the session ends. A source that reads the other batch's state makes the same cells at every scan, which the automatic reduction needs: +An edit is lost when the session ends. A source that reads the other batch's state makes the same cells at every scan, which automatic reduction needs: ```python def with_centre(files: Sequence[FileInfo]) -> list[Row]: @@ -154,34 +154,36 @@ Scan `find` before a batch with this source, so that its rows see the newest cen ## Automatic reduction -`Automatic` reduces a batch as its files arrive. It needs a source, and the batch should persist what it submits, since nothing releases its records: +Automatic reduction reduces a batch as its files arrive. Its *loop*, an `Automatic`, needs a batch with a source, which should persist what it submits, since nothing releases its records: ```python from ess.apps.batch.auto import Automatic, run batch = Batch(client, IOFQ, columns=('sample', 'sample_transmission'), label='auto/iofq', source=pair_by_sample, staged=True, persist=True) -auto = Automatic(batch, at='sample', settings=lambda: settings) -auto.step() # scans, and reduces the rows with a new file at `sample`; their records by key -auto.refused # why each such row did not run, by key: {'60339': '60339: sample_transmission: empty'} -run([auto]) # steps forever, once a second +loop = Automatic(batch, at='sample', settings=lambda: settings) +loop.step() # scans, and reduces the rows with a new file at `sample`; their records by key +loop.refused # why each such row did not run, by key: {'60339': '60339: sample_transmission: empty'} +run([loop]) # steps forever, once a second ``` -`at` is the *trigger column*, of a file or a list of files. `settings` is called at each step, so a change applies to the rows not reduced yet. +`at` is the *trigger column*, of a file or a list of files, each as `FileInfo.ref` gives it. `settings` is called at each step, so a change applies to the rows not reduced yet. -**A row runs if its cell at `at` holds a file that no record under the label, with the row's key, holds at `at`.** Any record counts, failed, cancelled, of any version of the spec, and of any submitter. So the automatic reduction differs from a person's reduce: +**A row runs if its cell at `at` holds a file that no record under the label, with the row's key, holds at `at`.** Any record counts: failed, cancelled, of any spec, and of any submitter. So automatic reduction differs from a person's reduce: | A row runs if | no result | failed or cancelled | a setting or another cell changed | a new file at the trigger column | |---|---|---|---|---| | the page, "reduce remaining" | yes | yes | yes (stale) | yes | | `essapps-batch reduce --remaining` | yes | yes | no | no | -| automatic | yes | no | no | yes | +| the loop | yes | no | no | yes | -A row that cannot run, such as one without its transmission run, or whose request is refused, does not hold back the others, and `refused` says why. A reason that names no row, about the settings, refuses every row. A row that waits is looked at again when its cells change, a row still running when its record finishes, and a refused row also when the settings change. +Which rows are reduced is defined by the records under the label and by nothing else. So a loop on a batch with a new label reduces from the first file, and a person's record under the loop's label counts as a row reduced: give the loop a label of its own. A row whose key changes from one scan to the next, such as a key made from a sample name that a correction changed, is a new row, and is reduced again under its new key; so choose a key that does not change, such as the run number ([Terms](#terms)). -The loop reads the records from the proposal's changes, from the start when an `Automatic` is made, so a restart needs nothing on disk and reduces no row again. Give the automatic reduction its own label: a person's record under it counts as a row reduced. +A row that cannot run, such as one without its transmission run, or whose request is refused, does not hold back the others, and `refused` says why. A reason that names no row, about the settings, refuses every row. A row that waits is tried again when its cells change, a row still running when its record finishes, and a refused row also when its records or the settings change. -In a chain, step the batch that is read first: `run([Automatic(find, ...), Automatic(iofq, ...)])`. [automatic-reduction.md](../../../docs/developer/automatic-reduction.md) holds the design and what is open. +The loop reads the records from the proposal's changes, from the start when it is made, so a restart needs nothing on disk and reduces no row again. A step that raises, in the settings, the source, or a submission, loses nothing: the next step tries again. + +In a chain, step the loop of the batch that is read first: `run([Automatic(find, ...), Automatic(iofq, ...)])`. [automatic-reduction.md](../../../docs/developer/automatic-reduction.md) holds the design, its cost, and what is open. ## From files, and from the command line @@ -292,7 +294,7 @@ run,role,sample ,direct_beam, <- with a column file: a file that is no run ``` -`read_metadata` is a function of the instrument package, from the path of a file to its metadata, or `None` for a file that is not the instrument's (`ess.apps.folder.scan_folder`). The runs come first, by their number, since the order of the rows is the order of measurement. A file that cannot be read is left out, with a warning. Two files of one run are refused. Running `files` again after files gained a `pid` changes their identity: the automatic reduction reduces them again, and earlier records name files the folder no longer knows. +`read_metadata` is a function of the instrument package, from the path of a file to its metadata, or `None` for a file that is not the instrument's (`ess.apps.folder.scan_folder`). The runs come first, by their number, since the order of the rows is the order of measurement. A file that cannot be read is left out, with a warning. Two files of one run are refused. Running `files` again after files gained a `pid` changes their identity: the loop of automatic reduction reduces them again, and earlier records name files the folder no longer knows. A file says its kind, its run, and perhaps its sample, but not what a run is for. A row source that pairs runs reads the `role`, and a field such as the direct beam requires it. The run table gives it (`frames.described`): a row describes the file of its `run`, or the file at its `file` if it has no run. A run of the table that has no file in the folder is ignored, so the table may cover more than the folder holds. A file that no row describes keeps what it says of itself, and where both have a column, a cell of the run table that is not empty wins. Two rows for one run are refused. Without `--with`, the roles are typed into `files.csv`, so `files` then refuses to write over a table that exists. From 9c28338630d4ed7894d3b889372866a879679e54 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 13 Nov 2026 12:13:08 +0000 Subject: [PATCH 7/7] fix: the loop stops on a trigger cell that names a file A cell at the trigger column that names a file, such as `file(run=1)`, is resolved by the workspace, so its record holds the file's identity and never counts as holding the cell: the row was reduced at every step. A client has no call that resolves a file name, so the loop checks, for the rows it submitted, that each record holds the files of its row's cell, and raises `ValueError` naming the row if one does not. Every later step raises too, before it submits anything, until that cell changes. Co-Authored-By: Claude Opus 5.5 --- docs/developer/automatic-reduction.md | 2 +- packages/essapps/docs/batch.md | 2 +- packages/essapps/src/ess/apps/batch/auto.py | 53 ++++++++++++++++++++- packages/essapps/tests/batch/auto_test.py | 21 +++++++- 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/developer/automatic-reduction.md b/docs/developer/automatic-reduction.md index 13630172..8c0e4259 100644 --- a/docs/developer/automatic-reduction.md +++ b/docs/developer/automatic-reduction.md @@ -169,7 +169,7 @@ A cut-off that a person sets on purpose, "only the files since Tuesday", is a se ``` A class with state that the author writes, `add(file) -> changed rows`, is not offered. It would have to give the same rows for any order of arrival, a role corrected later, a file that arrives again, and a restart, which a function over the files gives without effort. -- **A cell at `at` that names a file.** Files are compared by identity, as `FileInfo.ref` gives it. A source that writes `file(run=1)` at `at` makes a row that no record holds, since the record holds the identity, so the row runs at every step. Comparing them needs the file source's `resolve`, which a client cannot reach. +- **A cell at `at` that names a file.** Files are compared by identity, as `FileInfo.ref` gives it, and a client has no call that resolves a file name such as `file(run=1)`. So the loop cannot accept such a cell: once a record does not hold the files of its row's cell, the loop refuses the cell by stopping, and raises at every step until the cell changes. - **Starting a new label at a given run.** A loop on a new label consumes from the first file. Starting it at a run needs a range in `Selector`, which matches equal values only. - **Old rows reduced again for a new spec version.** One more term of the predicate, from the spec of each row's result (ADR 0009): `runs(row, held) or result.spec != spec`. - **The files held at `at`, by key, for a page.** A page reads `Batch.state`, which counts the results of the batch's spec only. After a new version of the spec, a page shows the old rows as not reduced, while the loop holds them. diff --git a/packages/essapps/docs/batch.md b/packages/essapps/docs/batch.md index 700900e4..c8e57999 100644 --- a/packages/essapps/docs/batch.md +++ b/packages/essapps/docs/batch.md @@ -167,7 +167,7 @@ loop.refused # why each such row did not run, by key: {'60339': '60339: sam run([loop]) # steps forever, once a second ``` -`at` is the *trigger column*, of a file or a list of files, each as `FileInfo.ref` gives it. `settings` is called at each step, so a change applies to the rows not reduced yet. +`at` is the *trigger column*, of a file or a list of files, each as `FileInfo.ref` gives it. A cell there that names a file another way, such as `file(run=1)`, which the workspace resolves, would never count as held, and its row would be reduced at every step. So once the record of a row does not hold the files of its cell, `step` raises `ValueError`, naming the row, and so does every later step, before it submits anything, until the cell changes. `settings` is called at each step, so a change applies to the rows not reduced yet. **A row runs if its cell at `at` holds a file that no record under the label, with the row's key, holds at `at`.** Any record counts: failed, cancelled, of any spec, and of any submitter. So automatic reduction differs from a person's reduce: diff --git a/packages/essapps/src/ess/apps/batch/auto.py b/packages/essapps/src/ess/apps/batch/auto.py index 7b83945a..6f4d7b65 100644 --- a/packages/essapps/src/ess/apps/batch/auto.py +++ b/packages/essapps/src/ess/apps/batch/auto.py @@ -74,6 +74,8 @@ def __init__( # The identity strings, which are equal only if their files are. self._held: dict[str, set[str]] = {} self._refused: dict[str, tuple[Row, str]] = {} + # Rows whose record did not hold their cell at `at`, as they were then. + self._unheld: dict[str, tuple[Row, str]] = {} self._settings: Mapping[str, Any] | None = None @property @@ -86,6 +88,15 @@ def step(self) -> dict[str, Record]: Scan, and reduce the rows with a new file at ``at``; their records by key. A row that cannot run, or whose request is refused, does not hold back the others, and :attr:`refused` says why. + + Raises + ------ + ValueError + If the record of a row does not hold the files of the row's cell + at ``at``, such as a cell that names a file by ``file(run=...)``, + which the workspace resolves: the next step would submit the row + again, so the loop stops. Each later step raises too, before it + submits anything, until that cell changes. """ batch = self.batch changes = batch.client.changes(after=self._cursor) @@ -113,7 +124,28 @@ def step(self) -> dict[str, Record]: if row.key not in self._refused and _new(row.values.get(self.at), self._held.get(row.key, set())) ] - return self._reduce(settings, todo) + self._unheld = { + key: (row, why) + for key, (row, why) in self._unheld.items() + if key in rows and rows[key] == row + } + if stuck := [why for key, (_, why) in self._unheld.items() if key in todo]: + raise ValueError('\n'.join(stuck)) + records = self._reduce(settings, todo) + self._check(records) + return records + + def _check(self, records: Mapping[str, Record]) -> None: + """Raise if a record does not hold the files of its row's cell at ``at``.""" + rows = self.batch.rows + unheld = {} + for key, record in records.items(): + cell = _cell_files(rows[key].values.get(self.at)) + if missing := cell - _files_at(record.request.params, self.at): + unheld[key] = (rows[key], _unheld(key, self.at, missing)) + if unheld: + self._unheld |= unheld + raise ValueError('\n'.join(why for _, why in unheld.values())) def _reduce( self, settings: Mapping[str, Any], keys: list[str] @@ -158,6 +190,25 @@ def _new(cell: Any, held: set[str]) -> bool: return False +def _cell_files(cell: Any) -> set[str]: + """The identities of the files of a cell: a file, or a list of files.""" + if isinstance(cell, FileRef): + return {cell.file} + if isinstance(cell, list): + return {f.file for f in cell if isinstance(f, FileRef)} + return set() + + +def _unheld(key: str, at: str, missing: set[str]) -> str: + return ( + f'row {key!r}: its record does not hold {", ".join(sorted(missing))} at ' + f'{at!r}. A cell at the trigger column holds the identity of each file, ' + 'as FileInfo.ref gives it. A name such as file(run=1) is resolved by the ' + 'workspace, so no record would count as holding it, and the row would be ' + 'reduced at every step.' + ) + + def _files_at(params: Mapping[str, Any], at: str) -> set[str]: """ The identities of the files at the parameter ``at`` of a request: its diff --git a/packages/essapps/tests/batch/auto_test.py b/packages/essapps/tests/batch/auto_test.py index 01d640bd..67f5c7b6 100644 --- a/packages/essapps/tests/batch/auto_test.py +++ b/packages/essapps/tests/batch/auto_test.py @@ -8,7 +8,7 @@ import pytest -from ess.apps import Client +from ess.apps import Client, file from ess.apps.batch import Batch, FileInfo, Row from ess.apps.batch.auto import Automatic from ess.apps.testing import CountingClient, FakeFiles @@ -407,3 +407,22 @@ def test_two_loops_on_one_label_submit_a_file_once( assert list(first.step()) == ['1'] assert second.step() == {} + + +def test_a_cell_that_names_a_file_stops_the_loop_at_every_step( + client: Client, sample: Measure +) -> None: + def by_run(files: Sequence[FileInfo]) -> list[Row]: + return [ + replace(row, values={**row.values, 'sample': file(run=int(row.key))}) + for row in pair(files) + ] + + batch = Batch(client, PLAIN, columns=COLUMNS, label='scan', source=by_run) + auto = Automatic(batch, at='sample', settings=lambda: SETTINGS) + sample(1, 'water') + + for _ in range(2): + with pytest.raises(ValueError, match=r"^row '1': its record does not hold"): + auto.step() + assert len(client.records(label='scan')) == 1