From cf1665a71f19ea1e0fe62561e694edd836ec3214 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 16:40:53 +0000 Subject: [PATCH 1/9] feat(language)!: an entry or a sum may range over a set `relations:` is `sets:`. A set named in the `dims:` of a parameter, a variable or a constraint, or in `over=`, stands for its dimensions at the combinations it lists: the entry has coordinates only there, and a tool refuses a data row of a parameter anywhere else. There is no `in:` key. A set's bare name in the `where:` of such an entry is refused, with the set in `dims:` as the rewrite. In math a sum or an index ranges over a set, so the construct that `dims:` and `over=` take is named a set. Co-Authored-By: Fabian Hofmann Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1xxh6NMJN6mrYY34Q --- CHANGELOG.md | 3 + docs/about/index.md | 4 +- docs/about/overview.md | 2 +- docs/about/relations-as-linear-maps.md | 4 +- docs/about/sql-and-dataframes.md | 4 +- docs/contributing.md | 2 +- docs/examples/connection.md | 4 +- docs/examples/index.md | 2 +- docs/examples/library/generator.md | 2 +- docs/examples/library/load.md | 2 +- docs/examples/operators.md | 4 +- docs/howto/declare-a-column.md | 23 +- docs/howto/index.md | 2 +- docs/reference/composition.md | 42 +- docs/reference/glossary.md | 14 +- docs/reference/index.md | 4 +- docs/reference/language/absence.md | 32 +- docs/reference/language/dimensions.md | 4 +- docs/reference/language/errors.md | 4 +- docs/reference/language/expressions.md | 70 +-- docs/reference/language/file.md | 4 +- docs/reference/language/index.md | 2 +- docs/reference/language/operators.md | 12 +- .../parameters-variables-constraints.md | 186 +++++++- docs/reference/language/relations.md | 153 ++++--- docs/reference/notation.md | 37 +- docs/reference/reading.md | 40 +- docs/static/relations-where-bare.svg | 8 +- docs/tutorials/first-spec.md | 4 +- docs/tutorials/storage.md | 4 +- examples/connection.yaml | 2 +- examples/operators/sum_by_column_lists.yaml | 2 +- examples/operators/sum_by_columns.yaml | 2 +- examples/pypsa.yaml | 2 +- examples/pypsa/generator_maintenance.yaml | 2 +- examples/pypsa/link_maintenance.yaml | 2 +- examples/pypsa/process_maintenance.yaml | 2 +- schema/mathspec.schema.json | 12 +- src/mathspec/_advising.py | 5 +- src/mathspec/_composition.py | 6 +- src/mathspec/_condition_parser.py | 2 +- src/mathspec/_condition_resolver.py | 5 +- src/mathspec/_dimensions.py | 6 +- src/mathspec/_exclusivity.py | 4 +- src/mathspec/_expansion.py | 2 +- src/mathspec/_expression_resolver.py | 25 +- src/mathspec/_lowering.py | 135 +++++- src/mathspec/_operators.py | 4 +- src/mathspec/_piecewise.py | 2 +- src/mathspec/_renaming.py | 4 +- src/mathspec/_resolution.py | 4 +- src/mathspec/_typesetting/__init__.py | 5 +- src/mathspec/_typesetting/legend.py | 10 +- src/mathspec/_typesetting/walk.py | 55 ++- src/mathspec/_validation.py | 66 ++- src/mathspec/program.py | 40 +- src/mathspec/spec.py | 142 +++++- tests/test_canonical.py | 2 +- tests/test_composition.py | 32 +- tests/test_dimensions.py | 20 +- tests/test_expansion.py | 4 +- tests/test_lowering.py | 16 +- tests/test_parser.py | 2 +- tests/test_piecewise.py | 22 +- tests/test_pypsa_references.py | 2 +- tests/test_renaming.py | 14 +- tests/test_schema.py | 4 +- tests/test_validation.py | 433 ++++++++++++++---- tests/typesetting/golden/latex.out | 9 +- tests/typesetting/golden/markdown.out | 14 +- tests/typesetting/golden/model.yaml | 15 +- tests/typesetting/golden/typst.out | 11 +- tests/typesetting/test_walk.py | 6 +- tools/notation.py | 3 +- tools/pypsa_split.py | 4 +- 75 files changed, 1330 insertions(+), 509 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 477e8f4da..5dc07cdfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ it releases that version ([RELEASING.md](https://github.com/energy-models/mathsp ## Upcoming version +- feat(language)!: an entry or a sum may range over a set ([#0000](https://github.com/energy-models/mathspec/pull/0000)) + - Breaks a file: `relations:` is `sets:`. A set named in the `dims:` of a parameter, a variable or a constraint, or in `over=`, stands for its dimensions at the combinations it lists. + - Breaks a file: the `where:` of a parameter, a variable or a constraint no longer takes a set's bare name. Name the set in `dims:`. - feat(language)!: maps give labels, relations give rows ([#1015](https://github.com/energy-models/mathspec/pull/1015)) - Breaks a file: `relations:` takes `dims:`, `missing:` and `description` only. Each value column becomes a map under `maps:`, keyed by its subject, with `dim:` naming its dimension and always written, and is read as `subject[map]`, such as `generator[bus]`. - Breaks the program API: `Program.relations` holds one table per subject, `Relation` gains `partial`, `RelationDefined` gains `column`, `Reach.name` names `subject[map]`. diff --git a/docs/about/index.md b/docs/about/index.md index 397777f88..b4eb27c96 100644 --- a/docs/about/index.md +++ b/docs/about/index.md @@ -23,9 +23,9 @@ understand a rule before you rely on it. added to the language, and which requests mathspec refuses and why. - **[Versions](versions.md)** describes how the version of the package and the version of the language relate, and why there are two. -- **[Relations as linear maps](relations-as-linear-maps.md)** describes a sum +- **[Sets as linear maps](relations-as-linear-maps.md)** describes a sum through a table that pairs labels - ([a relation](../reference/language/relations.md)) as the product of a matrix and a vector. + ([a set](../reference/language/relations.md)) as the product of a matrix and a vector. - **[Understanding a spec in SQL and dataframe terms](sql-and-dataframes.md)** describes each declaration and operator of a spec as a table or a query, in SQL and in polars. diff --git a/docs/about/overview.md b/docs/about/overview.md index a91b76880..421753fc3 100644 --- a/docs/about/overview.md +++ b/docs/about/overview.md @@ -93,7 +93,7 @@ generator either. The file specifies the condition that the data must meet, and the engine that attaches the data checks the condition. - [`missing:`](../reference/language/absence.md#missing) on a parameter, a table that pairs labels - ([a relation](../reference/language/relations.md)) or a + ([a set](../reference/language/relations.md)) or a [map](../reference/language/relations.md#maps) sets what a missing row or label means. When the engine attaches the data, it refuses a missing row by default. - An [`assumptions:`](../reference/language/assumptions.md) entry is a diff --git a/docs/about/relations-as-linear-maps.md b/docs/about/relations-as-linear-maps.md index 30aca65a5..a86386835 100644 --- a/docs/about/relations-as-linear-maps.md +++ b/docs/about/relations-as-linear-maps.md @@ -19,7 +19,7 @@ dimensions: snapshot: { dtype: int } generator: { dtype: str } zone: { dtype: str } -relations: +sets: zone_of: { dims: [generator, snapshot], missing: absent } maps: zone_of: { zone: { dim: zone } } @@ -70,7 +70,7 @@ generators, snapshots and zones, `zone_of[zone]` in the spec above is a partial $`f: \mathcal{G} \times \mathcal{T} \to \mathcal{Z}`$, because the relation `zone_of` declares `missing: absent`. -A [relation](../reference/language/relations.md#relations) with no map is a +A [relation](../reference/language/relations.md#sets) with no map is a subset with no function behind it. So one generator may sit in several rows, each with its own bus. By default a relation also lists every key tuple, so one that does not must declare `missing: absent`. diff --git a/docs/about/sql-and-dataframes.md b/docs/about/sql-and-dataframes.md index ca0905eb7..8089cc696 100644 --- a/docs/about/sql-and-dataframes.md +++ b/docs/about/sql-and-dataframes.md @@ -70,7 +70,7 @@ are tables of the rows that a tool builds. `dispatch` has one decision variable per coordinate of `(snapshot, generator)`, and `power_balance` has one row per coordinate of `(snapshot, bus)`. -A [relation](../reference/language/relations.md#relations) is a table whose +A [set](../reference/language/relations.md#sets) is a table whose columns are all in its primary key, so it lists which combinations of labels exist. A [map](../reference/language/relations.md#maps) is a column of the table of its subject. `generator: { bus: { dim: bus } }` under `maps:` is a table @@ -141,7 +141,7 @@ exact except in four cases: ## Queries outside the language A spec combines two tables only through arithmetic, `sum(by=)` and `at(by=)`. -Each of `sum(by=)` and `at(by=)` joins with the table of a relation or a +Each of `sum(by=)` and `at(by=)` joins with the table of a set or a subject, and drops the key column of the join in the same call. A separate `join` would return a table with a key column that no dimension declares, so mathspec has no `join` operator. diff --git a/docs/contributing.md b/docs/contributing.md index 0f7c8ac9e..3ad0a276c 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -183,7 +183,7 @@ only when it meets these conditions: argument, such as `over=snapshot`. - It reads a bounded number of rows for each row of its result. It may filter rows on a column they already have, or join each row against a parameter, a - relation or a map. It may also read neighbouring rows at a fixed offset, or only + set or a map. It may also read neighbouring rows at a fixed offset, or only the coordinate labels. - If it reads every row, it reads them in one pass over the data. That pass ends before a tool that builds the model diff --git a/docs/examples/connection.md b/docs/examples/connection.md index 8d22fb1f6..3a2d8631b 100644 --- a/docs/examples/connection.md +++ b/docs/examples/connection.md @@ -9,7 +9,7 @@ Units with ports is a specification of an optimisation model ([a spec](../reference/glossary.md#spec)) in which each unit attaches to buses through numbered ports. Each port number matches one of the `bus0`, `bus1`, … columns of an open-source power system modelling framework -([PyPSA](https://pypsa.org)). The [relation](../reference/language/relations.md) +([PyPSA](https://pypsa.org)). The [set](../reference/language/relations.md) `connection` lists the bus and the role of each port. Its key is the unit and the port, not the bus, so two ports of one unit may attach to the same bus. @@ -32,7 +32,7 @@ dimensions: port: { dtype: int, description: "an attachment of a unit, numbered as PyPSA's bus0, bus1, ..." } role: { description: input or output } -relations: +sets: connection: description: the bus each port of a unit attaches to, and whether the port draws or delivers dims: [unit, port] diff --git a/docs/examples/index.md b/docs/examples/index.md index 2bf3e031b..c9fa01ca4 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -23,7 +23,7 @@ like yours, and what each line of the file means. inequality covers a running unit and a starting unit. [Open →](commitment.md) - **Units with ports** - One relation lists the ports of each unit, maps give each port a bus and + One set lists the ports of each unit, maps give each port a bus and a role, and a parameter with `where:` keeps only the output ports. [Open →](connection.md) - **One construct per spec** diff --git a/docs/examples/library/generator.md b/docs/examples/library/generator.md index f85e64db7..ad3f9cd8d 100644 --- a/docs/examples/library/generator.md +++ b/docs/examples/library/generator.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: CC-BY-4.0 This [fragment](../../reference/glossary.md#fragment) states `Generator` from an open-source power system framework ([PyPSA](https://pypsa.org)). The file declares its three dimensions, the -relation `Generator_port`, two parameters and the variable `Generator_p`. The file +set `Generator_port`, two parameters and the variable `Generator_p`. The file lists `Port_p` and `total_cost` under [`given`](../../reference/language/parameters-variables-constraints.md#given), and [the surface](surface.md) declares them. `Generator_cost` is a [named expression](../../reference/language/named.md#expressions) that adds to diff --git a/docs/examples/library/load.md b/docs/examples/library/load.md index cbe13e09e..ae79a7fbd 100644 --- a/docs/examples/library/load.md +++ b/docs/examples/library/load.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: CC-BY-4.0 # Loads This [fragment](../../reference/glossary.md#fragment) states `Load` from an open-source power system framework ([PyPSA](https://pypsa.org)). -The file declares the relation `Load_port`, the parameter `Load_p_set` and the +The file declares the set `Load_port`, the parameter `Load_p_set` and the constraint `Load_withdrawal`. The minus sign in front of `Load_p_set` is the only place where the file uses the sign convention of [the surface](surface.md). diff --git a/docs/examples/operators.md b/docs/examples/operators.md index 0f1c49448..a1905e087 100644 --- a/docs/examples/operators.md +++ b/docs/examples/operators.md @@ -159,7 +159,7 @@ dimensions: period: { dtype: int } zone: { dtype: str } -relations: +sets: zone_of: { dims: [generator, period] } maps: @@ -198,7 +198,7 @@ dimensions: bus: { dtype: str } technology: { dtype: str } -relations: +sets: slot_of: { dims: [generator, period] } maps: diff --git a/docs/howto/declare-a-column.md b/docs/howto/declare-a-column.md index ec1241c75..c60088ec1 100644 --- a/docs/howto/declare-a-column.md +++ b/docs/howto/declare-a-column.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: mathspec contributors SPDX-License-Identifier: CC-BY-4.0 --> -# Choose between a dimension, a relation, a map and a parameter +# Choose between a dimension, a set, a map and a parameter This guide shows how to choose the declaration for a [column](../reference/glossary.md#column) of your data, in @@ -16,13 +16,13 @@ file and the columns of the data you plan to attach to the spec. Use the first row that fits what you do with the column in the spec: -| In the spec you… | For example | So declare it as | -| --------------------------------------------------------------------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| index parameters, variables or constraints over the rows the column lists | `generator`, `snapshot` | a [`dimension`](../reference/language/dimensions.md) | -| look up a label of another dimension, one per row | the bus of each generator | a [`map`](../reference/language/relations.md#maps) of `generator` into `bus`, a dimension | -| list which pairs of two dimensions exist, with no value | `connection`, the buses a generator may connect to | a [`relation`](../reference/language/relations.md#relations) with [`missing: absent`](../reference/language/relations.md#missing) | -| use its numbers in an expression or a bound | `cost` per generator, `availability` per generator and snapshot | a [`parameter`](../reference/language/parameters-variables-constraints.md#parameters) over the dimensions that index it | -| test its values in a [condition](../reference/language/expressions.md#conditions) | `fuel`, `is_committable` | a `str` or `bool` parameter | +| In the spec you… | For example | So declare it as | +| --------------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| index parameters, variables or constraints over the rows the column lists | `generator`, `snapshot` | a [`dimension`](../reference/language/dimensions.md) | +| look up a label of another dimension, one per row | the bus of each generator | a [`map`](../reference/language/relations.md#maps) of `generator` into `bus`, a dimension | +| list which pairs of two dimensions exist, with no value | `connection`, the buses a generator may connect to | a [`set`](../reference/language/relations.md#sets) with [`missing: absent`](../reference/language/relations.md#missing) | +| use its numbers in an expression or a bound | `cost` per generator, `availability` per generator and snapshot | a [`parameter`](../reference/language/parameters-variables-constraints.md#parameters) over the dimensions that index it | +| test its values in a [condition](../reference/language/expressions.md#conditions) | `fuel`, `is_committable` | a `str` or `bool` parameter | ## Declare each kind of column @@ -34,7 +34,7 @@ dimensions: snapshot: { dtype: int } bus: { dtype: str } -relations: +sets: connection: { dims: [generator, bus], missing: absent } maps: @@ -50,8 +50,7 @@ parameters: variables: dispatch: { dims: [snapshot, generator], bounds: { lower: 0, upper: availability } } reserve: - dims: [snapshot, generator, bus] - where: connection + dims: [snapshot, connection] bounds: { lower: 0, upper: availability } constraints: @@ -106,6 +105,6 @@ spec and has no remark that is not a refusal - [Check a spec without data](check.md), for how to read a refusal or advice. - [Dimensions](../reference/language/dimensions.md), - [relations and maps](../reference/language/relations.md) and + [sets and maps](../reference/language/relations.md) and [parameters](../reference/language/parameters-variables-constraints.md#parameters), for every key of each declaration. diff --git a/docs/howto/index.md b/docs/howto/index.md index 5560aa699..2da60b11f 100644 --- a/docs/howto/index.md +++ b/docs/howto/index.md @@ -28,7 +28,7 @@ To learn the language from the start, follow the
-- **[Choose between a dimension, a relation, a map and a parameter](declare-a-column.md)** +- **[Choose between a dimension, a set, a map and a parameter](declare-a-column.md)** for a column of your data. - **[State a rule that differs by case](cases.md)**, such as committable and non-committable generators, in one spec. diff --git a/docs/reference/composition.md b/docs/reference/composition.md index a14bc4936..214f0377b 100644 --- a/docs/reference/composition.md +++ b/docs/reference/composition.md @@ -176,21 +176,21 @@ The objective is the one that `network.yaml` sets. ### Each kind of entry Two fragments may share a [dimension](language/dimensions.md), a -[relation](language/relations.md#relations), a [map](language/relations.md#maps), a given entry or a +[set](language/relations.md#sets), a [map](language/relations.md#maps), a given entry or a sum. -| The entry | What `merge` does | -| ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | -| a dimension, a relation or a map | accepts it from every fragment that declares it, where every fragment declares it the same way | -| a `description` on a shared dimension, relation or map | keeps the first wording in the list | -| `ordered: true` on a shared dimension | makes the dimension ordered if one fragment sets it | -| `missing:` on a shared relation or map | requires the same value from every fragment that declares it, where `missing: refused` and no `missing:` are one value | -| an entry under `given:` | checks it against the fragment that declares the name, then folds it in ([given entries](#given-entries)) | -| an expression with `adds_to:` | adds it as a term to the sum in its `adds_to:` ([terms and sums](#terms-and-sums)) | -| `objective` | takes it from the one fragment that sets it, and refuses a second | -| `version` | requires the same value from every fragment | -| `description` at the top of a fragment | drops it | -| any other entry | takes it from the one fragment that declares it, and refuses a second | +| The entry | What `merge` does | +| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| a dimension, a set or a map | accepts it from every fragment that declares it, where every fragment declares it the same way | +| a `description` on a shared dimension, set or map | keeps the first wording in the list | +| `ordered: true` on a shared dimension | makes the dimension ordered if one fragment sets it | +| `missing:` on a shared set or map | requires the same value from every fragment that declares it, where `missing: refused` and no `missing:` are one value | +| an entry under `given:` | checks it against the fragment that declares the name, then folds it in ([given entries](#given-entries)) | +| an expression with `adds_to:` | adds it as a term to the sum in its `adds_to:` ([terms and sums](#terms-and-sums)) | +| `objective` | takes it from the one fragment that sets it, and refuses a second | +| `version` | requires the same value from every fragment | +| `description` at the top of a fragment | drops it | +| any other entry | takes it from the one fragment that declares it, and refuses a second | ### Terms and sums @@ -432,10 +432,10 @@ patched `dispatch` has no row at `old`. | a whole entry under a new name | adds it | | a named expression as a [bare string](language/named.md#expressions) | replaces only its body, `expression:` or `cases:`, and keeps its `dims:`, description and `adds_to:`, so the new body may have no dimension outside those `dims:` | | `null` on an entry, a field or a section | applies the rule in [`null` in a patch](#null-in-a-patch) | -| a dimension, a relation or a map | adds it, or accepts it restated as the base declares it, field by field, where a field left out equals its default | -| `description:` on a dimension, a relation or a map of the base | replaces the description of the base, alone or with other fields, and `null` removes it | +| a dimension, a set or a map | adds it, or accepts it restated as the base declares it, field by field, where a field left out equals its default | +| `description:` on a dimension, a set or a map of the base | replaces the description of the base, alone or with other fields, and `null` removes it | | `ordered:` on a restated dimension | makes the dimension ordered for `true`, and refuses `false` over an ordered dimension | -| `missing:` on a relation or a map, alone or with other fields | changes whether every combination needs a row, or every row a label, as on a parameter | +| `missing:` on a set or a map, alone or with other fields | changes whether every combination needs a row, or every row a label, as on a parameter | | an entry under one kind of `given:` | edits, adds or removes it like any entry, and keeps the other kinds | | `version`, `description` | replaces the value of the base | | a field that an earlier patch writes | replaces that value | @@ -522,8 +522,8 @@ A second call with the inverse map returns a spec with the | -------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | | a name as the key of an entry, in `dims:`, under `maps:`, in `adds_to:` or in `given:` | replaces the name | | a name in an expression, `where:`, bound, case or assumption | replaces the name, and keeps the spaces and line breaks around it | -| a map or a relation column named after its dimension, such as `bus` in `unit[bus]` | replaces it with the dimension | -| a map or a relation column named after something else, such as `bus0` in `{bus0: bus}` | keeps it, and refuses a map that changes a name to or from it | +| a map or a set column named after its dimension, such as `bus` in `unit[bus]` | replaces it with the dimension | +| a map or a set column named after something else, such as `bus0` in `{bus0: bus}` | keeps it, and refuses a map that changes a name to or from it | | a longer name that contains a renamed name, such as `Unit_p_nom` for `Unit_p` | keeps the longer name | | a keyword of an operator, such as `over=` | keeps the keyword | | a quoted label, a label under `one_of:`, or a number such as `1e6` | keeps the label or the number | @@ -601,9 +601,9 @@ patch 'project.yaml' edits the constraint 'power_balnce', which its base does no ### A dimension restated differently -Over a dimension, a relation or a map that its base declares, a patch may +Over a dimension, a set or a map that its base declares, a patch may change only `description`, a dimension's `ordered`, and the `missing` of a -relation or a map. A patch +set or a map. A patch that writes any other field restates the entry. Every other field must then be as the base declares it, and a field the patch leaves out equals its default. @@ -697,7 +697,7 @@ rename gives the parameter 'Unit_marginal_cost' the name 'x', which is an argume ### A name that is also a map `rename` refuses a map that changes a name to or from the name of a -[map](language/relations.md#maps) or a relation column that is not named after +[map](language/relations.md#maps) or a set column that is not named after its dimension. An expression such as `line[bus0]` reads the map by its name, so the rename would change the map too. Here `line` has the maps `{bus0: bus, bus1: bus}`, and the map is `{'bus0': 'price'}`: diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 9d35fc338..dad280c07 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -55,7 +55,7 @@ line](cli.md)). ## Column A column is one variable at one coordinate, so a variable indexed over -`generator` has one column per generator. In a data table or a relation, a +`generator` has one column per generator. In a data table or a set, a column is one named field of the table. ## Compile @@ -85,7 +85,7 @@ and a constraint has one row at each coordinate. ## Data preparation Before the data is attached, data preparation is the work of turning raw data -into the tables that a spec declares as parameters and relations. You compute +into the tables that a spec declares as parameters and sets. You compute in data preparation anything that the mathspec language cannot state ([what counts as data preparation](../about/limits.md#data-preparation)). @@ -142,7 +142,7 @@ does not build](reading.md#what-a-program-does-not-build)). ## Group -A group is the set of rows of a dimension or a relation to which one +A group is the set of rows of a dimension or a set to which one [map](#map) gives one label. `within=[]` makes `shift`, `sum_back` or `position()` work separately in each group, such as each season of a year ([partitions](language/relations.md#partitions)). @@ -169,14 +169,14 @@ expression before any tool reads it. A file declares its own macros under ## Map -A map gives each row of a dimension or a relation (its +A map gives each row of a dimension or a set (its [subject](#subject)) one label of a dimension, such as the bus of each generator. A condition or an operator reads it as `subject[map]`, as in `generator[bus]` ([maps](language/relations.md#maps)). ## Missing row -A missing row is a coordinate for which the table of a parameter or a relation +A missing row is a coordinate for which the table of a parameter or a set has no row. The `missing:` key of the entry sets how a tool treats that coordinate ([`missing` on a parameter](language/absence.md#missing-on-a-parameter)). @@ -233,7 +233,7 @@ program in this sense. ## Row A row is one constraint at one coordinate, so a constraint indexed over -`snapshot` has one row per snapshot. In a data table or a relation, a row is +`snapshot` has one row per snapshot. In a data table or a set, a row is one line of the table ([missing row](#missing-row)). ## Rung @@ -273,7 +273,7 @@ most two consecutive members are non-zero ([sos](language/piecewise.md#sos)). ## Subject -The subject of a map is a dimension or a relation, such as `generator` in +The subject of a map is a dimension or a set, such as `generator` in `generator[bus]` ([maps](language/relations.md#maps)). ## Surface diff --git a/docs/reference/index.md b/docs/reference/index.md index 11a935b6b..30c982823 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -24,8 +24,8 @@ optimisation model ([a spec](glossary.md#spec)). `given:` section. - **[Dimensions](language/dimensions.md)** lists the keys of a named set of labels (a dimension). -- **[Relations and maps](language/relations.md)** lists the keys of a set of - combinations of labels (a relation) and of the label each row has (a map), +- **[Sets and maps](language/relations.md)** lists the keys of a set of + combinations of labels (a set) and of the label each row has (a map), and how an expression uses them. - **[Expressions](language/expressions.md)** defines the grammar of arithmetic and of conditions, and how dimensions combine. diff --git a/docs/reference/language/absence.md b/docs/reference/language/absence.md index 731c34885..462fef295 100644 --- a/docs/reference/language/absence.md +++ b/docs/reference/language/absence.md @@ -51,19 +51,20 @@ contain. ## What creates absence -Eight constructs make a coordinate absent: - -| Construct | What is absent | -| ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `where:` on a variable | the variable, at the coordinates the condition excludes ([`where`](#where)) | -| `where:` on a parameter | the parameter, at the coordinates the condition excludes ([`where` on a parameter](parameters-variables-constraints.md#where)) | -| `where:` on a constraint | the row ([`where`](#where)) | -| `shift(x, along=d, offset=n)` without `edge=` | the expression, at the edge coordinate ([shift](operators.md#shift)) | -| a key with no row in a `missing: absent` relation | the relation, at that key ([relations](relations.md#missing)) | -| a row with no label in a `missing: absent` map | the map, at that row ([maps](relations.md#missing)) | -| a missing row of a `missing: absent` parameter | the parameter, at that coordinate ([`missing`](#missing)) | -| `within=` at a coordinate in no group | the expression, at that coordinate ([`within=`](operators.md#within)) | -| a quotient whose divisor is zero | the quotient, at that coordinate ([reported values](#reported-values)) | +Nine constructs make a coordinate absent: + +| Construct | What is absent | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `where:` on a variable | the variable, at the coordinates the condition excludes ([`where`](#where)) | +| `where:` on a parameter | the parameter, at the coordinates the condition excludes ([`where` on a parameter](parameters-variables-constraints.md#where)) | +| `where:` on a constraint | the row ([`where`](#where)) | +| `shift(x, along=d, offset=n)` without `edge=` | the expression, at the edge coordinate ([shift](operators.md#shift)) | +| a key with no row in a `missing: absent` set | the set, at that key ([sets](relations.md#missing)) | +| a row with no label in a `missing: absent` map | the map, at that row ([maps](relations.md#missing)) | +| a key with no row in a set in `dims:` | the parameter or variable, at that key ([a set in `dims`](parameters-variables-constraints.md#a-set-in-dims)) | +| a missing row of a `missing: absent` parameter | the parameter, at that coordinate ([`missing`](#missing)) | +| `within=` at a coordinate in no group | the expression, at that coordinate ([`within=`](operators.md#within)) | +| a quotient whose divisor is zero | the quotient, at that coordinate ([reported values](#reported-values)) | No value shifts in at the edge, so a `shift` without `edge=` has no row there: @@ -228,8 +229,9 @@ as [missing rows](../reading.md#missing-rows) describes. ### `missing` on a variable -Outside its `where:`, a variable is absent by default. Its `missing:` key can -set a number there instead, such as `0`. Under `absent`, a row that needs that +Outside its `where:` and its [set](parameters-variables-constraints.md#a-set-in-a-variables-dims), +a variable is absent by default. Its `missing:` key can set a number there +instead, such as `0`. Under `absent`, a row that needs that coordinate is absent. A file declares `missing: 0` for a quantity that is zero outside its condition: diff --git a/docs/reference/language/dimensions.md b/docs/reference/language/dimensions.md index 924af95f4..ce5d92718 100644 --- a/docs/reference/language/dimensions.md +++ b/docs/reference/language/dimensions.md @@ -101,7 +101,7 @@ Variable 'p': 'gnrtr' is not a label of 'carrier', which allows only ['ccgt', 'o mathspec lists the allowed labels and suggests two rewrites. With data attached, a tool refuses a label -outside the list: in the table of the dimension, in every column of a relation +outside the list: in the table of the dimension, in every column of a set over it, and in the column of every map into it. Only a dimension takes `one_of`, and a `one_of` key on a parameter is refused @@ -159,5 +159,5 @@ the dimension still needs `ordered: true`. !!! info "See also" - - [choose between a dimension, a relation, a map and a parameter](../../howto/declare-a-column.md) + - [choose between a dimension, a set, a map and a parameter](../../howto/declare-a-column.md) for a column of data. diff --git a/docs/reference/language/errors.md b/docs/reference/language/errors.md index a075238b4..a21253c7f 100644 --- a/docs/reference/language/errors.md +++ b/docs/reference/language/errors.md @@ -142,7 +142,7 @@ advice, so the file still loads and `check` exits `0`. | `kind` | The file has… | The advice is… | | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | -| `never-an-axis` | a dimension that indexes nothing, that nothing sums into, and that no relation or map has a column over | remove it, or keep it if the entries that use it are still to be written | +| `never-an-axis` | a dimension that indexes nothing, that nothing sums into, and that no set or map has a column over | remove it, or keep it if the entries that use it are still to be written | | `given` | a parameter, column, named expression or set of constraint rows (a row family) that it uses and does not declare ([given](parameters-variables-constraints.md#given)) | the file this one is layered onto provides it | | `unbounded` | a variable that no constraint, set or curve uses, whose objective term pushes it towards a bound it does not have | give it a finite bound, or the constraint that was meant to define it | @@ -185,7 +185,7 @@ objective: Both commands print the same two messages of advice: ```text -dimension 'region' is never used: nothing is indexed by it, nothing sums into it, and no relation or map is over it. Remove it, or keep it if the entries that use it are still to be written. +dimension 'region' is never used: nothing is indexed by it, nothing sums into it, and no set or map is over it. Remove it, or keep it if the entries that use it are still to be written. Variable 'slack' makes this spec unbounded: no constraint names it, and bounds.lower is open, and the +slack term in the minimize objective improves toward it. Give it a finite bounds.lower, or the constraint that was meant to define it. ``` diff --git a/docs/reference/language/expressions.md b/docs/reference/language/expressions.md index eda314a6c..303641b5a 100644 --- a/docs/reference/language/expressions.md +++ b/docs/reference/language/expressions.md @@ -207,7 +207,7 @@ An addition is `0` only where every term can be, so Every name a spec declares is in one flat set of names (a namespace). That namespace contains -the dimensions, [relations](relations.md), parameters, variables, named +the dimensions, [sets](relations.md), parameters, variables, named expressions, [named conditions](named.md#conditions), [macros](named.md#macros) and the built-in operators. Two entries with one name are refused at load, and mathspec names both, so no name hides another. For @@ -226,7 +226,7 @@ The place of a name in a string sets which kinds of name it may be: | expression (`p * cost`) | a variable, or a parameter whose values are numbers ([dtype](parameters-variables-constraints.md#parameters)) | | dimension argument (`over=`, `along=`) | a dimension, a list of them for `over=`, or with `by=` a column of the table in `by=` where one has that name | | column argument (`by=`, `within=`) | maps or columns of one subject, written `subject[column]` or `subject[column, …]` | -| condition | a parameter, variable, dimension, relation, map or named condition ([conditions](#conditions)) | +| condition | a parameter, variable, dimension, set, map or named condition ([conditions](#conditions)) | | `bounds.lower` / `bounds.upper` | a parameter name, or a number | | the `edge` key of `shift` | `'wrap'` in quotes, or a bare number | | `dual` argument (`dual(c)`) | a constraint, resolved against the constraints alone ([named expressions](named.md#dual)) | @@ -251,18 +251,18 @@ and never to a constraint of the same name. Before any data is attached, [`to_spec`](../api.md#mathspec.to_spec) works out the dimensions of every expression. Each part of an expression (a node) has a set of dimensions: -| Node | Dimensions of the result | Refused when | -| -------------------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------- | -| number | `{}` | | -| parameter / variable | its `dims` | | -| `-x`, `+x` | `dims(x)` | | -| `a + b`, `a * b`, `a / b` | `dims(a) ∪ dims(b)` | | -| `sum(x)` | `{}` | `dims(x)` is already empty | -| `sum(x, over=d)` | `dims(x) − {d}` | `d ∉ dims(x)` | -| `sum(x, over=d, by=s[c])` | `(dims(x) − joined) ∪ grouped` | a refusal under [how relations and maps are used](relations.md#how-relations-and-maps-are-used) applies | -| `at(x, by=s[m])` | `(dims(x) − joined) ∪ grouped` | the same refusals apply | -| `shift(x, along=d, offset=n)` | `dims(x)` | `d ∉ dims(x)` | -| `sum_back(x, along=d, window=n)` | `dims(x)` | `d ∉ dims(x)` | +| Node | Dimensions of the result | Refused when | +| -------------------------------- | ------------------------------ | --------------------------------------------------------------------------------------------- | +| number | `{}` | | +| parameter / variable | its `dims` | | +| `-x`, `+x` | `dims(x)` | | +| `a + b`, `a * b`, `a / b` | `dims(a) ∪ dims(b)` | | +| `sum(x)` | `{}` | `dims(x)` is already empty | +| `sum(x, over=d)` | `dims(x) − {d}` | `d ∉ dims(x)` | +| `sum(x, over=d, by=s[c])` | `(dims(x) − joined) ∪ grouped` | a refusal under [how sets and maps are used](relations.md#how-sets-and-maps-are-used) applies | +| `at(x, by=s[m])` | `(dims(x) − joined) ∪ grouped` | the same refusals apply | +| `shift(x, along=d, offset=n)` | `dims(x)` | `d ∉ dims(x)` | +| `sum_back(x, along=d, window=n)` | `dims(x)` | `d ∉ dims(x)` | mathspec refuses `sum(p, over=t)` with `p` over `g` alone, and prints: @@ -322,26 +322,26 @@ QUOTED ::= "'" chars "'" | '"' chars '"' A name written alone is a bare name, and what it tests depends on the kind of its entry. -| Written as | Kind of name | Meaning | -| ---------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `name` (bare) | parameter | For a number, it is true where the data has a row, whatever the `missing:`. For a `bool`, it is its value, and the `missing:` sets the value at a missing row ([`missing`](parameters-variables-constraints.md#missing)). `0.0` is a row, and `inflow != 0` tests for non-zero | -| `name` (bare) | variable | It is true where the variable exists | -| `name` (bare) | relation | It is true where the relation has a row. A `missing: absent` relation has no row for some combinations ([`missing`](relations.md#missing)), and the bare name selects the combinations it lists | -| `subject[map]` (bare) | map | It is true where the map gives the row of its subject a label. A `missing: absent` map gives some rows no label ([`missing`](relations.md#missing)) | -| `name` (bare) | dimension | mathspec refuses it at load | -| `name` (bare) | named condition | It tests the [named condition](named.md#conditions). A given condition is true or false at each coordinate, as a `bool` parameter is | -| `name OP value` | parameter | It compares element by element. A missing row with no `missing:` value compares false | -| `name OP value` | dimension | It filters the frame's own coordinate column | -| `subject[map] OP value` | map | It compares the label that the map gives each row of its subject. A comparison reads one map. Where the map gives no label, the comparison is false for every operator, `!=` included, so `NOT` of it is true there | -| `s[a] OP t[b]` | two maps | mathspec accepts it where both subjects are over the same dimensions and both maps refer to one dimension. `line[bus0] != line[bus1]` excludes a self-loop. Where either map gives no label, the comparison is false | -| `expression OP expression` | arithmetic over parameters | It compares coordinate by coordinate, over every dimension either side is over ([arithmetic in a comparison](#arithmetic-in-a-comparison)). A side with no value at a coordinate compares false | -| `position(name) OP i` | dimension | It tests where the row sits along the dimension's own order. `0` is first, and a negative number counts from the end | -| `position(name, within=s[m])` | dimension | It tests the same, counted within each group the map makes | -| `count(condition, over=name) OP i` | condition | It compares how many coordinates along the dimension the condition admits ([`count`](#count)) | -| `shift(condition, along=name, offset=i)` | condition | It is the condition `i` coordinates back, and false at the first `i` coordinates | -| `at(condition, by=s[m])` | condition | It is the condition through the map ([`at` in a condition](#at-in-a-condition)), and false where the map gives no label | -| `AND` `OR` `NOT` | | They are case-insensitive. `NOT` applies before `AND`, and `AND` before `OR` | -| `True` / `False` | | `True` is the same as no `where`. `False` leaves an entry with no rows | +| Written as | Kind of name | Meaning | +| ---------------------------------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` (bare) | parameter | For a number, it is true where the data has a row, whatever the `missing:`. For a `bool`, it is its value, and the `missing:` sets the value at a missing row ([`missing`](parameters-variables-constraints.md#missing)). `0.0` is a row, and `inflow != 0` tests for non-zero | +| `name` (bare) | variable | It is true where the variable exists | +| `name` (bare) | set | It is true where the set has a row. A `missing: absent` set has no row for some combinations ([`missing`](relations.md#missing)), and the bare name selects the combinations it lists. The `where:` of a parameter, a variable or a constraint cannot hold it, because naming the set in [`dims:`](parameters-variables-constraints.md#a-set-in-dims) states those rows | +| `subject[map]` (bare) | map | It is true where the map gives the row of its subject a label. A `missing: absent` map gives some rows no label ([`missing`](relations.md#missing)) | +| `name` (bare) | dimension | mathspec refuses it at load | +| `name` (bare) | named condition | It tests the [named condition](named.md#conditions). A given condition is true or false at each coordinate, as a `bool` parameter is | +| `name OP value` | parameter | It compares element by element. A missing row with no `missing:` value compares false | +| `name OP value` | dimension | It filters the frame's own coordinate column | +| `subject[map] OP value` | map | It compares the label that the map gives each row of its subject. A comparison reads one map. Where the map gives no label, the comparison is false for every operator, `!=` included, so `NOT` of it is true there | +| `s[a] OP t[b]` | two maps | mathspec accepts it where both subjects are over the same dimensions and both maps refer to one dimension. `line[bus0] != line[bus1]` excludes a self-loop. Where either map gives no label, the comparison is false | +| `expression OP expression` | arithmetic over parameters | It compares coordinate by coordinate, over every dimension either side is over ([arithmetic in a comparison](#arithmetic-in-a-comparison)). A side with no value at a coordinate compares false | +| `position(name) OP i` | dimension | It tests where the row sits along the dimension's own order. `0` is first, and a negative number counts from the end | +| `position(name, within=s[m])` | dimension | It tests the same, counted within each group the map makes | +| `count(condition, over=name) OP i` | condition | It compares how many coordinates along the dimension the condition admits ([`count`](#count)) | +| `shift(condition, along=name, offset=i)` | condition | It is the condition `i` coordinates back, and false at the first `i` coordinates | +| `at(condition, by=s[m])` | condition | It is the condition through the map ([`at` in a condition](#at-in-a-condition)), and false where the map gives no label | +| `AND` `OR` `NOT` | | They are case-insensitive. `NOT` applies before `AND`, and `AND` before `OR` | +| `True` / `False` | | `True` is the same as no `where`. `False` leaves an entry with no rows | With a dimension `bp` declared, mathspec prints for `where: "bp"`: @@ -360,7 +360,7 @@ under Variable 'p': 'undeclared_thing' not found. Parameters: ['bp_x', 'points'] Dimensions: ['bp', 'generator'] - Relations: [] + Sets: [] Check the spelling, or declare 'undeclared_thing'. ``` diff --git a/docs/reference/language/file.md b/docs/reference/language/file.md index 8ed84a423..9f3e3b5fb 100644 --- a/docs/reference/language/file.md +++ b/docs/reference/language/file.md @@ -18,8 +18,8 @@ declares only `dimensions:` loads. | ------------- | ------------------------------------------------------------------------------------------------------------------ | | `given` | what this file uses and another file declares ([given](parameters-variables-constraints.md#given)) | | `dimensions` | the sets of labels that entries are indexed over ([dimensions](dimensions.md)) | -| `relations` | named sets of combinations of labels of two or more dimensions ([relations](relations.md#relations)) | -| `maps` | the label of a dimension that each row of a dimension or a relation has ([maps](relations.md#maps)) | +| `sets` | named sets of combinations of labels of two or more dimensions ([sets](relations.md#sets)) | +| `maps` | the label of a dimension that each row of a dimension or a set has ([maps](relations.md#maps)) | | `parameters` | the data the spec expects ([parameters](parameters-variables-constraints.md#parameters)) | | `conditions` | named conditions, used by name in a `where:`, a `when:` or a `holds:` ([conditions](named.md#conditions)) | | `variables` | what the solver decides ([variables](parameters-variables-constraints.md#variables)) | diff --git a/docs/reference/language/index.md b/docs/reference/language/index.md index 12e9c2c51..832619cc4 100644 --- a/docs/reference/language/index.md +++ b/docs/reference/language/index.md @@ -18,7 +18,7 @@ _entry_, _coordinate_ and _absent_, in one fixed sense. - **[File shape](file.md)** lists the top-level keys of a spec file, and how mathspec reads the YAML. - **[Given, parameters, variables, objective and constraints](parameters-variables-constraints.md)** lists the keys of the data, the decisions, the goal and the rules, and of the `given:` section. - **[Dimensions](dimensions.md)** lists the keys of a dimension, the set of labels that entries are indexed over. -- **[Relations and maps](relations.md)** lists the keys of a relation, a set of combinations of labels, and of a map, which gives each row one label, and how each operator uses them. +- **[Sets and maps](relations.md)** lists the keys of a set, a set of combinations of labels, and of a map, which gives each row one label, and how each operator uses them. - **[Expressions](expressions.md)** defines the grammar of arithmetic and of conditions, and how dimensions combine. - **[Named expressions, conditions and macros](named.md)** lists the keys of the three kinds of named piece of an expression. - **[Operators](operators.md)** lists every operator, such as `sum` and `shift`, and the math each one prints. diff --git a/docs/reference/language/operators.md b/docs/reference/language/operators.md index e0a87860e..4a70329b6 100644 --- a/docs/reference/language/operators.md +++ b/docs/reference/language/operators.md @@ -26,7 +26,7 @@ A file cannot add an operator, and only | `sum(array)` | `sum` removes every [dimension](dimensions.md) of `array`. The result is a scalar | | `sum(array, over=dim)` | `sum` removes `dim` | | `sum(array, over=[a, …])` | `sum` removes each dimension in the list | -| `sum(array, over=dim, by=subject[c])` | `sum` removes `dim`, and groups the result by the column `c` of the table of `subject`, a [map](relations.md#maps) or a column of a [relation](relations.md#relations). The call joins on the other key columns, so `array` and the result are indexed over them | +| `sum(array, over=dim, by=subject[c])` | `sum` removes `dim`, and groups the result by the column `c` of the table of `subject`, a [map](relations.md#maps) or a column of a [set](relations.md#sets). The call joins on the other key columns, so `array` and the result are indexed over them | | `sum(array, over=[dim, …], by=subject[c, …])` | The call does the same with several dimensions and several columns of one table, joins on them together, and groups by their product | | `at(array, by=subject[m])` | The result is `array` read at the label that the map `m` gives each row of `subject`, once per row. The result gains the key of `subject` | | `shift(array, along=dim, offset=n)` | The result is the value `n` positions earlier along `dim`, and the vacated edge is [absent](absence.md) | @@ -54,7 +54,7 @@ each dimension once. `sum(x, over=[a, b])` is the same as one sum over both sets. With `by=`, the sum adds up each group that a [map](relations.md#maps) or a -column of a [relation](relations.md#relations) makes, and the result is +column of a [set](relations.md#sets) makes, and the result is indexed over that column. The balance at each bus (a nodal balance) is one `sum(by=)` per kind of component: @@ -110,8 +110,8 @@ sum. !!! info "See also" - - [How relations and maps are used](relations.md#how-relations-and-maps-are-used), - for the dimensions that a call through a relation or a map joins on, and + - [How sets and maps are used](relations.md#how-sets-and-maps-are-used), + for the dimensions that a call through a set or a map joins on, and those its result has. ## `at` @@ -428,8 +428,8 @@ The typesetter prints each form from the smallest spec that uses it. - [Absence and where](absence.md) for what each operator does with an absent input. - - [Relations and maps](relations.md#how-relations-and-maps-are-used) for the - relation or the map that `by=` and `within=` take. + - [Sets and maps](relations.md#how-sets-and-maps-are-used) for the + set or the map that `by=` and `within=` take. - [Macros](named.md#macros) for a composition of operators under one name. - [One construct per spec](../../examples/operators.md) for the smallest spec that uses each form. diff --git a/docs/reference/language/parameters-variables-constraints.md b/docs/reference/language/parameters-variables-constraints.md index 2240d688b..5630ba9f2 100644 --- a/docs/reference/language/parameters-variables-constraints.md +++ b/docs/reference/language/parameters-variables-constraints.md @@ -315,7 +315,8 @@ A `str` parameter takes any label. A dimension with `missing:` sets the meaning of a coordinate of the `dims` at which the data has no row. It takes `refused` (the default), `absent`, or a value of the `dtype` of the parameter. On a parameter with a [`where:`](#where), `missing:` applies -only where the condition is true. +only where the condition is true. On a parameter over a +[set](#a-set-in-dims), it applies only where the set has a row. ```yaml dimensions: @@ -404,7 +405,7 @@ dimensions: bus: { dtype: str } role: { dtype: str } snapshot: { dtype: int } -relations: +sets: connection: { dims: [entity, bus], missing: absent } maps: connection: { role: { dim: role } } @@ -449,17 +450,26 @@ snapshot of the example, and drops every other row. A tool drops a row of the data outside the condition without refusing the data. An [assumption](assumptions.md) on the parameter does not test that row -either, because the parameter has no row there. +either, because the parameter has no row there. To have a tool refuse a row at +a pair that is not wired, name the set in `dims:` +([a set in `dims`](#a-set-in-dims)). A set name alone in the `where:` would +state the same rows a second time, so mathspec refuses it. For +`where: connection` on `efficiency`, the message is: + +```text +Parameter 'efficiency': `where: connection` tests the rows of 'connection', which dims: states. Name it in dims: in place of the dimensions, and delete the `where:`. +``` A tool evaluates the condition from the data alone, before it reads the parameter. So the condition may contain [dimensions](dimensions.md), -[relations](relations.md) and parameters without a `where:`. +[sets](relations.md) and parameters with neither a `where:` nor a set in +`dims:`. mathspec refuses four conditions at load: - a condition that contains a variable - a condition that contains the parameter itself -- a condition that contains a parameter with a `where:` of its own +- a condition that contains a parameter with a `where:` or a set in its `dims:` - a condition over a dimension that the parameter does not have For a `where:` on `efficiency` that contains the variable `p`, the message @@ -475,6 +485,127 @@ The typeset legend lists the condition beside the dimensions, after `only where`. For `efficiency`, the legend entry ends in `only where` $\mathrm{role}(b,\ e) = \text{'}\mathrm{output}\text{'}$. +### A set in `dims` + +A [set](relations.md) named in `dims:` stands for its dimensions, at the +combinations it lists. The parameter has coordinates only there, and a tool +refuses a row of the data anywhere else. A [`where:`](#where) drops that row +instead. In this example, `efficiency` and `p_nom_max` have values only at the +pairs that `connection` lists: + +```yaml +dimensions: + entity: { dtype: str } + bus: { dtype: str } + snapshot: { dtype: int } +sets: + connection: { dims: [entity, bus], missing: absent } +parameters: + efficiency: { dims: [connection, snapshot], missing: 1 } + p_nom_max: { dims: [connection], missing: .inf } + distance: { dims: [entity, bus] } +variables: + p: { dims: [entity, snapshot] } +constraints: + balance: + dims: [bus, snapshot] + expression: sum(efficiency * p, over=entity) == 0 +``` + +`efficiency` is over `entity`, `bus` and `snapshot`: `connection` stands for +`entity` and `bus`. `distance` names the two dimensions themselves, so it is +over every pair and needs a row at each. + +With data attached, a tool decides the coordinates of `efficiency` in three +steps: + +1. Where `connection` has no row, `efficiency` is [absent](absence.md), and + the tool refuses any row of the data of `efficiency` there. +2. Where the [`where:`](#where) of `efficiency` is false, the tool drops the + row. `efficiency` has no `where:`, so the tool drops nothing. +3. At each remaining coordinate with no row, `missing: 1` sets the value. + +So `missing: 1` fills only the wired pairs, and a data row for an entity at a +bus it is not wired to is refused. + +An entry over two sets that share a dimension, such as +`dims: [connection, active]`, has coordinates only where both list them. A +sum over a set, `sum(x, over=connection)`, sums over its dimensions, as +`over=[entity, bus]` does. + +A set in `dims:` gives its rows, and its [maps](relations.md#maps) play no +part. They stay readable as `connection[bus]` in a `where:` and in a call. In +this example, `connection` has a row per port of a unit, and its maps `bus` and +`role` give each port a bus and a role: + +```yaml +dimensions: + unit: { dtype: str } + port: { dtype: int } + bus: { dtype: str } + role: { dtype: str } + snapshot: { dtype: int } +sets: + connection: { dims: [unit, port], missing: absent } +maps: + connection: { bus: { dim: bus }, role: { dim: role } } +parameters: + efficiency: + dims: [connection, snapshot] + where: "connection[role] == 'output'" + missing: 1 +variables: + flow: { dims: [unit, snapshot], bounds: { lower: 0 } } +constraints: + delivered: + dims: [bus, snapshot] + expression: sum(efficiency * flow, over=connection, by=connection[bus]) >= 0 +``` + +`efficiency` has coordinates at each port that `connection` lists, and a tool +refuses a data row for a port that `connection` does not list. The `where:` +then drops the input ports. `efficiency` is not over `bus` or `role`, the +dimensions that the maps refer to. + +mathspec refuses three sets in `dims:` at load: + +- a set named twice +- a set under `missing: refused`, which has a row at every combination and so + leaves nothing out +- a set with two columns over one dimension, because an entry is over each + dimension once + +For `p_nom_max: { dims: [connection] }` with `connection` under +`missing: refused`, the message is: + +```text +Parameter 'p_nom_max': dims: names 'connection', which under missing: refused has a row at every combination, so it leaves nothing out. Write missing: absent on 'connection', or name its dimensions in its place. +``` + +A `where:` on a parameter over a set narrows the coordinates inside the set, +and the tool drops the rows outside it, as in step 2. Inside the set, its bare +name is true. So mathspec refuses a `where:` that reads it, and names what the +`where:` is left with. `where: connection` on `efficiency` in the first example +is always true: + +```text +Parameter 'efficiency': dims: names 'connection', so 'efficiency' has coordinates only where it has a row, and `where: connection` narrows nothing. Delete the `where:`, or the set from it. +``` + +A `where:` that is always false, such as `where: NOT connection`, leaves the +parameter no coordinate, and mathspec suggests a `where:` that holds somewhere +inside the set. For `where: "connection AND active"`, where `active` is a +`bool` parameter, the message ends in +`` `where: connection AND active` repeats the set. Delete the set from it. `` +A test of a map, such as `connection[role] == 'output'`, is not the bare name +of the set, so it loads. + +The typeset legend prints the set as the domain of the parameter, after +`over`. For `efficiency` in the first example, the legend entry starts with +`efficiency` over $(b, e) \in \mathrm{connection},\ t \in \mathcal{T}$. In +the second example, it starts with `efficiency` over +$(p, u) \in \mathrm{connection},\ t \in \mathcal{T}$, without the maps. + ## `variables` A **variable** is a quantity that the solver decides, with one value per @@ -527,6 +658,51 @@ bounds pin a variable to one value ([fix a quantity](../../howto/pin-a-variable.md)), and a pinned variable is still a variable. +### A set in a variable's `dims` + +A variable names a [set](#a-set-in-dims) in `dims:` as a parameter does: it +exists only where the set has a row. In this example, `flow` exists at the +pairs that `connection` lists, at every snapshot: + +```yaml +dimensions: + generator: { dtype: str } + bus: { dtype: str } + snapshot: { dtype: int } +sets: + connection: { dims: [generator, bus], missing: absent } +parameters: + load: { dims: [bus, snapshot] } +variables: + flow: + dims: [connection, snapshot] + bounds: { lower: 0 } +constraints: + balance: + dims: [bus, snapshot] + expression: sum(flow, over=generator) == load +``` + +$$\mathit{flow}_{b,g,t} \ge 0 \qquad \forall\, \left( b,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T}$$ + +A variable has no data, so nothing is refused. mathspec refuses the same sets +at load as on a parameter, and a `where:` on the variable narrows inside the +set. A `missing:` number stands for the variable outside its set, as outside a +`where:`. A constraint names a set in `dims:` the same way, and has rows only +where the set has one. + +mathspec refuses a `where:` that tests the rows of a set, as on a parameter, +because `dims:` states them. For `where: connection` on `flow`, the message +is: + +```text +Variable 'flow': `where: connection` tests the rows of 'connection', which dims: states. Name it in dims: in place of the dimensions, and delete the `where:`. +``` + +Under `OR` or `NOT`, `dims:` cannot state the rows, and mathspec suggests a +`bool` parameter in their place. A [map](relations.md#maps) tested alone, such +as `connection[bus]`, asks for a label rather than a row, so it loads. + ## `objective` The **objective** is the one number that the solver minimises or maximises. diff --git a/docs/reference/language/relations.md b/docs/reference/language/relations.md index 771c3833d..8ed81d896 100644 --- a/docs/reference/language/relations.md +++ b/docs/reference/language/relations.md @@ -3,35 +3,41 @@ SPDX-FileCopyrightText: mathspec contributors SPDX-License-Identifier: CC-BY-4.0 --> -# Relations and maps +# Sets and maps -A **relation** is a set of combinations of labels of +A **set** is a set of combinations of labels of [dimensions](dimensions.md), such as the buses that each generator may connect -to. A **map** gives each row of a dimension or a relation one label of a +to. A **map** gives each row of a dimension or a set one label of a dimension, such as the bus of each generator. The file declares both, and the -rows come with the data. This page lists the keys of `relations:` and `maps:`, +rows come with the data. This page lists the keys of `sets:` and `maps:`, the data they expect, and how each operator uses them. -## `relations` +## `sets` -Each key under `relations:` declares one relation. +Each key under `sets:` declares one set. ```yaml dimensions: bus: { dtype: str } generator: { dtype: str } snapshot: { dtype: int } -relations: +sets: connection: { dims: [generator, bus], missing: absent } # the pairs that are wired neighbour: { dims: { from: snapshot, to: snapshot }, missing: absent } ``` Each row of `connection` is one pair of a generator and a bus, and a pair with -no row is not in the relation. +no row is not in the set. + +A set is what an entry is indexed over and what a sum runs over, as in +$`\sum_{(g, b) \in \mathrm{connection}}`$. So its name may stand in the +`dims:` of a parameter, a variable or a constraint, and in `over=`, for its +dimensions at the combinations it lists +([a set in `dims`](parameters-variables-constraints.md#a-set-in-dims)). | Key | What it sets | Default | | ------------- | ------------------------------------------------------------------------ | --------- | -| `dims` | required. The columns of the relation, at least two | | +| `dims` | required. The columns of the set, at least two | | | `missing` | [`refused` or `absent`](#missing): whether every combination needs a row | `refused` | | `description` | free text | `null` | @@ -43,22 +49,24 @@ dimension. With `zp: { dims: { period: zone, generator: generator } }`, mathspec prints: ```text -Relation 'zp' names column 'period' after dimension 'period', but the column is over 'zone'. Rename the column after what it holds. +Set 'zp' names column 'period' after dimension 'period', but the column is over 'zone'. Rename the column after what it holds. ``` -A relation with one column is refused, because one column holds labels of one +A set with one column is refused, because one column holds labels of one dimension, and a map or a condition states that. With `r: { dims: [generator] }`, mathspec prints: ```text -Relation 'r' has one column under 'dims:', and a relation needs at least two. To give each generator a label, declare a map under maps: generator:. To mark some of them, declare a bool parameter or a condition. +Set 'r' has one column under 'dims:', and a set needs at least two. To give each generator a label, declare a map under maps: generator:. To mark some of them, declare a bool parameter or a condition. ``` -A relation is not a dimension, so its name does not go in `dims:`, `over=` or -`along=`. With `flow: { dims: [connection] }`, mathspec prints: +A set is not one dimension, so its name does not go in `along=`, which steps +along one. A set with two columns over one dimension, such as `neighbour`, +does not go in `dims:` either, because an entry is over each dimension once. +With `flow: { dims: [neighbour] }`, mathspec prints: ```text -Variable 'flow' names the relation 'connection', and a relation is not a dimension. Name its dimensions, ['bus', 'generator'], under dims:, and confine the entry with where: connection. +Variable 'flow' names the set 'neighbour', which has two columns over one dimension, and an entry is over each dimension once. Name its dimensions, ['snapshot'], with a second name for the one it repeats. ``` The order of the columns under `dims:` changes nothing. The typesetter sorts @@ -67,7 +75,7 @@ $`\mathrm{connection} \subseteq \mathcal{B} \times \mathcal{G}`$. ## `maps` -Each key under `maps:` names a **subject**, which is a dimension or a relation. +Each key under `maps:` names a **subject**, which is a dimension or a set. Each key under the subject declares one map, which gives every row of the subject one label of the dimension that `dim:` names. @@ -98,8 +106,8 @@ typesetter prints a map as a function from its subject into that dimension, so `generator[bus]` prints as $`\mathrm{bus}: \mathcal{G} \to \mathcal{B}`$. mathspec refuses a map at load in four cases: -- The subject is neither a dimension nor a relation. -- `dim:` names a relation, or names no dimension. +- The subject is neither a dimension nor a set. +- `dim:` names a set, or names no dimension. - The map is named after a dimension that it does not refer to. - The map has the name of a column of its subject. @@ -109,27 +117,27 @@ With `generator: { period: { dim: zone } }`, mathspec prints: maps: 'generator': map 'period' is named after dimension 'period', but refers to 'zone'. Rename the map after what it holds. ``` -A relation with two columns over one dimension takes no maps, because a map +A set with two columns over one dimension takes no maps, because a map gives one label per combination of the dimensions of its subject. ### Cardinalities How many labels a row may have (the cardinality) decides whether a fact is a -map or a relation: - -| What the file states | Written | Cardinality | -| -------------------------------------------- | -------------------------------------------------------------------------------------------------------- | --------------------------- | -| each generator has one bus | `maps: { generator: { bus: { dim: bus } } }` | many-to-one | -| a generator may connect to several buses | `relations: { connection: { dims: [generator, bus], missing: absent } }` | many-to-many | -| a generator has one zone in each period | `relations: { zone_of: { dims: [generator, period] } }` and `maps: { zone_of: { zone: { dim: zone } } }` | many-to-one, from a pair | -| a snapshot has a month, a week and a weekday | `maps: { snapshot: { month: { dim: month }, week: { dim: week }, weekday: { dim: weekday } } }` | three maps of one subject | -| a line has two ends, both buses | `maps: { line: { bus0: { dim: bus }, bus1: { dim: bus } } }` | two maps into one dimension | -| a snapshot has a representative snapshot | `maps: { snapshot: { rep: { dim: snapshot } } }` | many-to-one, onto itself | -| a snapshot has neighbours | `relations: { neighbour: { dims: { from: snapshot, to: snapshot }, missing: absent } }` | many-to-many, onto itself | +map or a set: + +| What the file states | Written | Cardinality | +| -------------------------------------------- | --------------------------------------------------------------------------------------------------- | --------------------------- | +| each generator has one bus | `maps: { generator: { bus: { dim: bus } } }` | many-to-one | +| a generator may connect to several buses | `sets: { connection: { dims: [generator, bus], missing: absent } }` | many-to-many | +| a generator has one zone in each period | `sets: { zone_of: { dims: [generator, period] } }` and `maps: { zone_of: { zone: { dim: zone } } }` | many-to-one, from a pair | +| a snapshot has a month, a week and a weekday | `maps: { snapshot: { month: { dim: month }, week: { dim: week }, weekday: { dim: weekday } } }` | three maps of one subject | +| a line has two ends, both buses | `maps: { line: { bus0: { dim: bus }, bus1: { dim: bus } } }` | two maps into one dimension | +| a snapshot has a representative snapshot | `maps: { snapshot: { rep: { dim: snapshot } } }` | many-to-one, onto itself | +| a snapshot has neighbours | `sets: { neighbour: { dims: { from: snapshot, to: snapshot }, missing: absent } }` | many-to-many, onto itself | ## `missing` -`missing:` sets whether the data may leave something out. On a relation, it +`missing:` sets whether the data may leave something out. On a set, it sets whether every combination of the dimensions under `dims:` needs a row. On a map, it sets whether every row of the subject needs a label. @@ -138,7 +146,7 @@ dimensions: generator: { dtype: str } bus: { dtype: str } line: { dtype: str } -relations: +sets: connection: { dims: [generator, bus], missing: absent } # most pairs are not wired maps: generator: @@ -154,10 +162,10 @@ maps: At `l3`, the data gives `line[bus1]` no label, so `line[bus1]` is absent there. -| `missing:` | Result | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| `refused`, by default | a tool refuses the data when it is attached, and names the combination or the row | -| `absent` | a combination with no row is not in the relation, and a row with no label maps to nothing, which is [absent](absence.md) | +| `missing:` | Result | +| --------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `refused`, by default | a tool refuses the data when it is attached, and names the combination or the row | +| `absent` | a combination with no row is not in the set, and a row with no label maps to nothing, which is [absent](absence.md) | No number can replace a missing label, so at load mathspec refuses a value such as `missing: 0`: @@ -166,8 +174,8 @@ such as `missing: 0`: maps.generator.bus.missing: missing: 0 on a map, which takes refused or absent. A row with nothing there is refused, or is not in the data. ``` -Under the default `refused`, a relation needs a row for every combination, so -`connection` would need every generator wired to every bus. Most relations are +Under the default `refused`, a set needs a row for every combination, so +`connection` would need every generator wired to every bus. Most sets are sparse, so they declare `missing: absent`. The typeset legend lists `` `connection` is `absent` where the data has no row `` @@ -176,8 +184,8 @@ dimension they are over, and nothing for `refused`. ## The data contract -With data attached, each relation and each subject with maps has one table, -named after it. The table of a relation has one column per column under +With data attached, each set and each subject with maps has one table, +named after it. The table of a set has one column per column under `dims:`. The table of a subject also has one column per map, named after the map. So the data for the maps of `line` is a table `line` with the columns `line`, `bus0` and `bus1`. A piece of software that reads a spec @@ -188,38 +196,45 @@ map. So the data for the maps of `line` is a table `line` with the columns - Every label in the column of a map is a label of the dimension it refers to, and a label that matches none is refused. - The table has a row for every combination of its dimensions, unless the - relation declares `missing: absent`. The table of a dimension has a row for + set declares `missing: absent`. The table of a dimension has a row for every label of the dimension. - A null is refused, except in the column of a map that declares `missing: absent`. - The order of the rows has no meaning, because the order comes from the [dimension](dimensions.md#ordered). -A relation and a map have no `where:`, so their rows and labels come from the +A set and a map have no `where:`, so their rows and labels come from the data alone. A tool therefore knows them before it evaluates any condition, such as the [`where:` of a parameter](parameters-variables-constraints.md#where). -An [assumption](assumptions.md) can require rows of a relation or labels of a +The rows of a set can also be the coordinates of a parameter or a +variable. An entry with `connection` in its `dims:` has coordinates only at +the pairs that `connection` lists, and a tool refuses a row of the data of a +parameter at any other pair +([a set in `dims`](parameters-variables-constraints.md#a-set-in-dims)). A map +is no set to be over, because it adds no row to its subject. + +An [assumption](assumptions.md) can require rows of a set or labels of a map. For `line[bus1]` [above](#missing), `{holds: "line[bus1]", where: "closed"}` requires an end for every closed line, where `closed` is a `bool` parameter over `line`. -## How relations and maps are used +## How sets and maps are used -A relation fixes no direction, so the same table can map generators to buses +A set fixes no direction, so the same table can map generators to buses or buses to generators. A call **joins** the expression it acts on (its operand) to the table on their shared columns, and then **groups** the rows it has joined. In a call, `s[c]` names the subject `s` and its column `c`, and `s[c, …]` names several columns of one table. A column is a map of `s`, or one -of the columns of a relation under `dims:`. The subject appears once, so one +of the columns of a set under `dims:`. The subject appears once, so one call uses one table. -| Kind | What it does | Written as | -| ----------------- | ----------------------------------------------------------------------------- | -------------------------------------------------- | -| join and group by | rows of the operand match rows of the table, and each group is added up | `sum(x, over=d, by=s[c])` | -| join | each row of the operand matches one row of the table, and takes its label | `at(x, by=s[m])` | -| partition | the dimensions stay, and the rows are grouped | `shift`, `sum_back`, `position` with `within=s[m]` | -| test | a coordinate stays where the relation has a row, or where the map has a label | `r` or `s[m]` in a condition | +| Kind | What it does | Written as | +| ----------------- | ------------------------------------------------------------------------- | -------------------------------------------------- | +| join and group by | rows of the operand match rows of the table, and each group is added up | `sum(x, over=d, by=s[c])` | +| join | each row of the operand matches one row of the table, and takes its label | `at(x, by=s[m])` | +| partition | the dimensions stay, and the rows are grouped | `shift`, `sum_back`, `position` with `within=s[m]` | +| test | a coordinate stays where the set has a row, or where the map has a label | `r` or `s[m]` in a condition | Four rules apply to every call: @@ -260,8 +275,8 @@ Each column of the table is either **joined on** or not, and either | a key column the call does not list | yes | yes | kept | | a map the call does not list | no | no | not used | -With the relation `zone_of` over `[generator, period]` and its map `zone`, the -relation `connection` over `[generator, bus]`, and a variable `p` over +With the set `zone_of` over `[generator, period]` and its map `zone`, the +set `connection` over `[generator, bus]`, and a variable `p` over `[generator, period]`, three sums have three results: | Call | Joins on | Groups by | Result | @@ -324,7 +339,7 @@ it at `south`. Three rules apply to a lookup: - A lookup reads only maps. A key column in `by=` is refused, and so is a - relation with no map, where one combination can have several rows. + set with no map, where one combination can have several rows. - The result gains every key column that the operand is not already over. A key column over a dimension that the operand is over is joined on too, unless a map in `by=` already refers to that dimension. With `price` over @@ -359,7 +374,7 @@ The first snapshot of each season takes the value of the last snapshot of the same season, and no value crosses from winter to summer. `within=` may list two maps into one dimension, but not a key column. A -relation with no map makes no groups, so mathspec refuses it in `within=`: +set with no map makes no groups, so mathspec refuses it in `within=`: ```text Constraint 'd': shift(within=sp[period]): 'sp' has no map, so it makes no groups. Declare the group as a map of the dimension shift steps along, under maps: snapshot:. @@ -367,7 +382,7 @@ Constraint 'd': shift(within=sp[period]): 'sp' has no map, so it makes no groups ### Tests -A condition tests a relation or a map at the coordinates of its own entry. It +A condition tests a set or a map at the coordinates of its own entry. It compares a map, written `s[m]`, against a label, compares two maps, or tests that a row or a label exists. [Conditions](expressions.md#conditions) lists when two maps may be compared. A comparison reads one map, so @@ -381,23 +396,23 @@ Expected end of text, found '[' (at char 8), (line:1, col:9) ``` A map alone in a condition, such as `line[bus1]`, is true where the map gives -the row a label. A relation name alone is true where the relation has a row. -On a variable, the condition leaves the variable absent at every pair that the -relation does not list: +the row a label. A set name alone is true where the set has a row. +On a parameter, a variable or a constraint, the set in `dims:` states those +rows, so mathspec refuses the name in its `where:`. Over the set, a variable is +absent at every pair that the set does not list: ```yaml title="flow.yaml" dimensions: generator: { dtype: str } bus: { dtype: str } snapshot: { dtype: int } -relations: +sets: connection: { dims: [generator, bus], missing: absent } # the pairs that are wired parameters: load: { dims: [bus, snapshot] } variables: flow: - dims: [generator, bus, snapshot] - where: connection + dims: [connection, snapshot] bounds: { lower: 0 } constraints: balance: @@ -409,8 +424,14 @@ constraints: --8<-- "docs/static/relations-where-bare.svg" -`flow` exists at the pairs that `connection` lists and is [absent](absence.md) -at every other pair, so `balance` adds up the generators wired to each bus. +`flow` exists at the pairs that `connection` lists and is +[absent](absence.md) at every other pair, so `balance` adds up the generators +wired to each bus. `where: connection` beside `dims: [generator, bus, snapshot]` +is refused: + +```text +Variable 'flow': `where: connection` tests the rows of 'connection', which dims: states. Name it in dims: in place of the dimensions, and delete the `where:`. +``` `python -m mathspec markdown flow.yaml` prints the domain of `flow` and the constraint: @@ -429,5 +450,5 @@ $`\mathrm{bus1}(l) \text{ is defined}`$. !!! info "See also" - - [relations as linear maps](../../about/relations-as-linear-maps.md) for the + - [sets as linear maps](../../about/relations-as-linear-maps.md) for the same calls written as matrices diff --git a/docs/reference/notation.md b/docs/reference/notation.md index ab078981c..27a9740e2 100644 --- a/docs/reference/notation.md +++ b/docs/reference/notation.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: CC-BY-4.0 This page lists each construct of the language beside the math that [the typesetter](typeset.md) prints for it. Each section has the math of one -kind of construct, and the legend lists the symbol of each dimension, relation and parameter. +kind of construct, and the legend lists the symbol of each dimension, set and parameter. Each construct is printed as its YAML and then its equation. Most of the YAML comes from one test spec, @@ -35,9 +35,10 @@ dimensions: technology: { dtype: str } bp: { dtype: int, ordered: true } # the breakpoints every curve below runs through -relations: +sets: gen_zone: { dims: [generator, snapshot] } # a relation over two dimensions, with a map: a call sums one away and joins on the other connection: { dims: [generator, bus], missing: absent } # a relation with no map: many-to-many, read only by sum with both ends named + bus_zone: { dims: [bus, zone], missing: absent } # the pairs that exist maps: generator: { bus: { dim: bus }, technology: { dim: technology } } # two maps of one subject, read to both at once @@ -57,7 +58,8 @@ parameters: is_flexible: { dims: [generator], dtype: bool, missing: false } zone_cap: { dims: [zone] } tech_cap: { dims: [bus, technology], missing: absent } - link_eff: { dims: [generator, bus], where: connection, missing: 1 } + link_eff: { dims: [connection], missing: 1 } # over the pairs a relation lists; a `where:` may not test them + zone_share: { dims: [bus_zone], missing: 1 } # over the pairs that exist: data refused outside them, with no `where:` of its own area_weight: { dims: [bus], where: "bus[area]" } # a map's label as a condition: the buses that have an area min_up: { dims: [generator], dtype: int } eta: { dims: [generator] } # a Greek name that is *given*, so the rule wins and it prints as the word @@ -75,12 +77,12 @@ parameters: | Symbol | Meaning | |---|---| | $`\mathcal{B}`$ | index $`b`$ — `bp` | -| $`\mathcal{U}`$ | index $`u`$ — `bus` with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}`$, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | +| $`\mathcal{U}`$ | index $`u`$ — `bus` with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}`$, `bus_zone` is `absent` where the data has no row, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | | $`\mathcal{G}`$ | index $`g`$ — `generator` with $`\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}`$, `connection` is `absent` where the data has no row | | $`\mathcal{S}`$ | index $`s`$ — `season` with $`\mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | | $`\mathcal{T}`$ | index $`t`$ — `snapshot` (`int` coordinates) with $`\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | | $`\mathcal{E}`$ | index $`e`$ — `technology` with $`\mathrm{technology}: \mathcal{G} \to \mathcal{E}`$ | -| $`\mathcal{Z}`$ | index $`z`$ — `zone` (labels `north`, `south`) with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}`$, `bus[area]` is `absent` where the data gives no label | +| $`\mathcal{Z}`$ | index $`z`$ — `zone` (labels `north`, `south`) with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}`$, `bus_zone` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | #### Parameters @@ -97,13 +99,14 @@ parameters: | $`\mathrm{growth}`$ | `growth` (scalar) | | $`\mathrm{is\_flexible}`$ | `is_flexible` over $`\mathcal{G}`$, `false` where the data has no row | | $`\mathrm{lead}`$ | `lead` over $`\mathcal{G}`$ | -| $`\mathrm{link\_eff}`$ | `link_eff` over $`\mathcal{U} \times \mathcal{G}`$, only where $`\left( u,\ g \right) \in \mathrm{connection}`$, $`1`$ where the data has no row | +| $`\mathrm{link\_eff}`$ | `link_eff` over $`\left( u,\ g \right) \in \mathrm{connection}`$, $`1`$ where the data has no row | | $`\mathrm{load}`$ | `load` over $`\mathcal{U} \times \mathcal{T}`$ | | $`\mathrm{min\_up}`$ | `min_up` over $`\mathcal{G}`$ | | $`\mathrm{p}^{\mathrm{max}}`$ | `p_max` over $`\mathcal{G}`$, $`\infty`$ where the data has no row | | $`\mathrm{p}^{\mathrm{min}}`$ | `p_min` over $`\mathcal{G}`$ | | $`\mathrm{tech\_cap}`$ | `tech_cap` over $`\mathcal{U} \times \mathcal{E}`$, `absent` where the data has no row | | $`\mathrm{zone\_cap}`$ | `zone_cap` over $`\mathcal{Z}`$ | +| $`\mathrm{zone\_share}`$ | `zone_share` over $`\left( u,\ z \right) \in \mathrm{bus\_zone}`$, $`1`$ where the data has no row | #### Conditions @@ -116,6 +119,7 @@ parameters: | Symbol | Meaning | |---|---| +| $`\mathit{flow}`$ | `flow` over $`\left( u,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T}`$ | | $`\mathit{fuel}`$ | `fuel` over $`\mathcal{G} \times \mathcal{T}`$ | | $`\mathit{headroom}`$ | `headroom` (scalar), $`0`$ where the mask leaves it out | | $`\mathit{heat}`$ | `heat` over $`\mathcal{G} \times \mathcal{T}`$ | @@ -288,6 +292,22 @@ variables: \mathit{headroom} \ge 0 \qquad \text{where } \mathrm{budget} \text{ is defined} ``` +#### Variable over the rows of a relation + +a domain of the pairs `connection` lists, at every snapshot, and a `where:` inside it + +```yaml +variables: + flow: + dims: [snapshot, connection] + where: "p_max > 0" + bounds: { lower: 0 } +``` + +```math +\mathit{flow}_{u,g,t} \ge 0 \qquad \forall\, \left( u,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T} \,:\, \mathrm{p}^{\mathrm{max}}_{g} > 0 +``` + #### Variable in a special ordered set the family a sos runs along @@ -488,13 +508,12 @@ constraints: #### Relation as a condition -a relation as a where: the row of the frame has to be a member of the relation +a relation in dims: the row of the frame has to be a member of the relation ```yaml constraints: connected: - dims: [snapshot, generator, bus] - where: "connection" + dims: [snapshot, connection] expression: p <= load ``` diff --git a/docs/reference/reading.md b/docs/reference/reading.md index 62edcc9ab..280250a63 100644 --- a/docs/reference/reading.md +++ b/docs/reference/reading.md @@ -42,7 +42,7 @@ the file changes nothing: - the entries of each section, and of each kind under `given:` - each `dims:` list -- the columns of each relation, and the maps of each subject +- the columns of each set, and the maps of each subject - the cases under `cases:` - the `kwargs:` of a macro - the labels under `one_of:` @@ -164,9 +164,9 @@ the colon is the description that the file sets. `program.parameters[name].missing` is what the file sets under [`missing:`](language/absence.md#missing) for a missing row of that parameter. It is `'refused'`, `'absent'`, or the value that a tool uses for the row. -`program.relations` has one table per relation and per subject with +`program.sets` has one table per set and per subject with [maps](language/relations.md#the-data-contract), named after it, and each map -is one of its `values`. `program.relations[name].missing` is `'refused'` or +is one of its `values`. `program.sets[name].missing` is `'refused'` or `'absent'`, and `partial` lists the maps under `missing: absent`. The program of `spec.expand(...)` has the same `missing` values. @@ -185,13 +185,29 @@ comparison in a condition. Under `'absent'`, a tool leaves a bound open where the parameter has no row. `program.parameters[name].where` is a [`Mask`](#nodes-and-conditions) for the -`where:` of the parameter, or `None` when the parameter has every coordinate -of its `dims`. With data attached, an engine applies the condition before -`missing:`, as +coordinates of the parameter, or `None` when the parameter has every +coordinate of its `dims`. It contains each set named in the `dims:` of the +parameter and its `where:`. With data attached, an engine applies the +condition before `missing:`, as [`where` on a parameter](language/parameters-variables-constraints.md#where) describes. The condition contains no variable and no parameter with a `where:` -of its own. So an engine can evaluate every such condition before it reads any -parameter that has a `where:`. +or a set in its `dims:`. So an engine can evaluate every such condition before +it reads any parameter that has one. + +`program.parameters[name].in_` lists the sets the file named in the `dims:` +of the parameter, sorted by name. `program.parameters[name].dims` holds their +dimensions in their place. With data attached, an engine refuses a row of the parameter at a +combination that one of them has no row for, and reads no map. It does this +before it applies the `where`, and so before `missing:`, as +[a set in `dims`](language/parameters-variables-constraints.md#a-set-in-dims) describes. An engine +drops a row outside the rest of the `where`, and refuses a row outside the +sets. + +`program.variables[name].where` is the same kind of `Mask` for the coordinates +of the variable, with the sets of its `dims:` in it, and +`program.variables[name].in_` lists those sets. A variable has no data, so +an engine builds it where the `where` is true and needs nothing more from +`in_`. With data attached, an engine must refuse a divisor that is `0` in a row it builds, and name the divisor. The same applies to a term of a sum that it @@ -223,7 +239,7 @@ An `ExpressionReference` is the node for a use of an `expressions:` entry. Its Every expression node has `dims`, the dimensions that it is indexed over, sorted by name. A sum has the dimensions of its operand, less the ones it sums -over. A `Join` under a sum through a relation has the dimensions of that sum. +over. A `Join` under a sum through a set has the dimensions of that sum. A `Join` also opens an index per dropped column (an `Axis`). An axis is not a dimension and is not in `dims`. An `ExpressionReference` has the dimensions of its body. @@ -256,7 +272,7 @@ A `Region`'s `when` is a `Mask` too, and every `Mask` has five attributes: A comparison of expressions arrives as an `ExpressionComparison`. Its two sides are program expressions, as a constraint's are, and its `dims` are every dimension that either side has. Its `names_read` are every parameter and -relation that the sides use, including the relation of a grouping. +set that the sides use, including the set of a grouping. A name compared against a literal is a different node. `p_max > 5` is a `ParameterComparison` and `1 * p_max > 5` is an `ExpressionComparison`, though @@ -265,7 +281,7 @@ both are true at the same coordinates. A comparison of a map against a label is a `RelationComparison`, and a comparison of two maps is a `RelationPairComparison`. The page on [conditions](language/expressions.md#conditions) defines both, and both are false for every operator where the map gives no label. A -relation name alone is a `RelationDefined`, and a map alone, `subject[map]`, is a +set name alone is a `RelationDefined`, and a map alone, `subject[map]`, is a `RelationDefined` with `column` set to the map. Three conditions contain another condition rather than an entry. A @@ -445,7 +461,7 @@ Writing the canonical form out again gives the same text. The canonical form has these rules: - The sections come in one fixed order: `version`, `description`, `given`, - `dimensions`, `relations`, `maps`, `parameters`, `conditions`, `variables`, + `dimensions`, `sets`, `maps`, `parameters`, `conditions`, `variables`, `expressions`, `objective`, `constraints`, `sos`, `piecewise`, `assumptions`, `macros`. The keys of an entry also come in one order. - Entries are sorted by name within each section, and so is every other list diff --git a/docs/static/relations-where-bare.svg b/docs/static/relations-where-bare.svg index 9d1806b52..18a36f0a2 100644 --- a/docs/static/relations-where-bare.svg +++ b/docs/static/relations-where-bare.svg @@ -3,8 +3,8 @@ SPDX-FileCopyrightText: mathspec contributors SPDX-License-Identifier: CC-BY-4.0 --> - A bare relation in a where condition keeps the pairs it lists - The bare relation connection is a table with three rows: wind and north, gas and north, and gas and south. The variable flow, declared with where connection, is a table with the same three pairs of generator and bus. The pair wind and south, which connection does not list, has no row, so flow is absent there. + A set in dims gives the pairs it lists + The set connection is a table with three rows: wind and north, gas and north, and gas and south. The variable flow, declared over connection in its dims, is a table with the same three pairs of generator and bus. The pair wind and south, which connection does not list, has no row, so flow is absent there. @@ -36,8 +36,8 @@ SPDX-License-Identifier: CC-BY-4.0 - where: connection - keeps the pairs + dims: [connection, …] + gives the pairs the relation lists diff --git a/docs/tutorials/first-spec.md b/docs/tutorials/first-spec.md index bb3ad7c45..6f81c3b10 100644 --- a/docs/tutorials/first-spec.md +++ b/docs/tutorials/first-spec.md @@ -46,8 +46,8 @@ Check the file: ``` ```text -dimension 'generator' is never used: nothing is indexed by it, nothing sums into it, and no relation or map is over it. Remove it, or keep it if the entries that use it are still to be written. -dimension 'snapshot' is never used: nothing is indexed by it, nothing sums into it, and no relation or map is over it. Remove it, or keep it if the entries that use it are still to be written. +dimension 'generator' is never used: nothing is indexed by it, nothing sums into it, and no set or map is over it. Remove it, or keep it if the entries that use it are still to be written. +dimension 'snapshot' is never used: nothing is indexed by it, nothing sums into it, and no set or map is over it. Remove it, or keep it if the entries that use it are still to be written. ``` mathspec accepts the file. `check` prints one remark that is not a refusal (a line of diff --git a/docs/tutorials/storage.md b/docs/tutorials/storage.md index 4df67b0b1..b33451358 100644 --- a/docs/tutorials/storage.md +++ b/docs/tutorials/storage.md @@ -44,8 +44,8 @@ Check the file: ``` ```text -dimension 'snapshot' is never used: nothing is indexed by it, nothing sums into it, and no relation or map is over it. Remove it, or keep it if the entries that use it are still to be written. -dimension 'store' is never used: nothing is indexed by it, nothing sums into it, and no relation or map is over it. Remove it, or keep it if the entries that use it are still to be written. +dimension 'snapshot' is never used: nothing is indexed by it, nothing sums into it, and no set or map is over it. Remove it, or keep it if the entries that use it are still to be written. +dimension 'store' is never used: nothing is indexed by it, nothing sums into it, and no set or map is over it. Remove it, or keep it if the entries that use it are still to be written. ``` mathspec accepts the file. `check` prints one remark that is not a refusal (a line of diff --git a/examples/connection.yaml b/examples/connection.yaml index 37ef20837..092b37746 100644 --- a/examples/connection.yaml +++ b/examples/connection.yaml @@ -11,7 +11,7 @@ dimensions: port: { dtype: int, description: "an attachment of a unit, numbered as PyPSA's bus0, bus1, ..." } role: { description: input or output } -relations: +sets: connection: description: the bus each port of a unit attaches to, and whether the port draws or delivers dims: [unit, port] diff --git a/examples/operators/sum_by_column_lists.yaml b/examples/operators/sum_by_column_lists.yaml index ca61c040d..907aeaf2d 100644 --- a/examples/operators/sum_by_column_lists.yaml +++ b/examples/operators/sum_by_column_lists.yaml @@ -12,7 +12,7 @@ dimensions: bus: { dtype: str } technology: { dtype: str } -relations: +sets: slot_of: { dims: [generator, period] } maps: diff --git a/examples/operators/sum_by_columns.yaml b/examples/operators/sum_by_columns.yaml index e417f783c..2f83d345b 100644 --- a/examples/operators/sum_by_columns.yaml +++ b/examples/operators/sum_by_columns.yaml @@ -13,7 +13,7 @@ dimensions: period: { dtype: int } zone: { dtype: str } -relations: +sets: zone_of: { dims: [generator, period] } maps: diff --git a/examples/pypsa.yaml b/examples/pypsa.yaml index bf6655374..6882e6d6c 100644 --- a/examples/pypsa.yaml +++ b/examples/pypsa.yaml @@ -1581,7 +1581,7 @@ parameters: dims: [global_constraint, process] missing: absent -relations: +sets: Generator_maintenance_cover: description: >- the snapshots a maintenance event covers, by the snapshot it starts in, diff --git a/examples/pypsa/generator_maintenance.yaml b/examples/pypsa/generator_maintenance.yaml index 6bc608280..919358ed2 100644 --- a/examples/pypsa/generator_maintenance.yaml +++ b/examples/pypsa/generator_maintenance.yaml @@ -28,7 +28,7 @@ dimensions: generator: description: generating units, each on one bus -relations: +sets: Generator_maintenance_cover: description: >- the snapshots a maintenance event covers, by the snapshot it starts in, diff --git a/examples/pypsa/link_maintenance.yaml b/examples/pypsa/link_maintenance.yaml index 98c5ed585..5dad5d540 100644 --- a/examples/pypsa/link_maintenance.yaml +++ b/examples/pypsa/link_maintenance.yaml @@ -28,7 +28,7 @@ dimensions: link: description: controllable connections, each from one bus to the buses it delivers to -relations: +sets: Link_maintenance_cover: description: >- the snapshots a maintenance event covers, by the snapshot it starts in, diff --git a/examples/pypsa/process_maintenance.yaml b/examples/pypsa/process_maintenance.yaml index 15533b742..6b44c2819 100644 --- a/examples/pypsa/process_maintenance.yaml +++ b/examples/pypsa/process_maintenance.yaml @@ -28,7 +28,7 @@ dimensions: process: description: generalised multi-port converters, each with an internal power that every port draws or delivers at its own rate -relations: +sets: Process_maintenance_cover: description: >- the snapshots a maintenance event covers, by the snapshot it starts in, diff --git a/schema/mathspec.schema.json b/schema/mathspec.schema.json index 31392b32f..d1b5477aa 100644 --- a/schema/mathspec.schema.json +++ b/schema/mathspec.schema.json @@ -882,9 +882,9 @@ "title": "PiecewiseSpec", "type": "object" }, - "RelationSpec": { + "SetSpec": { "additionalProperties": false, - "description": "A named relation: the combinations of its dimensions that exist, one row each.\n\n``dims:`` is a list of dimensions, or a mapping of column name to\ndimension where two columns share one. A row is its own identity, so the\nrelation is a subset of the product of its dimensions::\n\n relations:\n connection: {dims: [unit, port], missing: absent}\n corridor: {dims: {from: bus, to: bus}}\n\nA relation is not a dimension. Its name stands in a condition, where it\nis true at the rows it lists, and before a\nbracket, where it reads one of its maps (``maps:``, [`MapSpec`][]). The\nrows are data, and arrive under the relation's name, one column per\ncolumn name, with a column per map beside them.", + "description": "A named relation: the combinations of its dimensions that exist, one row each.\n\n``dims:`` is a list of dimensions, or a mapping of column name to\ndimension where two columns share one. A row is its own identity, so the\nrelation is a subset of the product of its dimensions::\n\n sets:\n connection: {dims: [unit, port], missing: absent}\n corridor: {dims: {from: bus, to: bus}}\n\nA relation is not a dimension. Its name stands under an entry's ``in:``,\nin a condition, where it is true at the rows it lists, and before a\nbracket, where it reads one of its maps (``maps:``, [`MapSpec`][]). The\nrows are data, and arrive under the relation's name, one column per\ncolumn name, with a column per map beside them.", "properties": { "description": { "anyOf": [ @@ -931,7 +931,7 @@ "required": [ "dims" ], - "title": "RelationSpec", + "title": "SetSpec", "type": "object" }, "SosSpec": { @@ -1152,12 +1152,12 @@ "title": "Piecewise", "type": "object" }, - "relations": { + "sets": { "additionalProperties": { - "$ref": "#/$defs/RelationSpec" + "$ref": "#/$defs/SetSpec" }, "default": {}, - "title": "Relations", + "title": "Sets", "type": "object" }, "sos": { diff --git a/src/mathspec/_advising.py b/src/mathspec/_advising.py index 036e683b9..5bf6ccd3f 100644 --- a/src/mathspec/_advising.py +++ b/src/mathspec/_advising.py @@ -35,7 +35,8 @@ def advice(spec: str | Path | Mapping[str, object] | Spec | Program) -> tuple[Ad Returns: The never-an-axis advice sorted by dimension name, then one note per - entry the program reads and does not build, then the + entry the program reads and does not build, then one per variable + whose ``where:`` names a relation ``in:`` could name, then the unboundedness advice; ``str()`` of each is its sentence. Raises: @@ -116,7 +117,7 @@ def _never_an_axis(program: Program) -> list[Advice]: 'never-an-axis', name, f"dimension '{name}' is never used: nothing is indexed by it, nothing " - f'sums into it, and no relation or map is over it. Remove it, or keep it ' + f'sums into it, and no set or map is over it. Remove it, or keep it ' f'if the entries that use it are still to be written.', ) for name in program.dimensions diff --git a/src/mathspec/_composition.py b/src/mathspec/_composition.py index 6bf124783..d3397651e 100644 --- a/src/mathspec/_composition.py +++ b/src/mathspec/_composition.py @@ -116,11 +116,11 @@ from mathspec.program import Program from mathspec.spec import DimensionSpec -SHARED_SECTIONS = ('dimensions', 'relations', 'maps') +SHARED_SECTIONS = ('dimensions', 'sets', 'maps') SETTINGS: dict[str, frozenset[str]] = { 'dimensions': frozenset({'ordered'}), - 'relations': frozenset({'missing'}), + 'sets': frozenset({'missing'}), 'maps': frozenset({'missing'}), } """The fields of a dimension, a relation or a map, other than ``description``, that a patch may change. A patch may @@ -191,7 +191,7 @@ def merge(fragments: Sequence[Source], description: str | None = None) -> Spec: merged: dict[str, object] = {'version': _one_version(read)} if description is not None: merged['description'] = description - for section in ('dimensions', 'relations'): + for section in ('dimensions', 'sets'): if agreed := _agreed(read, section, _singular(section), 'give one of them a name of its own'): merged[section] = agreed flat: dict[str, dict[str, object]] = { diff --git a/src/mathspec/_condition_parser.py b/src/mathspec/_condition_parser.py index 66d30e6dd..b51e562cb 100644 --- a/src/mathspec/_condition_parser.py +++ b/src/mathspec/_condition_parser.py @@ -251,7 +251,7 @@ def _named_rewrite(text: str, loc: int) -> str | None: _DEEP_REWRITE = ( - 'Declare a parameter or relation carrying part of the test and name that here, or split the ' + 'Declare a parameter or set carrying part of the test and name that here, or split the ' 'entry into two, each masked by one half.' ) diff --git a/src/mathspec/_condition_resolver.py b/src/mathspec/_condition_resolver.py index a133446fe..2a67ff4d7 100644 --- a/src/mathspec/_condition_resolver.py +++ b/src/mathspec/_condition_resolver.py @@ -157,6 +157,7 @@ def _bare_name(self, node: NameNode) -> Condition | UnresolvedConditionNode: and declared.missing == 'refused' and declared.where is None and declared.dtype in ('int', 'str') + and not declared.in_ ): rewrite = ( f'Declare `missing: absent`, or compare the label: where: "{node.name} == \'...\'".' @@ -552,13 +553,13 @@ def _relation_column(self, name: str, column: str | None, spelling: str, op: Com shape = ns.relations[name] if not shape.values: self.errors.append( - f"{context}: '{spelling}' compares the relation '{name}', which has no map, so its rows hold no " + f"{context}: '{spelling}' compares the set '{name}', which has no map, so its rows hold no " f"label. Test the bare name '{name}' for whether a row exists, or declare a map under maps: {name}:." ) return None if column is None: self.errors.append( - f"{context}: '{spelling}' compares the relation '{name}' itself, and a comparison reads a label. " + f"{context}: '{spelling}' compares the set '{name}' itself, and a comparison reads a label. " f'Name one of its maps: {name}[{shape.values[0]}].' ) return None diff --git a/src/mathspec/_dimensions.py b/src/mathspec/_dimensions.py index 84eb13c20..b2b561c9f 100644 --- a/src/mathspec/_dimensions.py +++ b/src/mathspec/_dimensions.py @@ -78,7 +78,7 @@ def _check_sum(node: Sum, inner: frozenset[str], context: str) -> None: every one, and the join's own rules are the sum's. """ if isinstance(node.operand, Join) and node.operand.columns.axes: - assert node.over == node.operand.columns.axes, 'a sum through a relation closes exactly the axes its join opens' + assert node.over == node.operand.columns.axes, 'a sum through a set closes exactly the axes its join opens' return for summed in node.over: if summed.dimension not in inner: @@ -171,7 +171,7 @@ def _check_joined(call: str, use: JoinColumns | Partition, inner: frozenset[str] if twice := sorted({d for d in dims if dims.count(d) > 1}): raise DimensionError( f"{context}: {call} joins '{use.name}' on {twice} through more than one column. Join on " - f'different dimensions, or use a relation whose columns are over different dimensions.' + f'different dimensions, or use a set whose columns are over different dimensions.' ) @@ -347,7 +347,7 @@ def _check_condition_dims( case DimensionComparison() | DimensionPosition(): leaf = f"where-dimension '{atom.name}'" case RelationDefined(column=None): - leaf = f"where-relation '{atom.name}'" + leaf = f"where-set '{atom.name}'" case RelationComparison() | RelationPairComparison() | RelationDefined(): leaf = f"where-map '{atom.name}[{atom.column}]'" case ExpressionComparison(): diff --git a/src/mathspec/_exclusivity.py b/src/mathspec/_exclusivity.py index 8dc194e70..6766fd772 100644 --- a/src/mathspec/_exclusivity.py +++ b/src/mathspec/_exclusivity.py @@ -375,7 +375,7 @@ def _subject_of(node: TypedCondition) -> Subject: case TranslatedCondition(): return Subject('expression', 'a condition read at a neighbouring coordinate') case JoinedCondition(): - return Subject('expression', 'a condition read through a relation') + return Subject('expression', 'a condition read through a set') case _: assert_never(node) @@ -598,7 +598,7 @@ def _atom(node: TypedCondition, cell: dict[Subject, Cell], grid: _Grid) -> bool: raise AssertionError(msg) case CountComparison() | TranslatedCondition() | JoinedCondition(): msg = ( - 'a condition read as a count, at a neighbour or through a relation is refused as undecidable ' + 'a condition read as a count, at a neighbour or through a set is refused as undecidable ' 'before any cell is read' ) raise AssertionError(msg) diff --git a/src/mathspec/_expansion.py b/src/mathspec/_expansion.py index 6915cb8be..aac58f979 100644 --- a/src/mathspec/_expansion.py +++ b/src/mathspec/_expansion.py @@ -163,7 +163,7 @@ def _substitute_columns(node: ColumnsNode, bindings: dict[str, ArithmeticNode], if bound is not None and not isinstance(bound, NameNode): msg = ( f"{caller}: the formal '{name}' is inside {node}, where only a name is allowed, and the call " - f'passes {bound}. Pass the name of a relation or of one of its columns.' + f'passes {bound}. Pass the name of a set or of one of its columns.' ) raise SchemaError(msg) names.append(name if bound is None else bound.name) diff --git a/src/mathspec/_expression_resolver.py b/src/mathspec/_expression_resolver.py index cd4e9a8e6..ecc49316a 100644 --- a/src/mathspec/_expression_resolver.py +++ b/src/mathspec/_expression_resolver.py @@ -208,7 +208,7 @@ def _name(self, node: NameNode) -> Expression | None: return None case 'relation': self.errors.append( - f"{self.context}: '{node.name}' is a relation, and a relation is not a value. Sum through it " + f"{self.context}: '{node.name}' is a set, and a set is not a value. Sum through it " f'as sum(x, over=, by={node.name}[]), test it in a condition, or declare a ' f'parameter to carry numbers.' ) @@ -502,7 +502,7 @@ def _over(self, value: ArithmeticNode, operator: str, *, relation: bool) -> tupl f"'{value.relation}', or a dimension." ) return None - names = names_in(value) + names = names_in(value) if relation else _members_of_sets(names_in(value), self.ns) if not names: self.errors.append( f'{self.context}: {operator}(over=...) is not a name. Write a declared dimension, or a list of them.' @@ -709,9 +709,9 @@ def not_a_relation(self, name: str, operator: str, key: str) -> str | None: f'under maps: {name}:, or read a subject that has it: {key}=[].' ) kinds = ( - 'a relation, a dimension with maps or a formal argument of this macro' + 'a set, a dimension with maps or a formal argument of this macro' if self.formals - else 'a relation or a dimension with maps' + else 'a set or a dimension with maps' ) return ( f'{context}: {operator}({key}={name}[...]): {name!r} is not {kinds}. ' @@ -733,13 +733,26 @@ def not_a_number(name: str, dtype: str, context: str) -> str: return f"{context}: '{name}' is declared dtype: {dtype}, and an expression reads only float and int. {instead}" +def _members_of_sets(names: tuple[str, ...], ns: Namespace) -> tuple[str, ...]: + """*names* with each relation over distinct dimensions replaced by them, as in an entry's ``dims:``. + + A relation with two columns over one dimension stays a name, which the + dimension check then refuses. + """ + out: list[str] = [] + for name in names: + relation = None if name in ns.dimensions else ns.spec.sets.get(name) + dims: list[str] = [d for _, d in relation.pairs] if relation is not None else [] + out.extend(dims if dims and len(set(dims)) == len(dims) else [name]) + return tuple(out) + + def _undeclared_dim(context: str, operator: str, call: str, name: str, ns: Namespace, formals: frozenset[str]) -> str: if name in ns.relations and name not in ns.dimensions: dims = list(ns.relations[name].dims) which = 'its dimensions' if call.startswith('over=') else 'one of its dimensions' return ( - f"{context}: {operator}({call}) names the relation '{name}', and a relation is not a dimension. " - f'Name {which}, {dims}.' + f"{context}: {operator}({call}) names the set '{name}', and a set is not a dimension. Name {which}, {dims}." ) return ( f'{context}: {operator}({call}) does not name a declared dimension{_or_a_formal(formals)}. ' diff --git a/src/mathspec/_lowering.py b/src/mathspec/_lowering.py index 3bb1cd912..3c1f4d146 100644 --- a/src/mathspec/_lowering.py +++ b/src/mathspec/_lowering.py @@ -15,6 +15,7 @@ from __future__ import annotations +import functools from typing import TYPE_CHECKING from mathspec._dimensions import check_spec @@ -41,6 +42,7 @@ from mathspec._validation import emitted_name_errors, reference_errors from mathspec.errors import SchemaError, did_you_mean, prefixed from mathspec.program import ( + And, Assumption, BooleanLiteral, Constant, @@ -57,6 +59,7 @@ Parameter, ParameterReference, Program, + RelationDefined, Sos, Variable, VariableDefined, @@ -67,7 +70,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Mapping - from mathspec.program import Expression, Gate, JoinColumns + from mathspec.program import Condition, Expression, Gate, JoinColumns from mathspec.spec import AssumptionSpec, Spec @@ -149,8 +152,9 @@ def lower(spec: Spec) -> Program: variables = {} for vname, vdef in spec.variables.items(): - where = resolve_condition_text(vdef.where, ns, f"Variable '{vname}'", errors, self_variable=vname) - if where is not None and (refusal := self_existence(where, vname, f"Variable '{vname}'")) is not None: + context = f"Variable '{vname}'" + where = resolve_condition_text(vdef.where, ns, context, errors, self_variable=vname) + if where is not None and (refusal := self_existence(where, vname, context)) is not None: errors.append(refusal) if vdef.domain == 'binary': lower_bound, upper_bound = Constant(0.0, dims=()), Constant(1.0, dims=()) @@ -158,12 +162,13 @@ def lower(spec: Spec) -> Program: lower_bound, upper_bound = _bound(vdef.bounds.lower, ns), _bound(vdef.bounds.upper, ns) variables[vname] = Variable( vdef.dims, - where=mask_of(where), + where=_index_mask(vname, vdef.in_, vdef.where, where, ns, context, errors), lower=lower_bound, upper=upper_bound, domain=vdef.domain, missing=vdef.missing, description=vdef.description, + in_=vdef.in_, ) constraints: dict[str, Constraint] = {} @@ -172,7 +177,8 @@ def lower(spec: Spec) -> Program: where = resolve_condition_text(cdef.where, ns, context, errors) if (sides := resolve_constraint_text(cdef.expression, ns, context, errors)) is not None: lhs, sense, rhs = sides - constraints[cname] = Constraint(cdef.dims, lhs, sense, rhs, mask_of(where), description=cdef.description) + mask = _index_mask(cname, cdef.in_, cdef.where, where, ns, context, errors) + constraints[cname] = Constraint(cdef.dims, lhs, sense, rhs, mask, description=cdef.description) objective = None if spec.objective is not None: @@ -223,7 +229,14 @@ def lower(spec: Spec) -> Program: program = Program( parameters={ - name: Parameter(pdef.dims, pdef.dtype, pdef.missing, pdef.description, parameter_masks.get(name)) + name: Parameter( + pdef.dims, + pdef.dtype, + pdef.missing, + pdef.description, + parameter_masks.get(name), + pdef.in_, + ) for name, pdef in spec.parameters.items() }, variables=variables, @@ -384,34 +397,116 @@ def _bound(value: float | str | None, ns: Namespace) -> Constant | ParameterRefe def _parameter_masks(spec: Spec, ns: Namespace, errors: list[str]) -> dict[str, Mask | None]: - """Each parameter's ``where:`` typed, a refusal appended for each mask that reads what it may not. + """Each parameter's ``in:`` and ``where:`` as one mask, a refusal appended for each ``where:`` that reads what it may not. A tool evaluates a parameter's mask from the data alone, before any variable has a value and before any mask has been applied. So a mask reads - dimensions, relations and unmasked parameters, and no masks can wait on - one another. + dimensions, relations and parameters with neither ``in:`` nor ``where:``, + and no masks can wait on one another. """ - masked = {name for name, pdef in spec.parameters.items() if pdef.where is not None} + masked = {name for name, pdef in spec.parameters.items() if pdef.where is not None or pdef.in_} masks: dict[str, Mask | None] = {} for pname in sorted(masked): - context = f"Parameter '{pname}'" - mask = masks[pname] = mask_of(resolve_condition_text(spec.parameters[pname].where, ns, context, errors)) - if mask is None: + pdef, context = spec.parameters[pname], f"Parameter '{pname}'" + where = resolve_condition_text(pdef.where, ns, context, errors) + masks[pname] = _index_mask(pname, pdef.in_, pdef.where, where, ns, context, errors) + if (mask := mask_of(where)) is None: continue errors.extend( f"{context}: its `where:` contains variable '{v}', and a tool evaluates the `where:` of a parameter " f'from the data alone, before any variable has a value. Test a parameter or a map instead.' for v in sorted(mask.names_read & {*spec.variables, *spec.given.variables}) ) - errors.extend( + errors.extend(_masked_read(context, pname, read, spec) for read in sorted(mask.names_read & masked)) + return masks + + +def _masked_read(context: str, pname: str, read: str, spec: Spec) -> str: + """The refusal for a parameter's ``where:`` that reads *read*, a parameter whose own mask decides its rows.""" + if read == pname: + return ( f"{context}: its `where:` contains '{pname}' itself, and a tool evaluates that `where:` before it " - f"reads '{pname}'. Test a relation or another parameter." - if read == pname - else f"{context}: its `where:` contains '{read}', which has a `where:` of its own. Test the " - f"condition in the `where:` of '{read}', or a relation." - for read in sorted(mask.names_read & masked) + f"reads '{pname}'. Test a set or another parameter." ) - return masks + if spec.parameters[read].where is None: + return ( + f"{context}: its `where:` contains '{read}', which names a set in its dims:. Test a set that " + f"the dims: of '{read}' names instead." + ) + return ( + f"{context}: its `where:` contains '{read}', which has a `where:` of its own. Test the condition in the " + f"`where:` of '{read}', or a set." + ) + + +def _index_mask( + name: str, + in_: tuple[str, ...], + text: str | None, + where: Condition | None, + ns: Namespace, + context: str, + errors: list[str], +) -> Mask | None: + """The key of each relation in *in_* and the resolved *where* as one mask, or ``None`` where neither limits anything. + + A refusal is appended where the relations already decide some of the + ``where:``: inside them each bare relation name is true, so reading one + there says nothing. A map read as ``subject[map]`` is a test, + and stays. + """ + if where is not None and in_ and (written := Mask(where)).within(in_) != written: + errors.append(_decided_by_index_set(context, name, text, in_, written.within(in_).root)) + elif where is not None and (problem := _rows_tested(context, text, Mask(where), in_, ns)) is not None: + errors.append(problem) + rows: list[Condition] = [ + RelationDefined(r, tuple(sorted(ns.relations[r].dim(k) for k in ns.relations[r].key))) for r in in_ + ] + conjuncts = [*rows, *(() if where is None else (where,))] + return mask_of(functools.reduce(And, conjuncts)) if conjuncts else None + + +def _rows_tested(context: str, text: str | None, written: Mask, in_: tuple[str, ...], ns: Namespace) -> str | None: + """The refusal for a ``where:`` that tests the rows of a relation, which ``dims:`` states, or ``None``. + + A relation in ``dims:`` and its bare name in ``where:`` would say one thing + twice, so only ``dims:`` is taken. A map read as ``subject[map]`` is a test + of a label, and stays. + """ + tested = sorted({a.name for a in written.atoms if isinstance(a, RelationDefined) and a.column is None}) + if not tested: + return None + named = ' and '.join(f"'{r}'" for r in tested) + head = f'{context}: `where: {text}` tests the rows of {named}' + if everywhere := [r for r in tested if ns.relations[r].missing == 'refused']: + return ( + f"{head}, and under missing: refused '{everywhere[0]}' has a row everywhere. Delete it from the `where:`." + ) + top = {c.name for c in written.conjuncts if isinstance(c, RelationDefined) and c.column is None} + if not set(tested) <= top: + return ( + f'{head} under OR or NOT, which dims: cannot state. Declare those rows as a bool parameter, and test ' + f'that in its place.' + ) + rest = ', and keep the rest in the `where:`.' if len(written.conjuncts) > len(top) else ', and delete the `where:`.' + them = 'it' if len(tested) == 1 else 'them' + return f'{head}, which dims: states. Name {them} in dims: in place of the dimensions{rest}' + + +def _decided_by_index_set(context: str, name: str, text: str | None, in_: tuple[str, ...], inside: Condition) -> str: + """The refusal for a ``where:`` that reads a relation of the entry's own ``dims:``, by what is left of it inside.""" + named = ' and '.join(f"'{r}'" for r in in_) + head = ( + f"{context}: dims: names {named}, so '{name}' has coordinates only where " + f'{"it has" if len(in_) == 1 else "they have"} a row, and `where: {text}`' + ) + match inside: + case BooleanLiteral(value=True): + return f'{head} narrows nothing. Delete the `where:`, or the set from it.' + case BooleanLiteral(): + return f"{head} leaves '{name}' no coordinate. Write a `where:` that holds somewhere inside the set." + case _: + return f'{head} repeats the set. Delete the set from it.' def _assumption(name: str, entry: AssumptionSpec, ns: Namespace, errors: list[str]) -> Assumption | None: diff --git a/src/mathspec/_operators.py b/src/mathspec/_operators.py index 8a410c913..fbf8ef821 100644 --- a/src/mathspec/_operators.py +++ b/src/mathspec/_operators.py @@ -142,8 +142,8 @@ def call_shape_error(name: str, positional: int, kwargs: Iterable[str]) -> str | optional = {*builtin.edge_kwargs, *builtin.optional_kwargs} if keys & set(builtin.column_kwargs) and (unsaid := sorted(frozenset(builtin.with_columns) - keys)): return ( - f'{name}() through a relation does not name {", ".join(f"{k}=" for k in unsaid)}. A sum through a ' - f'relation names in over= the columns it sums away.\n' + f'{name}() through a set does not name {", ".join(f"{k}=" for k in unsaid)}. A sum through a ' + f'set names in over= the columns it sums away.\n' f'Write: {builtin.usage}' ) fits = positional == 1 and keys - optional == builtin.required diff --git a/src/mathspec/_piecewise.py b/src/mathspec/_piecewise.py index 5b3ac8602..b3bd2be30 100644 --- a/src/mathspec/_piecewise.py +++ b/src/mathspec/_piecewise.py @@ -449,7 +449,7 @@ def leaves_ungated(gate: VariableSpec | Variable, walk: JoinColumns | None = Non Under ``missing: refused`` every curve has one. """ absent = walk is not None and walk.relation.missing_at(walk.dropped) == 'absent' - return absent or (gate.where is not None and gate.missing != 0) + return absent or ((gate.where is not None or bool(gate.in_)) and gate.missing != 0) def gate_of(text: str) -> str | None: diff --git a/src/mathspec/_renaming.py b/src/mathspec/_renaming.py index df2136d9b..bfc4e507f 100644 --- a/src/mathspec/_renaming.py +++ b/src/mathspec/_renaming.py @@ -120,8 +120,8 @@ def _checked(names: Mapping[str, str | None], declared: Mapping[str, str], spec: raise LanguageError(msg) changed = {old: new for old, new in names.items() if new is not None and new != old} columns = { - column: f"a column of the relation '{relation}'" - for relation, entry in spec.relations.items() + column: f"a column of the set '{relation}'" + for relation, entry in spec.sets.items() for column, dimension in entry.pairs if column != dimension } | { diff --git a/src/mathspec/_resolution.py b/src/mathspec/_resolution.py index 490d0c423..577c4609a 100644 --- a/src/mathspec/_resolution.py +++ b/src/mathspec/_resolution.py @@ -110,7 +110,9 @@ def __init__(self, spec: Spec) -> None: } self.relations: dict[str, Relation] = subject_tables(spec) self.defaults: dict[str, bool | float] = { - p: pd.missing for p, pd in spec.parameters.items() if not isinstance(pd.missing, str) and pd.where is None + p: pd.missing + for p, pd in spec.parameters.items() + if not isinstance(pd.missing, str) and pd.where is None and not pd.in_ } self.leaf_dims: dict[str, tuple[str, ...]] = { **{p: pd.dims for p, pd in parameters.items()}, diff --git a/src/mathspec/_typesetting/__init__.py b/src/mathspec/_typesetting/__init__.py index 4fed1203a..4cc2c231c 100644 --- a/src/mathspec/_typesetting/__init__.py +++ b/src/mathspec/_typesetting/__init__.py @@ -153,7 +153,10 @@ def typeset( blocks = [format_.note(format_.escape(program.description))] if program.description else [] if legend: - explained, noticed = Legend(program, walk.symbols, format_, walk.parameter_masks()), notice(program) + explained, noticed = ( + Legend(program, walk.symbols, format_, walk.parameter_masks(), walk.index_sets()), + notice(program), + ) blocks += [ format_.section(title, format_.glossary(items)) for title, items in explained.glossaries(noticed, walk.defined()) diff --git a/src/mathspec/_typesetting/legend.py b/src/mathspec/_typesetting/legend.py index e99c0f5a6..2e3f67823 100644 --- a/src/mathspec/_typesetting/legend.py +++ b/src/mathspec/_typesetting/legend.py @@ -135,6 +135,7 @@ class Legend: symbols: Symbols format: Format parameter_masks: Mapping[str, str] = field(default_factory=dict) + index_sets: Mapping[str, str] = field(default_factory=dict) def _op(self, name: OperatorName) -> str: return self.format.operators[name] @@ -157,7 +158,7 @@ def glossaries(self, noticed: Noticed, defined: Iterable[str]) -> list[tuple[str parameters = [ self._item( self.symbols.name[p], - f'{fmt.mono(p)}{self._over(list(entry.dims))}{self._only_where(p)}{self._missing(entry.missing)}', + f'{fmt.mono(p)}{self._over(list(entry.dims), p)}{self._only_where(p)}{self._missing(entry.missing)}', entry.description, ) for p, entry in program.parameters.items() @@ -165,7 +166,7 @@ def glossaries(self, noticed: Noticed, defined: Iterable[str]) -> list[tuple[str variables = [ self._item( self.symbols.name[v], - f'{fmt.mono(v)}{self._over(list(entry.dims))}{self._missing(entry.missing, "absent", _MASKED_OUT)}', + f'{fmt.mono(v)}{self._over(list(entry.dims), v)}{self._missing(entry.missing, "absent", _MASKED_OUT)}', entry.description, ) for v, entry in program.variables.items() @@ -238,7 +239,10 @@ def _item(self, symbol: str, what: str, description: str | None) -> LegendItem: meaning = f'{what} {self.format.dash} {self.format.escape(description)}' if description else what return LegendItem(symbol, meaning) - def _over(self, dims: list[str]) -> str: + def _over(self, dims: list[str], entry: str | None = None) -> str: + """The domain of a legend row: *entry*'s index set where it has one, else the product of its sets.""" + if entry in self.index_sets: + return f' over {self.format.math(self.index_sets[entry])}' if not dims: return ' (scalar)' product = self.format.joined([self.symbols.set[d] for d in dims], self._op('times')) diff --git a/src/mathspec/_typesetting/walk.py b/src/mathspec/_typesetting/walk.py index f0c0df9ad..238c15638 100644 --- a/src/mathspec/_typesetting/walk.py +++ b/src/mathspec/_typesetting/walk.py @@ -66,7 +66,7 @@ from mathspec._expression_parser import BinaryOperator from mathspec._typesetting.format import Format from mathspec._typesetting.symbols import Symbols - from mathspec.program import Link, Piecewise, Program + from mathspec.program import Link, Parameter, Piecewise, Program, Variable _PRECEDENCE: dict[BinaryOperator, int] = {'+': 1, '-': 1, '*': 2, '/': 2, '**': 3} _ATOM = 5 @@ -206,6 +206,14 @@ def _unsigned(node: Expression) -> Expression | None: return None +def _inside_on(entry: Parameter | Variable) -> Mask | None: + """*entry*'s mask with its ``in:`` read as true, since the domain states it: ``None`` where nothing is left.""" + if entry.where is None: + return None + inside = entry.where.within(entry.in_) + return None if isinstance(inside.root, BooleanLiteral) else inside + + class Walk: """Walks a program, emitting [`Line`][]s in one format. @@ -632,10 +640,10 @@ def _mask_condition(self, ctx: _Context, mask: Mask | None) -> str: """ return '' if mask is None else self._condition(mask.root, ctx) - def _quantifier(self, dims: list[str], condition: str) -> str: + def _quantifier(self, dims: list[str], condition: str, *, memberships: list[str] | None = None) -> str: if not dims and not condition: return '' - over = self.format.joined([self._membership(d) for d in dims], '') + over = self.format.joined(memberships or [self._membership(d) for d in dims], '') if not condition: return f'{self._op("forall")} {over}' if not over: @@ -704,13 +712,42 @@ def _definitions(self) -> list[Line]: return [self.definition(name) for name in self.defined()] def parameter_masks(self) -> dict[str, str]: - """Each masked parameter's mask, printed over the parameter's own indices for its legend row.""" + """Each masked parameter's ``where:``, inside its ``in:``, printed over the parameter's own indices for its legend row.""" return { - name: self._condition(entry.where.root, self._context(entry.dims)) + name: self._condition(mask.root, self._context(entry.dims)) for name, entry in self.program.parameters.items() - if entry.where is not None + if (mask := _inside_on(entry)) is not None } + def index_sets(self) -> dict[str, str]: + """Each parameter's and variable's ``in:``, printed as the domain its legend row is over.""" + entries = {**self.program.parameters, **self.program.variables} + return { + name: self.format.joined(self._index_set(entry.dims, entry.in_, self._context(entry.dims)), '') + for name, entry in entries.items() + if entry.in_ + } + + def _index_set(self, dims: tuple[str, ...], in_: tuple[str, ...], ctx: _Context) -> list[str]: + """The rows of each relation in *in_*, then each other dimension of *dims*, as its set. + + A relation with maps prints by its own columns too, ``(p, u) ∈ connection``: + its maps play no part in the index set. + """ + relations = self.program.relations + keyed = {relations[r].dim(k) for r in in_ for k in relations[r].key} + return [ + *(self._key_in(r, ctx) for r in in_), + *(self._membership(d) for d in dims if d not in keyed), + ] + + def _key_in(self, name: str, ctx: _Context) -> str: + """The key of relation *name* at the frame's own indices, as a member of the relation: ``(b, g) ∈ connection``.""" + at = self._frame_key(name, ctx) + key = [at[k] for k in self.program.relations[name].key] + member = key[0] if len(key) == 1 else self.format.parenthesise(self.format.joined(key, '')) + return f'{member} {self._op("in")} {self.format.upright(name)}' + def defined(self) -> list[str]: """The named expressions that print under their own symbol: every one, or only the unsubstitutable when inlining. @@ -815,7 +852,11 @@ def _variable(self, name: str) -> Line: entry = self.program.variables[name] ctx = self._context(frame=entry.dims) symbol = ctx.indexed(self.symbols.name[name], list(entry.dims)) - condition = self._quantifier(list(entry.dims), self._mask_condition(ctx, entry.where)) + condition = self._quantifier( + list(entry.dims), + self._mask_condition(ctx, _inside_on(entry)), + memberships=self._index_set(entry.dims, entry.in_, ctx), + ) lower, upper = entry.lower, entry.upper if entry.domain == 'binary': diff --git a/src/mathspec/_validation.py b/src/mathspec/_validation.py index d23cdd4f2..4699910a9 100644 --- a/src/mathspec/_validation.py +++ b/src/mathspec/_validation.py @@ -111,6 +111,7 @@ def reference_errors(spec: Spec) -> list[str]: *_frame_dimensions(spec), *_relation_targets(spec), *_map_targets(spec), + *_index_sets(spec), *_bound_names(spec), *_sos_shapes(spec), *_sos_bounds(spec), @@ -121,23 +122,24 @@ def reference_errors(spec: Spec) -> list[str]: def undeclared_dimension(kind: str, name: str, dimension: str, spec: Spec, *, along: bool = False) -> str: """The one wording for an entry naming a dimension the file does not declare, or naming a relation as one.""" - if (relation := spec.relations.get(dimension)) is None: + if (relation := spec.sets.get(dimension)) is None: return f"{kind} '{name}' references undeclared dimension '{dimension}'. Declare it under 'dimensions:'." dims = sorted({d for _, d in relation.pairs}) - if along: - rewrite = f'Name one of its dimensions, {dims}.' - elif kind in ('Parameter', 'Variable'): - rewrite = f'Name its dimensions, {dims}, under dims:, and confine the entry with where: {dimension}.' - else: - rewrite = f'Name its dimensions, {dims}.' - return f"{kind} '{name}' names the relation '{dimension}', and a relation is not a dimension. {rewrite}" + if not along and kind in ('Parameter', 'Variable', 'Constraint') and len(dims) < len(relation.pairs): + return ( + f"{kind} '{name}' names the set '{dimension}', which has two columns over one dimension, and an " + f'entry is over each dimension once. Name its dimensions, {dims}, with a second name for the one it ' + f'repeats.' + ) + rewrite = f'Name one of its dimensions, {dims}.' if along else f'Name its dimensions, {dims}.' + return f"{kind} '{name}' names the set '{dimension}', and a set is not a dimension. {rewrite}" def _flat_namespace(spec: Spec) -> list[tuple[str, Iterable[str]]]: """Each kind of entry whose names share the one namespace an expression reads.""" return [ ('dimension', spec.dimensions), - ('relation', spec.relations), + ('set', spec.sets), ('parameter', spec.parameters), ('given parameter', spec.given.parameters), ('variable', spec.variables), @@ -200,28 +202,28 @@ def _frame_dimensions(spec: Spec) -> Iterator[str]: def _relation_targets(spec: Spec) -> Iterator[str]: """A relation has at least two columns, over declared dimensions, each named once.""" - for lname, lk in spec.relations.items(): + for lname, lk in spec.sets.items(): if len(lk.pairs) < 2: dim = lk.pairs[0][1] if lk.pairs else '' yield ( - f"Relation '{lname}' has {'one column' if lk.pairs else 'no column'} under 'dims:', and a relation " + f"Set '{lname}' has {'one column' if lk.pairs else 'no column'} under 'dims:', and a set " f'needs at least two. To give each {dim} a label, declare a map under maps: {dim}:. To mark some ' f'of them, declare a bool parameter or a condition.' ) if not isinstance(lk.dims, Mapping): yield from ( - f"Relation '{lname}' names dimension '{d}' twice under 'dims:'. Give the two columns names: " + f"Set '{lname}' names dimension '{d}' twice under 'dims:'. Give the two columns names: " f'dims: {{{d}0: {d}, {d}1: {d}}}.' for d, count in Counter(lk.dims if isinstance(lk.dims, tuple) else (lk.dims,)).items() if count > 1 ) yield from ( - undeclared_dimension('Relation', lname, d, spec) + undeclared_dimension('Set', lname, d, spec) for d in dict.fromkeys(d for _, d in lk.pairs) if d not in spec.dimensions ) yield from ( - f"Relation '{lname}' names column '{role}' after dimension '{role}', but the column is over " + f"Set '{lname}' names column '{role}' after dimension '{role}', but the column is over " f"'{dim}'. Rename the column after what it holds." for role, dim in lk.pairs if role in spec.dimensions and role != dim @@ -236,19 +238,19 @@ def _map_targets(spec: Spec) -> Iterator[str]: """ for subject, maps in spec.maps.items(): head = f"maps: '{subject}'" - if subject in spec.relations: - pairs = spec.relations[subject].pairs + if subject in spec.sets: + pairs = spec.sets[subject].pairs if twice := [d for d, count in Counter(d for _, d in pairs).items() if count > 1]: yield ( - f"{head} is a relation with two columns over '{twice[0]}', and a map reads one row per " - f'combination of its dimensions. Declare the maps on a relation over each dimension once.' + f"{head} is a set with two columns over '{twice[0]}', and a map reads one row per " + f'combination of its dimensions. Declare the maps on a set over each dimension once.' ) elif subject in spec.dimensions: pairs = ((subject, subject),) else: - subjects = [*spec.dimensions, *spec.relations] + subjects = [*spec.dimensions, *spec.sets] yield ( - f'{head} is neither a dimension nor a relation, and a map gives each row of one a label. ' + f'{head} is neither a dimension nor a set, and a map gives each row of one a label. ' f'{did_you_mean(subject, subjects, label="Subjects")}' ) continue @@ -263,10 +265,10 @@ def _map_targets(spec: Spec) -> Iterator[str]: f"{head}: map '{role}' has the name of a column of '{subject}', which holds the row, not a " f'label of it. Rename the map after what it holds.' ) - if m.dim in spec.relations: + if m.dim in spec.sets: yield ( - f"{head}: map '{role}' refers to the relation '{m.dim}', and a map gives a label of one " - f"dimension. Refer to one of its dimensions, or declare '{role}' on the relation instead." + f"{head}: map '{role}' refers to the set '{m.dim}', and a map gives a label of one " + f"dimension. Refer to one of its dimensions, or declare '{role}' on the set instead." ) elif m.dim not in spec.dimensions: yield ( @@ -279,6 +281,24 @@ def _map_targets(spec: Spec) -> Iterator[str]: ) +def _index_sets(spec: Spec) -> Iterator[str]: + """Each relation an entry names in ``dims:`` is named once and leaves a combination out.""" + entries = [ + *((f"Parameter '{name}'", p.in_) for name, p in spec.parameters.items()), + *((f"Variable '{name}'", v.in_) for name, v in spec.variables.items()), + *((f"Constraint '{name}'", c.in_) for name, c in spec.constraints.items()), + ] + for context, in_ in entries: + head = f'{context}: dims: names' + yield from (f"{head} '{r}' twice. Delete one." for r, count in Counter(in_).items() if count > 1) + for rname in dict.fromkeys(in_): + if spec.sets[rname].missing == 'refused': + yield ( + f"{head} '{rname}', which under missing: refused has a row at every combination, so it " + f"leaves nothing out. Write missing: absent on '{rname}', or name its dimensions in its place." + ) + + def _bound_names(spec: Spec) -> Iterator[str]: """A named bound is a numeric parameter, declared here or given.""" parameters = {**spec.parameters, **spec.given.parameters} diff --git a/src/mathspec/program.py b/src/mathspec/program.py index 7f5c93718..ebb9dd3bf 100644 --- a/src/mathspec/program.py +++ b/src/mathspec/program.py @@ -731,9 +731,14 @@ class Parameter: """ description: str | None = None where: Mask | None = None - """The condition that limits the coordinates of the parameter, or ``None`` when it has every coordinate of its - ``dims``. Where the condition is false, the parameter is absent and a tool drops its rows of data there. ``missing`` - applies where the condition is true. + """Where the parameter has coordinates, or ``None`` when it has every coordinate of its ``dims``. It holds each + relation in [`in_`][mathspec.program.Parameter.in_] and the file's ``where:``. Where it is false, the parameter is + absent and a tool drops its rows of data there. ``missing`` applies where it is true. + """ + in_: tuple[str, ...] = () + """The relations of the parameter's index set, written ``in:``, sorted by name. A tool attaching the data refuses a + row at a key combination one of them has no row for, before ``where`` drops anything and before ``missing`` fills + anything. A value column of the relation plays no part. """ @@ -741,6 +746,9 @@ class Parameter: class Variable: dims: tuple[str, ...] where: Mask | None = None + """Where the variable exists, or ``None`` when it exists at every coordinate of its ``dims``. It holds each relation + in [`in_`][mathspec.program.Variable.in_] and the file's ``where:``. + """ lower: Expression | None = None """A number, a parameter, or ``None`` when that side is open. An engine decides how an open side enters the model. """ @@ -749,6 +757,8 @@ class Variable: domain: VariableDomain = 'continuous' missing: VariableMissing = 'absent' description: str | None = None + in_: tuple[str, ...] = () + """The relations of the variable's index set, written ``in:``, sorted by name.""" @dataclass(frozen=True) @@ -1737,6 +1747,23 @@ def _fold(node: Condition) -> Condition: return node +def _within(node: Condition, relations: frozenset[str]) -> Condition: + """*node* with each bare read of *relations* true, and a named condition left whole where nothing in it changes.""" + match node: + case RelationDefined(name=name, column=None) if name in relations: + return BooleanLiteral(value=True) + case Not(operand=operand): + return Not(_within(operand, relations)) + case And(left=left, right=right): + return And(_within(left, relations), _within(right, relations)) + case Or(left=left, right=right): + return Or(_within(left, relations), _within(right, relations)) + case ConditionReference(body=body): + return node if (inside := _within(body, relations)) == body else inside + case _: + return node + + @dataclass(frozen=True) class Mask: """A resolved ``where`` and the questions the language answers about it. @@ -1785,6 +1812,13 @@ def dims(self) -> tuple[str, ...]: """ return tuple(sorted({dim for atom in self.atoms for dim in _atom_dims(atom)})) + def within(self, relations: Iterable[str]) -> Mask: + """The mask read inside the rows of *relations*, where each of them, read at the mask's own coordinates, is true. + + A relation read anywhere else, as through ``shift`` or ``at``, is left as it is. + """ + return Mask(_within(self.root, frozenset(relations))) + def __invert__(self) -> Mask: """The mask admitting exactly the rows this one refuses — construction folds a double negation or a literal flip.""" return Mask(Not(self.root)) diff --git a/src/mathspec/spec.py b/src/mathspec/spec.py index 626a26fb8..c4dcd21fa 100644 --- a/src/mathspec/spec.py +++ b/src/mathspec/spec.py @@ -33,6 +33,7 @@ model_serializer, model_validator, ) +from pydantic.json_schema import SkipJsonSchema from mathspec._expression_parser import NAME, Sense from mathspec._frozen import Frozen @@ -79,7 +80,7 @@ 'ParameterSpec', 'PiecewiseLinkSpec', 'PiecewiseSpec', - 'RelationSpec', + 'SetSpec', 'SosSpec', 'Spec', 'VariableSpec', @@ -133,7 +134,8 @@ def _set_fields(self, handler: SerializerFunctionWrapHandler) -> dict[str, objec The canonical form dumps with ``exclude_defaults``, which leaves a default out whether the input set it or not. """ - return _as_set(self, cast('dict[str, object]', handler(self))) + keys = _written_keys(type(self)) + return {keys[k]: v for k, v in _as_set(self, cast('dict[str, object]', handler(self))).items()} @model_validator(mode='before') @classmethod @@ -145,7 +147,7 @@ def _reject_unknown_keys(cls, data: object) -> object: """ if not isinstance(data, dict): return data - known = set(cls.model_fields) + known = set(_written_keys(cls).values()) unknown = [k for k in data if isinstance(k, str) and k not in known] if unknown: raise ValueError( @@ -184,6 +186,11 @@ def _reject_unknown_keys(cls, data: object) -> object: _WORDS = ', '.join(get_args(RelationMissing)) +def _written_keys(cls: type[BaseModel]) -> dict[str, str]: + """Each field of *cls* by name, mapped to the key a file writes it under: its alias where the name is a keyword.""" + return {name: field.alias or name for name, field in cls.model_fields.items()} + + def _as_set(model: BaseModel, written: dict[str, object]) -> dict[str, object]: """*written* cut to the fields *model*'s input set, less a null and an empty mapping, which say nothing. @@ -243,25 +250,25 @@ def relation_columns(written: str | Sequence[str] | Mapping[str, str]) -> tuple[ return tuple((d, d) for d in ((written,) if isinstance(written, str) else written)) -class RelationSpec(_StrictSpec): +class SetSpec(_StrictSpec): """A named relation: the combinations of its dimensions that exist, one row each. ``dims:`` is a list of dimensions, or a mapping of column name to dimension where two columns share one. A row is its own identity, so the relation is a subset of the product of its dimensions:: - relations: + sets: connection: {dims: [unit, port], missing: absent} corridor: {dims: {from: bus, to: bus}} - A relation is not a dimension. Its name stands in a condition, where it - is true at the rows it lists, and before a + A relation is not a dimension. Its name stands under an entry's ``in:``, + in a condition, where it is true at the rows it lists, and before a bracket, where it reads one of its maps (``maps:``, [`MapSpec`][]). The rows are data, and arrive under the relation's name, one column per column name, with a column per map beside them. """ - _label: ClassVar[str] = 'a relation entry' + _label: ClassVar[str] = 'a set entry' dims: str | Dims | ByName[str] """The relation's columns: a list of dimensions, each a column named after it, or a mapping of column name to @@ -277,7 +284,7 @@ class RelationSpec(_StrictSpec): @classmethod def _refused_or_absent(cls, v: object) -> object: """A row is data, so no value fills a gap in a relation.""" - return _refused_or_absent(v, 'a relation', 'Write absent where a combination may have no row, or leave it out.') + return _refused_or_absent(v, 'a set', 'Write absent where a combination may have no row, or leave it out.') @property def pairs(self) -> tuple[tuple[str, str], ...]: @@ -402,9 +409,14 @@ class ParameterSpec(_StrictSpec): dims: Dims dtype: ParameterDtype = 'float' + in_: Annotated[Dims, SkipJsonSchema()] = () + """The relations named in the parameter's ``dims:``, which the load replaces there by their dimensions. The parameter + has coordinates only where each of them has a row, and a tool refuses a row of its data anywhere else. A map of the + relation plays no part. A file writes the relation in ``dims:``, never this field. + """ where: str | None = None - """The condition that limits the coordinates of the parameter, in the condition grammar. Where it is false, the - parameter is absent and a tool drops its rows of data there. ``missing:`` applies where it is true. + """The condition that limits the coordinates of the parameter inside its ``in:``, in the condition grammar. Where it + is false, the parameter is absent and a tool drops its rows of data there. ``missing:`` applies where it is true. """ missing: Annotated[RelationMissing | bool | int | float, WithJsonSchema(_MISSING_SCHEMA)] = 'refused' description: str | None = None @@ -498,7 +510,10 @@ class VariableSpec(_StrictSpec): _label: ClassVar[str] = 'a variable entry' dims: Dims + in_: Annotated[Dims, SkipJsonSchema()] = () + """The relations named in the variable's ``dims:``, as on a parameter: it exists only where each has a row.""" where: str | None = None + """The condition that limits where the variable exists inside its ``in:``, in the condition grammar.""" bounds: BoundsSpec = BoundsSpec() domain: VariableDomain = 'continuous' missing: Annotated[Literal['absent'] | float, WithJsonSchema(_VARIABLE_MISSING_SCHEMA)] = 'absent' @@ -512,7 +527,7 @@ def _absent_or_a_number(cls, v: object, info: ValidationInfo[object]) -> object: The number stands for the variable where its mask leaves it out, so a mask must leave something out. The number lies in the variable's domain. The bounds hold only inside the mask, so the number may lie outside them. It loads as a float, as a bound does, so ``0`` and ``0.0`` are one spec. The checks run on the number as written, - so a refusal quotes it, and skip a ``where:`` or ``domain:`` that failed to load. + so a refusal quotes it, and skip an ``in:``, ``where:`` or ``domain:`` that failed to load. """ if v is None: raise _missing_null('a variable', 'Write a number, or leave the key out for absent.') @@ -523,10 +538,11 @@ def _absent_or_a_number(cls, v: object, info: ValidationInfo[object]) -> object: f'missing: {_as_yaml(v)} on a variable, which takes absent or a finite number. A variable has no ' f'data, so refused and a flag apply only to a parameter.' ) - elif 'where' in info.data and info.data['where'] is None: + elif {'in_', 'where'} <= info.data.keys() and info.data['where'] is None and not info.data['in_']: msg = ( - f'missing: {_as_yaml(v)} needs a `where:`. A variable with no `where:` exists at every ' - f'coordinate of its dimensions, so no coordinate is missing. Add a `where:`, or delete `missing:`.' + f'missing: {_as_yaml(v)} needs a `where:` or an `in:`. A variable with neither exists at every ' + f'coordinate of its dimensions, so no coordinate is missing. Add a `where:` or an `in:`, or delete ' + f'`missing:`.' ) elif info.data.get('domain') == 'binary' and v not in (0, 1): msg = f'missing: {_as_yaml(v)} on a binary variable, which takes 0 or 1.' @@ -652,6 +668,8 @@ class ConstraintSpec(_StrictSpec): _label: ClassVar[str] = 'a constraint entry' dims: Dims + in_: Annotated[Dims, SkipJsonSchema()] = () + """The relations named in the constraint's ``dims:``, as on a parameter: it has rows only where each has a row.""" where: str | None = None expression: str description: str | None = None @@ -1058,7 +1076,7 @@ def _check_dims(cls, data: object) -> object: msg = ( 'a piecewise entry declares dims:, the dimensions with one curve per coordinate, as a variable ' 'declares its own. Write the dimensions its links are over, without any that a link adds through ' - 'a relation: dims: [generator, snapshot] for a link on p over [generator, snapshot].' + 'a set: dims: [generator, snapshot] for a link on p over [generator, snapshot].' ) raise ValueError(msg) return data @@ -1109,7 +1127,7 @@ def _check_links(cls, v: Frozen[str, PiecewiseLinkSpec]) -> Frozen[str, Piecewis if len(v) < 2 and not any(link.walks for link in v.values()): msg = ( 'piecewise needs at least two links ([expression, values, sign?]), or one link that walks a ' - 'relation. One quantity on a curve is a bound, not a curve.' + 'set. One quantity on a curve is a bound, not a curve.' ) raise ValueError(msg) if all(link.sign != '==' for link in v.values()): @@ -1200,7 +1218,7 @@ class is pydantic's, not a contract this package keeps. given: GivenSpec = GivenSpec() """What this file uses and does not build ([`GivenSpec`][]). It is empty when the file has no ``given:``.""" dimensions: ByName[DimensionSpec] = Frozen({}) - relations: ByName[RelationSpec] = Frozen({}) + sets: ByName[SetSpec] = Frozen({}) maps: ByName[ByName[MapSpec]] = Frozen({}) """Each subject's maps, by subject, a dimension or a relation, and then by map ([`MapSpec`][]).""" parameters: ByName[ParameterSpec] = Frozen({}) @@ -1286,7 +1304,19 @@ def _drop_absence(self, handler: SerializerFunctionWrapHandler) -> dict[str, obj the file wrote it or not, so a written file states the language it is read in. """ - return {'version': self.version, **_as_set(self, cast('dict[str, object]', handler(self)))} + written = _as_set(self, cast('dict[str, object]', handler(self))) + return {'version': self.version, **_sets_written_back(written, self.sets)} + + @model_validator(mode='before') + @classmethod + def _sets_in_dims(cls, data: object) -> object: + """A relation named in the ``dims:`` of a parameter, a variable or a constraint stands for its dimensions. + + The entry is over those dimensions, at the rows the relation lists, + so every check after this one sees plain dimensions. ``in_`` keeps + the relations, and the serializer writes them back into ``dims:``. + """ + return _over_sets(data) if isinstance(data, dict) else data def to_dict(self) -> dict[str, object]: """The spec as plain data. ``to_spec(m.to_dict())`` reproduces it.""" @@ -1388,6 +1418,76 @@ def _lower(self) -> Spec: return self +_SET_SECTIONS = ('parameters', 'variables', 'constraints') + + +def _members(relation: object) -> tuple[str, ...] | None: + """The dimensions a relation stands for in an entry's ``dims:``, or ``None`` where it lists one twice. + + An entry over a dimension has it once, so a relation with two columns over + one dimension stays a name the dimension check refuses. + """ + dims = relation.get('dims') if isinstance(relation, Mapping) else getattr(relation, 'dims', None) + listed = [dims] if isinstance(dims, str) else list(dims.values()) if isinstance(dims, Mapping) else dims + if not isinstance(listed, (list, tuple)) or len(set(listed)) != len(listed): + return None + return tuple(listed) + + +def _over_sets(data: dict[str, object]) -> dict[str, object]: + """*data* with each relation in an entry's ``dims:`` replaced by its dimensions and recorded under ``in_``.""" + relations = data.get('sets') + sets = { + name: members + for name, entry in (relations.items() if isinstance(relations, Mapping) else ()) + if (members := _members(entry)) is not None + } + out = dict(data) + for section in _SET_SECTIONS: + if isinstance(entries := data.get(section), Mapping): + out[section] = {name: _entry_over_sets(entry, sets, section) for name, entry in entries.items()} + return out + + +def _entry_over_sets(entry: object, sets: Mapping[str, tuple[str, ...]], section: str) -> object: + """One entry with the relations in its ``dims:`` replaced by their dimensions, each dimension once.""" + if not isinstance(entry, Mapping): + return entry + if 'in_' in entry: + kind = section.removesuffix('s') + msg = f"unknown key 'in_' in a {kind} entry. Name the set in dims: in place of its dimensions." + raise ValueError(msg) + dims = entry.get('dims') + named: list[str] = [d for d in dims if isinstance(d, str) and d in sets] if isinstance(dims, (list, tuple)) else [] + if not named: + return entry + flat = list(dict.fromkeys(m for d in cast('list[str]', dims) for m in (sets[d] if d in named else (d,)))) + return {**entry, 'dims': flat, 'in_': named} + + +def _sets_written_back(written: dict[str, object], relations: Mapping[str, SetSpec]) -> dict[str, object]: + """*written* with each entry's ``in_`` folded back into its ``dims:``, as the file writes it.""" + out = dict(written) + for section in _SET_SECTIONS: + if isinstance(entries := written.get(section), Mapping): + out[section] = {name: _entry_written_back(entry, relations) for name, entry in entries.items()} + return out + + +def _entry_written_back(entry: object, relations: Mapping[str, SetSpec]) -> object: + """One entry's ``dims:`` with the dimensions of each relation in ``in_`` written as the relation, at its first.""" + if not isinstance(entry, Mapping) or not (named := entry.get('in_')): + return entry + owner = {m: r for r in reversed(cast('list[str]', named)) for _, m in relations[r].pairs} + dims: list[str] = [] + for d in cast('list[str]', entry['dims']): + name = owner.get(d, d) + if name not in dims: + dims.append(name) + dims.extend(r for r in cast('list[str]', named) if r not in dims) + return {key: dims if key == 'dims' else value for key, value in entry.items() if key != 'in_'} + + def subject_tables(spec: Spec) -> dict[str, Relation]: """Each table the data supplies, by name: a relation with its maps as columns beside its own, and a dimension with maps. @@ -1398,9 +1498,9 @@ def subject_tables(spec: Spec) -> dict[str, Relation]: A subject the file does not declare is left out, which the load refuses. """ tables: dict[str, Relation] = {} - for name in sorted({*spec.relations, *spec.maps}): + for name in sorted({*spec.sets, *spec.maps}): maps = spec.maps.get(name, Frozen({})) - if (relation := spec.relations.get(name)) is not None: + if (relation := spec.sets.get(name)) is not None: key, missing, description = relation.pairs, relation.missing, relation.description elif name in spec.dimensions: key, missing, description = ((name, name),), cast('RelationMissing', 'refused'), None diff --git a/tests/test_canonical.py b/tests/test_canonical.py index 76fb4fc14..91600e312 100644 --- a/tests/test_canonical.py +++ b/tests/test_canonical.py @@ -153,7 +153,7 @@ def test_a_map_is_written_back_as_the_mapping_it_was_declared_as(declared): 'description', 'given', 'dimensions', - 'relations', + 'sets', 'maps', 'parameters', 'conditions', diff --git a/tests/test_composition.py b/tests/test_composition.py index 47996afe7..fa8aad802 100644 --- a/tests/test_composition.py +++ b/tests/test_composition.py @@ -232,7 +232,7 @@ def _mapped(missing: str | None = None, dim: str = 'bus') -> dict[str, object]: def _wired(missing: str | None = None) -> dict[str, object]: """A fragment that declares only the relation `wired` over `port` and `bus`, as `_mapped` does a map.""" entry = {'dims': ['port', 'bus']} | ({} if missing is None else {'missing': missing}) - return {'dimensions': {'port': {'dtype': 'str'}, 'bus': {'dtype': 'str'}}, 'relations': {'wired': entry}} + return {'dimensions': {'port': {'dtype': 'str'}, 'bus': {'dtype': 'str'}}, 'sets': {'wired': entry}} #: A fragment that gives `port` a map of its own beside the one `SURFACE` gives it. @@ -276,7 +276,7 @@ def test_two_fragments_that_say_one_missing_agree(fragments, missing): pytest.param([SURFACE, _mapped('absent')], "map 'port[bus]'", id='refused-against-absent'), pytest.param([_mapped('absent'), SURFACE], "map 'port[bus]'", id='absent-against-refused'), pytest.param([SURFACE, _mapped(dim='node')], "map 'port[bus]'", id='two-dimensions-referred-to'), - pytest.param([_wired(), _wired('absent')], "relation 'wired'", id='a-relation-refused-against-absent'), + pytest.param([_wired(), _wired('absent')], "set 'wired'", id='a-set-refused-against-absent'), ], ) def test_two_fragments_that_say_different_things_about_one_entry_are_refused(fragments, entry): @@ -599,7 +599,7 @@ def test_a_patch_restates_a_dimension_as_the_schema_reads_it(dimension, restated #: `SURFACE` with a relation, `wired`, over the two dimensions `port[bus]` maps between. -WIRED = varied(SURFACE, **{'relations.wired': {'dims': ['port', 'bus']}}) +WIRED = varied(SURFACE, **{'sets.wired': {'dims': ['port', 'bus']}}) @pytest.mark.parametrize( @@ -613,9 +613,9 @@ def test_a_patch_restates_a_dimension_as_the_schema_reads_it(dimension, restated id='a-dimension', ), pytest.param( - varied(WIRED, **{'relations.wired.description': 'which ports reach which buses'}), - {'relations': {'wired': {'dims': ['port', 'bus'], 'description': 'a port and the buses it reaches'}}}, - lambda spec: spec.relations['wired'].description, + varied(WIRED, **{'sets.wired.description': 'which ports reach which buses'}), + {'sets': {'wired': {'dims': ['port', 'bus'], 'description': 'a port and the buses it reaches'}}}, + lambda spec: spec.sets['wired'].description, 'a port and the buses it reaches', id='a-relation', ), @@ -641,9 +641,9 @@ def test_a_patch_restates_a_dimension_as_the_schema_reads_it(dimension, restated id='null-removes-a-dimension-description', ), pytest.param( - varied(WIRED, **{'relations.wired.description': 'which ports reach which buses'}), - {'relations': {'wired': {'description': None}}}, - lambda spec: spec.relations['wired'].description, + varied(WIRED, **{'sets.wired.description': 'which ports reach which buses'}), + {'sets': {'wired': {'description': None}}}, + lambda spec: spec.sets['wired'].description, None, id='null-removes-a-relation-description', ), @@ -725,10 +725,8 @@ def test_a_patch_that_rewrites_a_dimension_is_refused(patch, says): @pytest.mark.parametrize( ('patch', 'kind', 'name'), [ - pytest.param({'relations': {'wired': {}}}, 'relation', 'wired', id='a-relation-with-no-field'), - pytest.param( - {'relations': {'wired': {'dims': ['port']}}}, 'relation', 'wired', id='a-relation-over-one-dimension' - ), + pytest.param({'sets': {'wired': {}}}, 'set', 'wired', id='a-set-with-no-field'), + pytest.param({'sets': {'wired': {'dims': ['port']}}}, 'set', 'wired', id='a-set-over-one-dimension'), pytest.param({'maps': {'port': {'bus': {}}}}, 'map', 'port[bus]', id='a-map-with-no-field'), ], ) @@ -747,8 +745,8 @@ def test_a_patch_that_restates_an_entry_in_part_is_refused_with_the_rewrite(patc ('patch', 'says'), [ pytest.param( - {'relations': {'wired': {'dmis': ['port', 'bus']}}}, - "unknown key 'dmis' in a relation entry. Did you mean 'dims'?", + {'sets': {'wired': {'dmis': ['port', 'bus']}}}, + "unknown key 'dmis' in a set entry. Did you mean 'dims'?", id='a-relation', ), pytest.param( @@ -804,7 +802,7 @@ def test_a_patch_changes_what_a_row_the_map_gives_no_label_means(base, map_, mis def test_a_patch_changes_what_a_combination_the_relation_leaves_out_means(): - assert override(WIRED, [{'relations': {'wired': {'missing': 'absent'}}}]).relations['wired'].missing == 'absent' + assert override(WIRED, [{'sets': {'wired': {'missing': 'absent'}}}]).sets['wired'].missing == 'absent' @pytest.mark.parametrize( @@ -827,7 +825,7 @@ def test_a_patch_changes_what_a_missing_row_of_a_parameter_means(base, written, ('section', 'name', 'declared'), [ pytest.param('parameters', 'c', {'dims': ['g']}, id='a-parameter'), - pytest.param('relations', 'gh', {'dims': ['g', 'h']}, id='a-relation'), + pytest.param('sets', 'gh', {'dims': ['g', 'h']}, id='a-relation'), pytest.param('maps', 'g.h', {'dim': 'h'}, id='a-map'), ], ) diff --git a/tests/test_dimensions.py b/tests/test_dimensions.py index d09cc1ce7..ada336922 100644 --- a/tests/test_dimensions.py +++ b/tests/test_dimensions.py @@ -67,7 +67,7 @@ 'bus': {'dtype': 'str'}, 'zone': {'dtype': 'str'}, }, - 'relations': {'gen_zone': {'dims': ['generator', 'snapshot']}}, + 'sets': {'gen_zone': {'dims': ['generator', 'snapshot']}}, 'maps': { 'generator': {'bus': {'dim': 'bus'}, 'zone': {'dim': 'zone'}, 'b0': {'dim': 'bus'}, 'b1': {'dim': 'bus'}}, 'snapshot': {'bus': {'dim': 'bus'}, 'rep': {'dim': 'snapshot'}}, @@ -253,7 +253,7 @@ def _dims_with(expr: str, **overrides) -> frozenset[str]: ) def test_a_bare_relation_is_summed_between_its_key_columns(expr, expected): """A bare relation holds no value column, so the column a sum groups by is a key column.""" - assert _dims_with(expr, **{'relations.connection': {'dims': ['generator', 'bus']}}) == expected + assert _dims_with(expr, **{'sets.connection': {'dims': ['generator', 'bus']}}) == expected def test_a_value_column_added_to_a_relation_changes_no_call(): @@ -441,7 +441,7 @@ def test_an_ill_dimensioned_expression_is_rejected(expr, error, match): def test_two_joined_columns_do_not_share_a_dimension(expr, diag): """The operand carries one coordinate per dimension, so the `over=` column and an unnamed key column cannot share one.""" with pytest.raises(DimensionError, match=r"joins 'diag' on \[.*\] through more than one column"): - _dims_with(expr, **{'relations.diag': diag}) + _dims_with(expr, **{'sets.diag': diag}) @pytest.mark.parametrize( @@ -695,7 +695,7 @@ def test_a_condition_names_the_entries_its_leaves_test(namespace, condition, exp ) def test_names_read_takes_both_sides_of_a_map_pair(condition, expected): """The one leaf that can name two entries — two maps compared row by row on the key their subjects share.""" - home_of = {'relations.home_of': {'dims': ['generator', 'snapshot']}, 'maps.home_of': {'zone': {'dim': 'zone'}}} + home_of = {'sets.home_of': {'dims': ['generator', 'snapshot']}, 'maps.home_of': {'zone': {'dim': 'zone'}}} where = where_of(condition, Namespace(_spec(**home_of)), 'test') assert where is not None, 'a condition the connectives cannot settle survives the fold' @@ -899,9 +899,7 @@ def test_the_order_a_file_writes_a_list_in_reaches_no_value_of_the_spec(path): one spec loaded to two specs, and wrote back two texts.""" spec = to_spec(path) backwards = to_spec(sets_reversed(spec.to_dict())) - assert backwards == spec, ( - 'every dims:, one_of: and kwargs: list and each side of a relation is sorted as the spec loads' - ) + assert backwards == spec, 'every dims:, one_of: and kwargs: list and each side of a set is sorted as the spec loads' assert backwards.to_yaml() == spec.to_yaml(), 'so the spec writes back one text' @@ -969,7 +967,7 @@ def test_every_dims_of_the_program_is_sorted_by_name(path): @pytest.mark.parametrize( 'entry', [ - pytest.param({'where': 'zone_of'}, id='the-rows-of-the-relation'), + pytest.param({'dims': ['zone_of']}, id='the-rows-of-the-relation'), pytest.param({'where': "zone_of[zone] == 'north'"}, id='a-column-against-a-label'), pytest.param({'where': 'zone_of[zone] == home_of[zone]'}, id='a-column-against-another-relation'), ], @@ -979,8 +977,8 @@ def test_a_condition_on_a_relation_states_its_key_dims_sorted_by_name(entry): keyed_backwards = {'dims': ['snapshot', 'generator'], 'missing': 'absent'} spec = _spec( **{ - 'relations.zone_of': keyed_backwards, - 'relations.home_of': keyed_backwards, + 'sets.zone_of': keyed_backwards, + 'sets.home_of': keyed_backwards, 'maps.zone_of': {'zone': {'dim': 'zone'}}, 'maps.home_of': {'zone': {'dim': 'zone'}}, } @@ -989,5 +987,5 @@ def test_a_condition_on_a_relation_states_its_key_dims_sorted_by_name(entry): where = to_spec(varied(spec.to_dict(), **patch)).program.variables['p'].where assert where is not None assert [dims for _, dims in _dims_tuples(where.root)] == [('generator', 'snapshot')], ( - 'the condition states the relation key dims, sorted by name' + 'the condition states the set key dims, sorted by name' ) diff --git a/tests/test_expansion.py b/tests/test_expansion.py index e5d3d33a9..9c569ff55 100644 --- a/tests/test_expansion.py +++ b/tests/test_expansion.py @@ -290,7 +290,7 @@ def test_macro_collisions_rejected(patch, match): ), pytest.param( {'grouped': {'args': ['x'], 'template': 'sum(x, over=g, by=nope[h])'}}, - r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a relation", + r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a set", id='a-typo-in-a-relation-kwarg', ), pytest.param( @@ -300,7 +300,7 @@ def test_macro_collisions_rejected(patch, match): ), pytest.param( {'grouped': {'args': ['x', 'a', 'b'], 'template': 'sum(x, over=a, by=nope[b])'}}, - r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a relation, a dimension with maps or a formal argument of this macro", + r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a set, a dimension with maps or a formal argument of this macro", id='a-typo-in-a-relation-beside-formal-columns', ), pytest.param( diff --git a/tests/test_lowering.py b/tests/test_lowering.py index fd0067258..6ecd0e037 100644 --- a/tests/test_lowering.py +++ b/tests/test_lowering.py @@ -720,11 +720,11 @@ def test_a_relation_lowers_with_the_join_each_call_names(): program = to_spec( { 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'generator': {'ordered': True}, 'zone': {}}, - 'relations': {'zone_of': {'dims': ['generator', 'snapshot'], 'missing': 'absent'}}, + 'sets': {'zone_of': {'dims': ['generator', 'snapshot'], 'missing': 'absent'}}, 'maps': {'zone_of': {'zone': {'dim': 'zone'}}}, 'parameters': {'price': {'dims': ['snapshot', 'zone']}}, 'variables': { - 'p': {'dims': ['snapshot', 'generator'], 'where': "zone_of AND zone_of[zone] == 'A'"}, + 'p': {'dims': ['zone_of'], 'where': "zone_of[zone] == 'A'"}, 'first': { 'dims': ['snapshot', 'generator'], 'where': 'position(generator, within=zone_of[zone]) == 0', @@ -749,7 +749,7 @@ def test_a_relation_lowers_with_the_join_each_call_names(): columns = (('generator', 'generator'), ('snapshot', 'snapshot'), ('zone', 'zone')) declared = Relation(columns, ('generator', 'snapshot'), missing='absent') - assert program.relations == {'zone_of': declared}, 'the relation sits once in the program, under its name' + assert program.relations == {'zone_of': declared}, 'the set sits once in the program, under its name' zonal = program.constraints['zonal'].lhs columns = JoinColumns('zone_of', declared, ('generator', 'snapshot'), ('snapshot', 'zone')) assert zonal == Sum( @@ -796,7 +796,7 @@ def test_a_relation_lowers_with_the_join_each_call_names(): assert [(type(a).__name__, a.dims) for a in p_where.atoms] == [ ('RelationDefined', ('generator', 'snapshot')), ('RelationComparison', ('generator', 'snapshot')), - ], 'a row and a comparison are both read at the key of a keyed relation' + ], 'the rows `dims:` names and a comparison are both read at the key of a keyed set' first_where = program.variables['first'].where assert first_where is not None assert first_where.dims == ('generator', 'snapshot'), 'a position within a group is read at every key column' @@ -805,14 +805,14 @@ def test_a_relation_lowers_with_the_join_each_call_names(): @pytest.mark.parametrize( ('variable', 'entry', 'atom'), [ - pytest.param('q', {'where': 'link'}, RelationDefined('link', ('g', 'h')), id='a-bare-relation-tests-a-row'), + pytest.param('q', {'dims': ['link']}, RelationDefined('link', ('g', 'h')), id='a-relation-in-dims-tests-a-row'), pytest.param('p', {'where': 'g[h]'}, RelationDefined('g', ('g',), 'h'), id='a-map-alone-tests-a-label'), ], ) def test_a_condition_tests_a_row_or_a_label_by_how_it_is_written(variable, entry, atom): """A relation's bare name asks for a row, and `subject[map]` alone asks whether the map gives the row a label.""" patch = { - 'relations.link': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}, 'maps.g.h': {'dim': 'h', 'missing': 'absent'}, **{f'variables.{variable}.{key}': value for key, value in entry.items()}, } @@ -897,7 +897,7 @@ def test_a_relation_is_declared_as_the_file_declares_it(): varied( TINY, dimensions={'g': {}, 'bus': {}, 'season': {}}, - relations={'link': {'dims': ['g', 'bus'], 'missing': 'absent'}}, + sets={'link': {'dims': ['g', 'bus'], 'missing': 'absent'}}, maps={'g': {'season': {'dim': 'season'}, 'bus': {'dim': 'bus', 'missing': 'absent'}}}, ) ).program @@ -907,7 +907,7 @@ def test_a_relation_is_declared_as_the_file_declares_it(): 'link': Relation((('bus', 'bus'), ('g', 'g')), ('bus', 'g'), missing='absent'), }, 'a table per subject, its own columns first, then one column per map' assert program.relations['g'].values == ('bus', 'season'), 'the maps are what a row of the subject determines' - assert program.relations['link'].values == (), 'a relation with no maps determines nothing' + assert program.relations['link'].values == (), 'a set with no maps determines nothing' assert program.dimensions['g'] == Dimension(dtype='str'), 'a dimension carries its dtype and no map' diff --git a/tests/test_parser.py b/tests/test_parser.py index a095f528b..3278127bc 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -340,7 +340,7 @@ def test_a_relation_column_is_named_in_brackets(): ) def test_a_column_selection_names_one_relation_and_its_columns(text, columns): """The relation is written once, before its columns, so a kwarg cannot name columns of two tables.""" - assert parse_expression(text).kwargs['by'] == columns, 'one relation, its columns in the order written' + assert parse_expression(text).kwargs['by'] == columns, 'one set, its columns in the order written' @pytest.mark.parametrize( diff --git a/tests/test_piecewise.py b/tests/test_piecewise.py index cafd5c33d..d3516cc4e 100644 --- a/tests/test_piecewise.py +++ b/tests/test_piecewise.py @@ -12,7 +12,7 @@ from __future__ import annotations import re -from typing import get_args +from typing import Any, get_args import pytest @@ -782,13 +782,21 @@ def test_a_ragged_where_reaches_the_weights_as_written_and_the_frame_rows_as_a_c assert expanded.constraints['cost_curve_p'].where == 'count(has_curve AND bp_x, over=bp) > 0' -def test_a_where_joins_both_gate_rows(): +@pytest.mark.parametrize( + 'gate', + [ + pytest.param({'where': 'committable'}, id='a-gate-with-a-where'), + pytest.param({'dims': ['running']}, id='a-gate-in-a-relation'), + ], +) +def test_a_where_joins_both_gate_rows(gate: dict[str, Any]): """`activity:` splits the convexity row across the gate's own mask, and the block's where holds over both halves.""" spec = spec_of( MASKED, **{ - 'variables.u': {'dims': ['snapshot', 'generator'], 'domain': 'binary', 'where': 'committable'}, + 'variables.u': {'dims': ['snapshot', 'generator'], 'domain': 'binary', **gate}, 'parameters.committable': {'dims': ['generator'], 'dtype': 'bool'}, + 'sets.running': {'dims': ['generator', 'snapshot'], 'missing': 'absent'}, 'piecewise.cost_curve.activity': 'u', }, ) @@ -975,7 +983,7 @@ def _walk(**written: object) -> dict[str, object]: def _joined(dims: list[str] | dict[str, str]) -> dict[str, object]: """The `power` link walked through `generator_of`, a relation over *dims* that maps each row to a generator.""" - return {'relations.generator_of': {'dims': dims}, 'maps.generator_of.generator': {'dim': 'generator'}} | _walk( + return {'sets.generator_of': {'dims': dims}, 'maps.generator_of.generator': {'dim': 'generator'}} | _walk( by='generator_of[generator]' ) @@ -1295,7 +1303,7 @@ def test_a_gate_read_through_a_relation_switches_the_curves_it_maps_to(): assert (rows['curve_convexity_ungated'].where, rows['curve_convexity_ungated'].expression) == ( f'NOT ({RUNNING})', 'sum(curve_lam, over=bp) == 1', - ), 'a converter with no row in the relation is ungated' + ), 'a converter with no row in the set is ungated' def test_a_gate_read_through_a_relation_is_written_as_the_at_call_it_reads(): @@ -1400,7 +1408,7 @@ def test_a_gate_read_through_a_relation_prints_as_the_read(): ), pytest.param( {'piecewise.curve.activity': 'at(running, by=bp_p[converter])'}, - r"piecewise 'curve' activity: at\(by=bp_p\[\.\.\.\]\): 'bp_p' is not a relation", + r"piecewise 'curve' activity: at\(by=bp_p\[\.\.\.\]\): 'bp_p' is not a set", id='a-walk-through-something-no-relation', ), pytest.param( @@ -1518,7 +1526,7 @@ def test_a_walked_row_exists_only_where_its_relation_has_a_row(patch, where): INVESTED = { 'dimensions': {name: {'dtype': 'str'} for name in ('cost_curve', 'effect', 'flow')} | {'bp': {'dtype': 'int', 'ordered': True}}, - 'relations': {'cost_of': {'dims': ['flow', 'effect']}}, + 'sets': {'cost_of': {'dims': ['flow', 'effect']}}, 'maps': {'cost_of': {'cost_curve': {'dim': 'cost_curve'}}}, 'parameters': { 'has_cost_curve': {'dims': ['cost_curve'], 'dtype': 'bool'}, diff --git a/tests/test_pypsa_references.py b/tests/test_pypsa_references.py index 9532b9310..498872a4a 100644 --- a/tests/test_pypsa_references.py +++ b/tests/test_pypsa_references.py @@ -180,7 +180,7 @@ def test_a_patch_keeps_the_pypsa_name_of_what_it_changes(page: str): """A patch rewrites entries of the base; a rewritten name keeps its PyPSA name and its dtype, or it has drifted.""" own = SPECS[page] drifted = [] - for section in ('parameters', 'relations', 'variables', 'constraints'): + for section in ('parameters', 'sets', 'variables', 'constraints'): theirs, ours = getattr(BASE, section), getattr(own, section) for name in set(theirs) & set(ours): if (theirs[name].description or '').split(' — ')[0] != (ours[name].description or '').split(' — ')[0]: diff --git a/tests/test_renaming.py b/tests/test_renaming.py index a6c741a4e..98eadceb1 100644 --- a/tests/test_renaming.py +++ b/tests/test_renaming.py @@ -25,7 +25,7 @@ #: `unit_` entries. Every place a name can stand appears once. UNIT = { 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'unit': {'dtype': 'str'}, 'bus': {'dtype': 'str'}}, - 'relations': {'unit_link': {'dims': ['bus', 'unit'], 'missing': 'absent'}}, + 'sets': {'unit_link': {'dims': ['bus', 'unit'], 'missing': 'absent'}}, 'maps': {'unit': {'bus': {'dim': 'bus'}, 'bus0': {'dim': 'bus'}}}, 'given': {'expressions': {'total_cost': {'dims': []}}}, 'parameters': { @@ -61,11 +61,7 @@ #: The placeholder names, as a class takes them. PLACEHOLDERS = ( - *( - name - for section in ('relations', 'parameters', 'conditions', 'variables', 'expressions') - for name in UNIT[section] - ), + *(name for section in ('sets', 'parameters', 'conditions', 'variables', 'expressions') for name in UNIT[section]), 'unit_ramp', 'unit_bounded', 'unit_twice', @@ -93,7 +89,7 @@ def _as(prefix: str, dim: str) -> dict[str, str]: [ pytest.param(lambda s: list(s['dimensions']), ['bus', 'generator', 'snapshot'], id='dimension'), pytest.param( - lambda s: s['relations']['Generator_link'], + lambda s: s['sets']['Generator_link'], {'dims': ['bus', 'generator'], 'missing': 'absent'}, id='relation', ), @@ -348,14 +344,14 @@ def test_a_map_the_spec_cannot_take_is_refused(names, message): ('old', 'message'), [ pytest.param('bus0', "'bus0' is also a map of 'line'", id='a-map'), - pytest.param('from_bus', "'from_bus' is also a column of the relation 'corridor'", id='a-relation-column'), + pytest.param('from_bus', "'from_bus' is also a column of the set 'corridor'", id='a-relation-column'), ], ) def test_a_name_that_is_also_a_map_or_a_relation_column_is_refused(old, message): """`line[bus0]` reads a map, and a rename of the parameter `bus0` would change it too.""" spec = { 'dimensions': {'line': {'dtype': 'str'}, 'bus': {'dtype': 'str'}}, - 'relations': {'corridor': {'dims': {'from_bus': 'bus', 'to_bus': 'bus'}}}, + 'sets': {'corridor': {'dims': {'from_bus': 'bus', 'to_bus': 'bus'}}}, 'maps': {'line': {'bus0': {'dim': 'bus'}, 'bus1': {'dim': 'bus'}}}, 'parameters': {'bus0': {'dims': ['bus']}, 'from_bus': {'dims': ['bus']}}, 'variables': {'flow': {'dims': ['line']}}, diff --git a/tests/test_schema.py b/tests/test_schema.py index 31c409289..1e14e1a27 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -59,7 +59,7 @@ def test_no_definition_refers_only_to_itself(): pytest.param('ObjectiveSpec', 'sense', spec.ObjectiveSense, id='sense'), pytest.param('VariableSpec', 'domain', spec.VariableDomain, id='domain'), pytest.param('VariableSpec', 'missing', Literal['absent'], id='variable-missing'), - pytest.param('RelationSpec', 'missing', spec.RelationMissing, id='relation-missing'), + pytest.param('SetSpec', 'missing', spec.RelationMissing, id='relation-missing'), pytest.param('ParameterSpec', 'missing', spec.RelationMissing, id='parameter-missing'), pytest.param('ParameterSpec', 'dtype', spec.ParameterDtype, id='parameter-dtype'), pytest.param('DimensionSpec', 'dtype', spec.DimensionDtype, id='dimension-dtype'), @@ -80,7 +80,7 @@ def test_a_closed_vocabulary_is_published_as_an_enum(entry, field, alias): ('entry', 'default'), [ pytest.param('ParameterSpec', 'refused', id='parameter'), - pytest.param('RelationSpec', 'refused', id='relation'), + pytest.param('SetSpec', 'refused', id='relation'), pytest.param('VariableSpec', 'absent', id='variable'), ], ) diff --git a/tests/test_validation.py b/tests/test_validation.py index 327193671..e13efdfcf 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -88,7 +88,7 @@ class TestValidateExpressions: ), pytest.param( { - 'relations.link': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}, 'constraints': {'cap': {'dims': ['g'], 'where': 'lnk', 'expression': 'p <= c'}}, }, ("'lnk' not found", "'link']"), @@ -329,7 +329,7 @@ class TestDimensionKwargs: ), pytest.param( 'sum(p, over=generator, by=generatr[zone]) == load', - ("'generatr' is not a relation or a dimension with maps", "Did you mean 'generator'?"), + ("'generatr' is not a set or a dimension with maps", "Did you mean 'generator'?"), id='by-relation-typo', ), pytest.param( @@ -779,7 +779,7 @@ def test_it_resolves(self, mask: str, position: int, by: str | None): ( 'position(snapshot, within=load[snapshot]) == 0', [ - "position(within=load[...]): 'load' is not a relation or a dimension with maps", + "position(within=load[...]): 'load' is not a set or a dimension with maps", 'Write within=[]', ], ), @@ -788,7 +788,7 @@ def test_it_resolves(self, mask: str, position: int, by: str | None): ["'period' has no column of its own over 'snapshot'", "Its own columns are ['period']"], ), ], - ids=['a parameter', 'undeclared', 'within= is not a relation', 'within= is over another dim'], + ids=['a parameter', 'undeclared', 'within= is not a set', 'within= is over another dim'], ) def test_it_refuses(self, mask: str, fragments: list[str]): with pytest.raises(LanguageError) as excinfo: @@ -1054,7 +1054,7 @@ def test_a_read_names_several_columns_as_an_expression_does(self, where, dims, r spec = to_spec( { 'dimensions': {name: {'dtype': 'str'} for name in ('curve', 'effect', 'flow')}, - 'relations': {'cost_of': {'dims': ['flow', 'effect']}}, + 'sets': {'cost_of': {'dims': ['flow', 'effect']}}, 'maps': { 'cost_of': {'curve': {'dim': 'curve'}}, 'curve': {'flow': {'dim': 'flow'}, 'effect': {'dim': 'effect'}}, @@ -1071,7 +1071,7 @@ def test_a_read_names_several_columns_as_an_expression_does(self, where, dims, r mask = spec.program.constraints['k'].where assert mask is not None assert sorted(mask.dims) == dims, 'the read lands on the columns the other end names' - assert mask.names_read == reads, 'a consumer attaches the relation as well as the operand' + assert mask.names_read == reads, 'a consumer attaches the set as well as the operand' assert to_spec(spec.to_yaml()).program == spec.program, 'the where string reads back to the same mask' for fmt in ('markdown', 'latex', 'typst'): assert all(name in typeset_line(spec, 'k', fmt).replace('\\_', '_') for name in printed), ( @@ -1198,7 +1198,7 @@ def test_a_read_names_several_columns_as_an_expression_does(self, where, dims, r ), pytest.param( 'at(r, by=nope[h])', - ("at(by=nope[...]): 'nope' is not a relation or a dimension with maps",), + ("at(by=nope[...]): 'nope' is not a set or a dimension with maps",), id='a-read-through-no-relation', ), pytest.param( @@ -1435,41 +1435,41 @@ class TestRulesDecidedWithoutData: id='sos-whose-expansion-collides-with-an-entry', ), pytest.param( - {'relations.link': {'dims': ['g', 'h'], 'dtype': 'str'}}, - ("unknown key 'dtype' in a relation entry. Valid keys: description, dims, missing.",), + {'sets.link': {'dims': ['g', 'h'], 'dtype': 'str'}}, + ("unknown key 'dtype' in a set entry. Valid keys: description, dims, missing.",), id='relation-with-a-dtype-of-its-own', ), pytest.param( - {'relations.link': {'dims': ['g']}}, + {'sets.link': {'dims': ['g']}}, ( - "Relation 'link' has one column under 'dims:', and a relation needs at least two", + "Set 'link' has one column under 'dims:', and a set needs at least two", 'To give each g a label, declare a map under maps: g:.', ), id='relation-with-one-column', ), pytest.param( - {'relations.link': {'dims': ['g', 'z']}}, - ("Relation 'link' references undeclared dimension 'z'",), + {'sets.link': {'dims': ['g', 'z']}}, + ("Set 'link' references undeclared dimension 'z'",), id='relation-over-undeclared', ), pytest.param( - {'relations.odd': {'dims': {'h': 'g', 'x': 'h'}}}, + {'sets.odd': {'dims': {'h': 'g', 'x': 'h'}}}, ("names column 'h' after dimension 'h', but the column is over 'g'",), id='relation-column-named-after-a-dimension-it-is-not-over', ), pytest.param( - {'relations.link': {'dims': ['g', 'g']}}, + {'sets.link': {'dims': ['g', 'g']}}, ("names dimension 'g' twice under 'dims:'", 'dims: {g0: g, g1: g}'), id='relation-naming-a-dim-twice-without-names', ), pytest.param( - {'relations.link': {'dims': []}}, - ("Relation 'link' has no column under 'dims:', and a relation needs at least two",), + {'sets.link': {'dims': []}}, + ("Set 'link' has no column under 'dims:', and a set needs at least two",), id='relation-with-no-columns', ), pytest.param( {'maps.nope': {'h': {'dim': 'h'}}}, - ("maps: 'nope' is neither a dimension nor a relation", 'Subjects: g, h.'), + ("maps: 'nope' is neither a dimension nor a set", 'Subjects: g, h.'), id='map-of-no-subject', ), pytest.param( @@ -1478,13 +1478,13 @@ class TestRulesDecidedWithoutData: id='map-whose-name-is-not-a-name', ), pytest.param( - {'relations.link': {'dims': ['g', 'h']}, 'maps.link': {'g': {'dim': 'g'}}}, + {'sets.link': {'dims': ['g', 'h']}, 'maps.link': {'g': {'dim': 'g'}}}, ("maps: 'link': map 'g' has the name of a column of 'link'", 'Rename the map after what it holds'), id='map-named-after-a-column-of-its-subject', ), pytest.param( - {'relations.link': {'dims': ['g', 'h']}, 'maps.g': {'x': {'dim': 'link'}}}, - ("maps: 'g': map 'x' refers to the relation 'link', and a map gives a label of one dimension",), + {'sets.link': {'dims': ['g', 'h']}, 'maps.g': {'x': {'dim': 'link'}}}, + ("maps: 'g': map 'x' refers to the set 'link', and a map gives a label of one dimension",), id='map-referring-to-a-relation', ), pytest.param( @@ -1498,8 +1498,8 @@ class TestRulesDecidedWithoutData: id='map-named-after-another-dimension', ), pytest.param( - {'relations.pair': {'dims': {'g0': 'g', 'g1': 'g'}}, 'maps.pair': {'x': {'dim': 'h'}}}, - ("maps: 'pair' is a relation with two columns over 'g'", 'Declare the maps on a relation over each'), + {'sets.pair': {'dims': {'g0': 'g', 'g1': 'g'}}, 'maps.pair': {'x': {'dim': 'h'}}}, + ("maps: 'pair' is a set with two columns over 'g'", 'Declare the maps on a set over each'), id='map-on-a-relation-with-two-columns-over-one-dimension', ), pytest.param( @@ -1525,7 +1525,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'relations.bare': {'dims': {'a': 'g', 'b': 'g', 'm': 'z'}}, + 'sets.bare': {'dims': {'a': 'g', 'b': 'g', 'm': 'z'}}, 'objective': {'expression': 'sum(sum(p, over=g, by=bare[m]))'}, }, ( @@ -1537,7 +1537,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'relations.gz': {'dims': ['g', 'z']}, + 'sets.gz': {'dims': ['g', 'z']}, 'objective': {'expression': 'sum(sum(q, over=h, by=gz[z]))'}, }, ("over=h names no column of 'gz', so the sum reads nothing through 'gz'", "its columns are ['g', 'z']"), @@ -1607,8 +1607,8 @@ class TestRulesDecidedWithoutData: id='a-selection-in-over-without-by', ), pytest.param( - {'relations.rel': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(sum(p, by=rel[h]))'}}, - ('sum() through a relation does not name over=', 'names in over= the columns it sums away'), + {'sets.rel': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(sum(p, by=rel[h]))'}}, + ('sum() through a set does not name over=', 'names in over= the columns it sums away'), id='a-sum-through-a-relation-names-what-leaves', ), pytest.param( @@ -1632,7 +1632,7 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'relations.rel': {'dims': ['g', 'h']}, + 'sets.rel': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(at(r, by=rel[h]))'}, }, ("'rel' has no map for at() to read. Sum through it instead, or declare a map under maps: rel:.",), @@ -1649,16 +1649,16 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'relations.rel': {'dims': ['g', 'h']}, + 'sets.rel': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(shift(p, along=g, offset=1, edge=0, within=rel[h]))'}, }, ("'rel' has no map, so it makes no groups", 'under maps: g:'), id='a-partition-through-a-relation-with-no-map', ), pytest.param( - {'relations.rel': {'dims': ['g', 'h']}, 'variables.q.where': "rel[h] == 'x'"}, + {'sets.rel': {'dims': ['g', 'h']}, 'variables.q.where': "rel[h] == 'x'"}, ( - "compares the relation 'rel', which has no map, so its rows hold no label", + "compares the set 'rel', which has no map, so its rows hold no label", "Test the bare name 'rel' for whether a row exists, or declare a map under maps: rel:.", ), id='where-compares-a-relation-with-no-map', @@ -1670,25 +1670,25 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'relations.pair': {'dims': {'g0': 'g', 'g1': 'g'}}, + 'sets.pair': {'dims': {'g0': 'g', 'g1': 'g'}}, 'variables.q.where': 'pair', }, ("'pair' has two columns over one dimension (['g0', 'g1']), so a bare name is ambiguous",), id='where-bare-name-of-a-relation-with-two-columns-over-one-dim', ), pytest.param( - {'relations.h': {'dims': ['g', 'h']}}, - ("Relation 'h' collides with the dimension",), + {'sets.h': {'dims': ['g', 'h']}}, + ("Set 'h' collides with the dimension",), id='relation-named-after-a-dimension', ), pytest.param( - {'relations.link': {'dims': ['g', 'h'], 'rows': {'g1': 'h1'}}}, - ("unknown key 'rows' in a relation entry", 'Valid keys'), + {'sets.link': {'dims': ['g', 'h'], 'rows': {'g1': 'h1'}}}, + ("unknown key 'rows' in a set entry", 'Valid keys'), id='a-relation-declaring-its-rows', ), pytest.param( - {'relations.link': {'dim': ['g', 'h']}}, - ("unknown key 'dim' in a relation entry", "Did you mean 'dims'?"), + {'sets.link': {'dim': ['g', 'h']}}, + ("unknown key 'dim' in a set entry", "Did you mean 'dims'?"), id='a-relation-with-a-misspelt-key', ), pytest.param({'variables.p.missing': 0}, ('missing: 0 needs a `where:`',), id='zero-without-a-mask'), @@ -1767,8 +1767,8 @@ class TestRulesDecidedWithoutData: id='a-dimension-as-a-value', ), pytest.param( - {'relations.link': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(link + p, over=g)'}}, - ("'link' is a relation, and a relation is not a value",), + {'sets.link': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(link + p, over=g)'}}, + ("'link' is a set, and a set is not a value",), id='a-relation-as-a-value', ), pytest.param( @@ -1816,7 +1816,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'relations.bare': {'dims': {'k': 'g', 'j0': 'h', 'j1': 'h', 'm': 'z'}}, + 'sets.bare': {'dims': {'k': 'g', 'j0': 'h', 'j1': 'h', 'm': 'z'}}, 'objective': {'expression': 'sum(sum(q, over=k, by=bare[m]))'}, }, ("joins 'bare' on ['h'] through more than one column",), @@ -1825,7 +1825,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'relations.gz': {'dims': ['g', 'z']}, + 'sets.gz': {'dims': ['g', 'z']}, 'maps.gz': {'h': {'dim': 'h'}}, 'variables.q.where': 'g[h] != gz[h]', }, @@ -1838,7 +1838,7 @@ class TestRulesDecidedWithoutData: id='where-against-a-variable', ), pytest.param( - {'relations.link': {'dims': ['g', 'h']}, 'variables.p.where': 'c > link'}, + {'sets.link': {'dims': ['g', 'h']}, 'variables.p.where': 'c > link'}, ("compares 'c' against 'link', which is no label", 'Compare against a literal'), id='where-against-a-relation', ), @@ -1921,11 +1921,11 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'relations.link': {'dims': ['g', 'h']}, + 'sets.link': {'dims': ['g', 'h']}, 'maps.link': {'y': {'dim': 'h'}, 'z': {'dim': 'h'}}, 'variables.q.where': "link == 'x'", }, - ("'link' compares the relation 'link' itself, and a comparison reads a label", 'link[y]'), + ("'link' compares the set 'link' itself, and a comparison reads a label", 'link[y]'), id='where-a-relation-compared-bare', ), pytest.param( @@ -1986,7 +1986,7 @@ def test_the_rewrite_a_sum_grouped_by_the_whole_key_names_is_one_the_language_ta ('patch', 'where'), [ pytest.param( - {'relations.link': {'dims': ['g', 'h']}, 'maps.link': {'y': {'dim': 'h'}, 'z': {'dim': 'h'}}}, + {'sets.link': {'dims': ['g', 'h']}, 'maps.link': {'y': {'dim': 'h'}, 'z': {'dim': 'h'}}}, "link == 'x'", id='a-relation-with-two-maps-read-bare', ), @@ -2004,76 +2004,142 @@ def test_the_map_a_bare_comparison_is_told_to_read_is_one_the_language_takes(sel _spec(**{'variables.q.where': where.replace('link', named.group(1), 1)}, **patch) -_LINKED = {'relations.link': {'dims': ['g', 'h'], 'missing': 'absent'}} +_LINKED = {'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}} @pytest.mark.parametrize( ('patch', 'message'), [ pytest.param( - {'variables.x': {'dims': ['link']}}, - "Variable 'x' names the relation 'link', and a relation is not a dimension. Name its dimensions, " - "['g', 'h'], under dims:, and confine the entry with where: link.", - id='dims-of-a-variable', - ), - pytest.param( - {'constraints.c': {'dims': ['link'], 'expression': 'k >= 0'}}, - "Constraint 'c' names the relation 'link', and a relation is not a dimension. Name its dimensions, " - "['g', 'h'].", - id='dims-of-a-constraint', - ), - pytest.param( - {'constraints.c': {'dims': [], 'expression': 'sum(q, over=link) >= 0'}}, - "Constraint 'c': sum(over=link) names the relation 'link', and a relation is not a dimension. Name its " - "dimensions, ['g', 'h'].", - id='over', + { + 'sets.next': {'dims': {'from': 'g', 'to': 'g'}, 'missing': 'absent'}, + 'variables.x': {'dims': ['next']}, + }, + "Variable 'x' names the set 'next', which has two columns over one dimension, and an entry is over " + "each dimension once. Name its dimensions, ['g'], with a second name for the one it repeats.", + id='dims-of-a-relation-over-one-dimension-twice', ), pytest.param( {'constraints.c': {'dims': ['g'], 'expression': 'p <= shift(p, along=link, offset=1)'}}, - "Constraint 'c': shift(along=link) names the relation 'link', and a relation is not a dimension. Name " + "Constraint 'c': shift(along=link) names the set 'link', and a set is not a dimension. Name " "one of its dimensions, ['g', 'h'].", id='along', ), ], ) def test_a_relation_named_where_a_dimension_belongs_is_refused(patch: dict[str, Any], message: str): - """mathspec told the writer to declare the relation under `dimensions:`, which makes a second, unrelated name.""" + """A relation stands for its dimensions in `dims:` and `over=`, and nowhere it would have to be one dimension.""" assert message in _refusal(**_LINKED, **patch) +@pytest.mark.parametrize( + ('patch', 'kind', 'name'), + [ + pytest.param({'variables.x': {'dims': ['link']}}, 'variables', 'x', id='dims-of-a-variable'), + pytest.param({'parameters.k': {'dims': ['link']}}, 'parameters', 'k', id='dims-of-a-parameter'), + pytest.param( + {'constraints.c': {'dims': ['link'], 'expression': 'q >= 0'}}, 'constraints', 'c', id='dims-of-a-constraint' + ), + ], +) +def test_a_relation_in_dims_stands_for_its_dimensions_at_its_rows(patch: dict[str, Any], kind: str, name: str): + """An entry over a relation is over its dimensions, and has coordinates only where the relation has a row.""" + spec = to_spec(varied(SMALL_MODEL, **_LINKED, **patch)) + entry = getattr(spec.program, kind)[name] + assert entry.dims == ('g', 'h'), 'the set stands for its dimensions' + assert entry.where is not None and entry.where.names_read == {'link'}, 'the entry exists at the rows of link' + assert spec.to_dict()[kind][name]['dims'] == ['link'], 'the file writes the set back, not its dimensions' + + +def test_a_sum_over_a_relation_sums_over_its_dimensions(): + spec = to_spec( + varied(SMALL_MODEL, **_LINKED, **{'constraints.c': {'dims': [], 'expression': 'sum(q, over=link) >= 0'}}) + ) + assert spec.program.constraints['c'].dims == (), 'over=link sums g and h away' + + #: `p` over the pairs of `g` and `h`, with two relations that list some of them. ON_A_RELATION: dict[str, Any] = { 'dimensions': {'g': {}, 'h': {}}, - 'relations': {'pair': {'dims': ['g', 'h'], 'missing': 'absent'}, 'link': {'dims': ['g', 'h'], 'missing': 'absent'}}, + 'sets': {'pair': {'dims': ['g', 'h'], 'missing': 'absent'}, 'link': {'dims': ['g', 'h'], 'missing': 'absent'}}, 'parameters': {'flag': {'dims': ['g', 'h'], 'dtype': 'bool'}}, 'variables': {'p': {'dims': ['g', 'h'], 'where': 'pair'}}, } @pytest.mark.parametrize( - 'patch', + ('patch', 'message'), [ - pytest.param({}, id='the-rows-of-a-relation'), - pytest.param({'variables.p.where': 'pair AND NOT link'}, id='the-rows-of-two-relations'), - pytest.param({'variables.p.where': 'flag'}, id='a-test-of-a-value'), pytest.param( - {'maps.g.zone': {'dim': 'h', 'missing': 'absent'}, 'variables.p.where': 'g[zone]'}, - id='a-label-of-a-map-of-a-dimension', + {}, + "Variable 'p': `where: pair` tests the rows of 'pair', which dims: states. Name it in dims: in place of " + 'the dimensions, and delete the `where:`.', + id='a-relation', ), pytest.param( - {'maps.pair.zone': {'dim': 'h', 'missing': 'absent'}, 'variables.p.where': 'pair[zone]'}, - id='a-label-of-a-map-of-the-relation', + {'variables.p.where': 'pair AND link'}, + "tests the rows of 'link' and 'pair', which dims: states. Name them in dims: in place of the " + 'dimensions, and delete the `where:`.', + id='two-relations', + ), + pytest.param( + {'variables.p.where': 'pair AND flag'}, + 'Name it in dims: in place of the dimensions, and keep the rest in the `where:`.', + id='a-relation-beside-a-test', + ), + pytest.param( + {'variables.p': {'dims': ['link'], 'where': 'pair AND flag'}}, + 'Name it in dims: in place of the dimensions, and keep the rest in the `where:`.', + id='a-relation-beside-a-relation-in-dims', + ), + pytest.param( + {'variables.p': {'dims': ['g', 'h']}, 'parameters.cap': {'dims': ['g', 'h'], 'where': 'pair'}}, + "Parameter 'cap': `where: pair` tests the rows of 'pair', which dims: states.", + id='a-parameter', ), pytest.param( { 'variables.p': {'dims': ['g', 'h']}, 'constraints.c': {'dims': ['g', 'h'], 'where': 'pair', 'expression': 'p >= 0'}, }, - id='on-a-constraint', + "Constraint 'c': `where: pair` tests the rows of 'pair', which dims: states.", + id='a-constraint', + ), + pytest.param( + {'variables.p.where': 'pair OR flag'}, + "tests the rows of 'pair' under OR or NOT, which dims: cannot state. Declare those rows as a bool " + 'parameter, and test that in its place.', + id='under-an-or', + ), + pytest.param({'variables.p.where': 'NOT pair'}, "rows of 'pair' under OR or NOT", id='under-a-not'), + pytest.param( + {'sets.pair.missing': 'refused'}, + "under missing: refused 'pair' has a row everywhere. Delete it from the `where:`.", + id='a-relation-with-a-row-everywhere', + ), + ], +) +def test_a_where_that_tests_the_rows_of_a_relation_is_refused(patch: dict[str, Any], message: str): + """`where: pair` and `pair` in `dims:` said one thing twice, and a parameter's `where:` dropped what `dims:` refuses.""" + assert message in _refusal(ON_A_RELATION, **patch) + + +@pytest.mark.parametrize( + 'patch', + [ + pytest.param({'variables.p': {'dims': ['pair']}}, id='the-rows-in-dims'), + pytest.param({'variables.p.where': 'flag'}, id='a-test-of-a-value'), + pytest.param( + {'maps.g.zone': {'dim': 'h', 'missing': 'absent'}, 'variables.p.where': 'g[zone]'}, + id='a-label-of-a-map-of-a-dimension', + ), + pytest.param( + {'maps.pair.zone': {'dim': 'h', 'missing': 'absent'}, 'variables.p.where': 'pair[zone]'}, + id='a-label-of-a-map-of-the-relation', ), ], ) -def test_a_where_that_reads_a_relation_or_a_map_loads(patch: dict[str, Any]): +def test_a_where_that_reads_a_map_or_a_value_loads(patch: dict[str, Any]): to_spec(varied(ON_A_RELATION, **patch)) @@ -2285,7 +2351,7 @@ def _cased(cases: dict[str, Any] | None = None, **entry: Any) -> dict[str, Any]: #: columns written as a mapping, and the maps of a subject. _PROBED = { **DISPATCH_MODEL, - 'relations': {'pair': {'dims': {'from_gen': 'generator', 'to_gen': 'generator'}}}, + 'sets': {'pair': {'dims': {'from_gen': 'generator', 'to_gen': 'generator'}}}, 'maps': {'generator': {'peer': {'dim': 'generator'}}}, 'expressions': { 'price': { @@ -2562,7 +2628,7 @@ class TestAnEntryIsNamed: 'section', [ 'dimensions', - 'relations', + 'sets', 'parameters', 'variables', 'expressions', @@ -2575,7 +2641,7 @@ class TestAnEntryIsNamed: def test_a_name_no_expression_could_write_is_refused(self, section: str, name: str): entries: dict[str, Any] = { 'dimensions': {'dtype': 'str'}, - 'relations': {'dims': ['g', 'h']}, + 'sets': {'dims': ['g', 'h']}, 'parameters': {'dims': ['g']}, 'variables': {'dims': ['g']}, 'expressions': {'expression': 'c'}, @@ -2871,6 +2937,14 @@ def test_a_values_table_under_a_ragged_where_is_not_refused(parameter, declared) pytest.param({'parameters.n': {'dims': ['g'], 'dtype': 'str', 'missing': 'absent'}}, id='a-label-under-absent'), pytest.param({'given.parameters.n': {'dims': ['g'], 'dtype': 'int'}}, id='a-given-int'), pytest.param({'parameters.n': {'dims': ['g'], 'dtype': 'int', 'where': 'flag'}}, id='a-masked-int'), + pytest.param( + { + 'parameters.n': {'dims': ['link'], 'dtype': 'int'}, + 'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'variables.p.dims': ['g', 'h'], + }, + id='an-int-on-a-relation', + ), ], ) def test_a_bare_parameter_whose_answer_needs_the_data_loads(patch): @@ -2881,17 +2955,103 @@ def test_a_bare_parameter_whose_answer_needs_the_data_loads(patch): #: A parameter over the pairs of `g` and `h` that a relation lists, with a fill. _MASKED = { - 'relations.pair': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'sets.pair': {'dims': ['g', 'h'], 'missing': 'absent'}, 'maps.pair.kind': {'dim': 'h', 'missing': 'absent'}, 'parameters.w': {'dims': ['g', 'h'], 'where': 'pair[kind]', 'missing': 1}, } +#: The same parameter, over the rows of the relation rather than masked by it. +_IN = { + 'sets.pair': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'parameters.w': {'dims': ['pair'], 'missing': 1}, +} + +_LINK = {'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}} + +#: Each kind of entry that names a relation in `dims:`, as the label its refusals open with and `w` of that kind over `pair`. +_IN_KINDS = { + 'parameters': ('Parameter', _IN), + 'variables': ( + 'Variable', + {'sets.pair': _IN['sets.pair'], 'variables.w': {'dims': ['pair']}}, + ), +} + + +def _in(kind: str, *relations: str, **entry: Any) -> dict[str, Any]: + """`w` of *kind* over the rows of *relations*, *entry* laid over it.""" + _, patch = _IN_KINDS[kind] + return {**patch, f'{kind}.w': {**patch[f'{kind}.w'], 'dims': list(relations), **entry}} + + +@pytest.mark.parametrize('kind', _IN_KINDS) +@pytest.mark.parametrize( + ('patch', 'in_'), + [ + pytest.param( + {'parameters.v': {'dims': ['g', 'h'], 'missing': 'absent'}}, {'w': ('pair',)}, id='only-the-named' + ), + pytest.param( + { + 'dimensions.t': {'dtype': 'int'}, + 'parameters.v': {'dims': ['pair', 't'], 'missing': 'absent'}, + }, + {'w': ('pair',), 'v': ('pair',)}, + id='a-parameter-over-one-more-dimension', + ), + pytest.param( + {'sets.pair': {'dims': {'a': 'g', 'b': 'h'}, 'missing': 'absent'}}, + {'w': ('pair',)}, + id='columns-named-apart-from-their-dimensions', + ), + pytest.param({**_LINK, 'w.in': ['pair', 'link']}, {'w': ('link', 'pair')}, id='two-relations-sorted-by-name'), + pytest.param( + {'maps.pair': {'to': {'dim': 'g'}}}, + {'w': ('pair',)}, + id='a-relation-with-a-map-over-one-of-its-dimensions', + ), + ], +) +def test_an_entry_has_the_index_set_it_names(kind: str, patch: dict[str, Any], in_: dict[str, tuple[str, ...]]): + """A tool reads an entry's index set off the program, inside its `where`, and an entry that names none stays free.""" + rest = {key: value for key, value in patch.items() if key != 'w.in'} + spec = _spec(**{**_in(kind, *patch.get('w.in', ['pair'])), **rest}) + entries = {**spec.program.parameters, **spec.program.variables} + assert {name: entry.in_ for name, entry in entries.items() if entry.in_} == in_, ( + 'exactly the entries that name a set have it as their index set, and it alone' + ) + where = entries['w'].where + assert where is not None and where.names_read == set(in_['w']), 'the index set is part of where the entry exists' + assert to_spec(spec.to_yaml()).program == spec.program, 'a set in dims: survives a round trip through YAML' + + +@pytest.mark.parametrize( + ('path', 'entry'), + [ + pytest.param('parameters.efficiency', {'missing': 1}, id='a-parameter'), + pytest.param('variables.delivered', {}, id='a-variable'), + ], +) +def test_a_relation_with_maps_is_an_index_set_by_its_own_columns(path: str, entry: dict[str, Any]): + """`connection` is over unit and port; its maps bus and role are labels a `where:` still tests beside it.""" + raw = read_yaml(EXAMPLES / 'connection.yaml') + written = {'dims': ['connection', 'snapshot'], 'where': "connection[role] == 'output'"} + spec = to_spec(varied(raw, **{path: {**written, **entry}})) + kind, name = path.split('.') + indexed = getattr(spec.program, kind)[name] + assert indexed.in_ == ('connection',), 'the set with maps is the index set' + assert indexed.where is not None and indexed.where.dims == ('port', 'unit'), 'by its own columns, not its maps' + assert indexed.where.within(indexed.in_) == to_spec(raw).program.parameters['efficiency'].where, ( + 'inside the index set, the `where:` on the role is what the file wrote' + ) + + def test_a_parameter_mask_is_part_of_the_program(): """A tool reads where a parameter has coordinates off the program, and the file writes the mask back.""" spec = _spec(**_MASKED) assert spec.program.parameters['w'].where is not None, 'the mask reaches the program' - assert spec.program.parameters['w'].where.names_read == {'pair'}, 'the mask reads the relation and nothing else' + assert spec.program.parameters['w'].where.names_read == {'pair'}, 'the mask reads the set and nothing else' assert spec.program.parameters['c'].where is None, 'a parameter with no `where:` has every coordinate' assert to_spec(spec.to_yaml()).program == spec.program, 'the mask survives a round trip through YAML' @@ -2922,9 +3082,15 @@ def test_a_parameter_mask_is_part_of_the_program(): pytest.param( {'parameters.c.where': 'flag', 'parameters.flag.where': "g[h] == 'x'"}, "Parameter 'c': its `where:` contains 'flag', which has a `where:` of its own. Test the condition in " - "the `where:` of 'flag', or a relation.", + "the `where:` of 'flag', or a set.", id='a-mask-that-reads-a-masked-parameter', ), + pytest.param( + {**_IN, 'parameters.u': {'dims': ['g', 'h'], 'where': 'w > 0.5'}}, + "Parameter 'u': its `where:` contains 'w', which names a set in its dims:. Test a set that " + "the dims: of 'w' names instead.", + id='a-mask-that-reads-a-parameter-with-an-index-set', + ), pytest.param( {'parameters.c.where': 'flag', 'parameters.flag.where': 'c > 0'}, "Parameter 'c': its `where:` contains 'flag', which has a `where:` of its own", @@ -2941,6 +3107,66 @@ def test_a_parameter_mask_that_nothing_can_evaluate_is_refused(patch, fragment): assert fragment in _refusal(**patch) +_HEAD = "{entry} 'w': dims: names 'pair', so 'w' has coordinates only where it has a row, and " + + +@pytest.mark.parametrize('kind', _IN_KINDS) +@pytest.mark.parametrize( + ('sets', 'patch', 'fragment'), + [ + pytest.param( + ['pair'], + {'where': 'pair'}, + _HEAD + '`where: pair` narrows nothing. Delete the `where:`, or the set from it.', + id='a-where-that-repeats-the-relation', + ), + pytest.param( + ['pair'], + {'where': 'pair OR flag'}, + '`where: pair OR flag` narrows nothing.', + id='a-where-the-relation-satisfies', + ), + pytest.param( + ['pair'], + {'where': 'NOT pair'}, + "`where: NOT pair` leaves 'w' no coordinate. Write a `where:` that holds somewhere inside the set.", + id='a-where-the-relation-leaves-empty', + ), + pytest.param( + ['pair'], + {'where': 'pair AND flag'}, + _HEAD + '`where: pair AND flag` repeats the set. Delete the set from it.', + id='a-where-that-repeats-the-relation-beside-a-test', + ), + pytest.param( + ['pair', 'pair'], {}, "{entry} 'w': dims: names 'pair' twice. Delete one.", id='a-relation-named-twice' + ), + pytest.param( + ['next'], + {'sets.next': {'dims': {'from': 'g', 'to': 'g'}, 'missing': 'absent'}}, + "{entry} 'w' names the set 'next', which has two columns over one dimension, and an entry is over " + 'each dimension once.', + id='a-relation-with-two-columns-over-one-dimension', + ), + pytest.param( + ['every'], + {'sets.every': {'dims': ['g', 'h']}}, + "{entry} 'w': dims: names 'every', which under missing: refused has a row at every combination, so it " + "leaves nothing out. Write missing: absent on 'every', or name its dimensions in its place.", + id='a-relation-with-a-row-everywhere', + ), + ], +) +def test_a_relation_in_dims_that_says_nothing_is_refused( + kind: str, sets: list[str], patch: dict[str, Any], fragment: str +): + """Each refusal names the rewrite, alike on a parameter and on a variable.""" + entry, _ = _IN_KINDS[kind] + declared = {key: value for key, value in patch.items() if key.startswith(('sets.', 'maps.'))} + fields = {key: value for key, value in patch.items() if key not in declared} + assert fragment.format(entry=entry) in _refusal(**_in(kind, *sets, **fields), **declared) + + def test_a_masked_parameter_is_not_read_as_its_fill_outside_the_mask(): """Outside its mask a parameter is absent, so a comparison on it is false there, whatever `missing:` says. @@ -2960,6 +3186,14 @@ def test_a_masked_parameter_is_not_read_as_its_fill_outside_the_mask(): assert _spec(**patch).program.expressions['level'] is not None, 'unmasked, a missing row reads 0 and no case holds' masked = {**patch, **_MASKED, 'parameters.cost': {**patch['parameters.cost'], 'where': 'pair[kind]'}} assert "cases 'high' and 'low' both claim the value where cost has no row" in _refusal(**masked) + indexed = { + **patch, + 'sets.pair': _IN['sets.pair'], + 'parameters.cost': {**patch['parameters.cost'], 'dims': ['pair']}, + } + assert "cases 'high' and 'low' both claim the value where cost has no row" in _refusal(**indexed), ( + 'outside its index set a parameter is absent too, so it has no fill there' + ) @pytest.mark.parametrize('fmt', FORMATS) @@ -2971,6 +3205,18 @@ def test_the_legend_says_where_a_parameter_has_coordinates(fmt: FormatName): assert row.index('only where') < row.index('where the data has no row'), 'the mask comes before the fill' +@pytest.mark.parametrize('kind', _IN_KINDS) +@pytest.mark.parametrize('fmt', FORMATS) +def test_the_legend_prints_an_index_set_as_the_domain(fmt: FormatName, kind: str): + """A relation in `dims:` is where the entry has coordinates, so the legend prints its rows after `over`, not as a mask.""" + spec = _spec(**_in(kind, 'pair', where='flag'), **{'variables.q.bounds': {'upper': 'c'}}) + (row,) = [line for line in typeset(spec, fmt).splitlines() if re.search(r'w[`}] over ', line)] + assert 'pair' in row[: row.index(' over ') + 60], 'the domain is the rows of the set' + if kind == 'parameters': + assert row.index('pair') < row.index('only where') < row.index('flag'), 'the domain, then the mask inside it' + assert 'pair' not in row[row.index('only where') :], 'the set is not printed again as a mask' + + def test_a_given_parameter_has_no_missing(): """The file that declares the parameter owns what a missing row means, as it owns a variable's bounds.""" message = _refusal(**{'given.parameters.d': {'dims': ['g'], 'missing': 'absent'}}) @@ -2987,7 +3233,7 @@ def test_a_given_parameter_has_no_missing(): ) def test_a_relation_says_whether_every_combination_needs_a_row(declared, read): """A relation takes `missing:`, and reads `refused` where it is left out.""" - spec = to_spec(varied(SMALL_MODEL, **{'relations.pair': declared})) + spec = to_spec(varied(SMALL_MODEL, **{'sets.pair': declared})) assert spec.program.relations['pair'].missing == read @@ -3010,23 +3256,23 @@ def test_a_map_says_whether_every_row_needs_a_label(declared, partial): ('patch', 'fragment'), [ pytest.param( - {'relations.link': {'dims': ['g', 'h'], 'missing': 0}}, - 'missing: 0 on a relation, which takes refused or absent', + {'sets.link': {'dims': ['g', 'h'], 'missing': 0}}, + 'missing: 0 on a set, which takes refused or absent', id='a-number-on-a-relation', ), pytest.param( - {'relations.link': {'dims': ['g', 'h'], 'missing': 'h1'}}, - 'missing: h1 on a relation', + {'sets.link': {'dims': ['g', 'h'], 'missing': 'h1'}}, + 'missing: h1 on a set', id='a-label-on-a-relation', ), pytest.param( - {'relations.link': {'dims': ['g', 'h'], 'missing': True}}, - 'missing: true on a relation', + {'sets.link': {'dims': ['g', 'h'], 'missing': True}}, + 'missing: true on a set', id='a-flag-on-a-relation', ), pytest.param( - {'relations.link': {'dims': ['g', 'h'], 'missing': None}}, - 'missing: null on a relation says nothing', + {'sets.link': {'dims': ['g', 'h'], 'missing': None}}, + 'missing: null on a set says nothing', id='null-on-a-relation', ), pytest.param( @@ -3079,6 +3325,11 @@ def test_a_variable_missing_outside_what_the_variable_takes_is_refused(declared, assert fragment in _refusal(**{'variables.p': {'dims': ['g'], 'where': 'c', **declared}}) +def test_a_number_stands_for_a_variable_outside_its_index_set(): + """Outside its `in:` a variable does not exist, so `missing:` has coordinates to stand for, as outside a `where:`.""" + assert _spec(**_in('variables', 'pair', missing=0)).program.variables['w'].missing == 0 + + def test_a_variable_missing_is_not_blamed_for_a_where_that_failed_to_load(): """A `where:` that fails to load is absent from what `missing:` is checked against, not null.""" message = _refusal(**{'variables.p': {'dims': ['g'], 'where': 3, 'missing': 0}}) diff --git a/tests/typesetting/golden/latex.out b/tests/typesetting/golden/latex.out index 6c1023216..39ed54132 100644 --- a/tests/typesetting/golden/latex.out +++ b/tests/typesetting/golden/latex.out @@ -10,12 +10,12 @@ \paragraph{Sets} \begin{description} \item[{$\mathcal{B}$}] index $b$ --- \texttt{bp} -\item[{$\mathcal{U}$}] index $u$ --- \texttt{bus} with $\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}$, \texttt{connection} is \texttt{absent} where the data has no row, \texttt{bus[area]} is \texttt{absent} where the data gives no label +\item[{$\mathcal{U}$}] index $u$ --- \texttt{bus} with $\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}$, \texttt{bus\_zone} is \texttt{absent} where the data has no row, \texttt{connection} is \texttt{absent} where the data has no row, \texttt{bus[area]} is \texttt{absent} where the data gives no label \item[{$\mathcal{G}$}] index $g$ --- \texttt{generator} with $\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}$, \texttt{connection} is \texttt{absent} where the data has no row \item[{$\mathcal{S}$}] index $s$ --- \texttt{season} with $\mathrm{season}: \mathcal{T} \to \mathcal{S}$ \item[{$\mathcal{T}$}] index $t$ --- \texttt{snapshot} (\texttt{int} coordinates) with $\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}$ \item[{$\mathcal{E}$}] index $e$ --- \texttt{technology} with $\mathrm{technology}: \mathcal{G} \to \mathcal{E}$ -\item[{$\mathcal{Z}$}] index $z$ --- \texttt{zone} (labels \texttt{north}, \texttt{south}) with $\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}$, \texttt{bus[area]} is \texttt{absent} where the data gives no label +\item[{$\mathcal{Z}$}] index $z$ --- \texttt{zone} (labels \texttt{north}, \texttt{south}) with $\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}$, \texttt{bus\_zone} is \texttt{absent} where the data has no row, \texttt{bus[area]} is \texttt{absent} where the data gives no label \end{description} \paragraph{Parameters} @@ -31,13 +31,14 @@ \item[{$\mathrm{growth}$}] \texttt{growth} (scalar) \item[{$\mathrm{is\_flexible}$}] \texttt{is\_flexible} over $\mathcal{G}$, \texttt{false} where the data has no row \item[{$\mathrm{lead}$}] \texttt{lead} over $\mathcal{G}$ -\item[{$\mathrm{link\_eff}$}] \texttt{link\_eff} over $\mathcal{U} \times \mathcal{G}$, only where $\left( u,\ g \right) \in \mathrm{connection}$, $1$ where the data has no row +\item[{$\mathrm{link\_eff}$}] \texttt{link\_eff} over $\left( u,\ g \right) \in \mathrm{connection}$, $1$ where the data has no row \item[{$\mathrm{load}$}] \texttt{load} over $\mathcal{U} \times \mathcal{T}$ \item[{$\mathrm{min\_up}$}] \texttt{min\_up} over $\mathcal{G}$ \item[{$\mathrm{p}^{\mathrm{max}}$}] \texttt{p\_max} over $\mathcal{G}$, $\infty$ where the data has no row \item[{$\mathrm{p}^{\mathrm{min}}$}] \texttt{p\_min} over $\mathcal{G}$ \item[{$\mathrm{tech\_cap}$}] \texttt{tech\_cap} over $\mathcal{U} \times \mathcal{E}$, \texttt{absent} where the data has no row \item[{$\mathrm{zone\_cap}$}] \texttt{zone\_cap} over $\mathcal{Z}$ +\item[{$\mathrm{zone\_share}$}] \texttt{zone\_share} over $\left( u,\ z \right) \in \mathrm{bus\_zone}$, $1$ where the data has no row \end{description} \paragraph{Conditions} @@ -48,6 +49,7 @@ \paragraph{Variables} \begin{description} +\item[{$\mathit{flow}$}] \texttt{flow} over $\left( u,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T}$ \item[{$\mathit{fuel}$}] \texttt{fuel} over $\mathcal{G} \times \mathcal{T}$ \item[{$\mathit{headroom}$}] \texttt{headroom} (scalar), $0$ where the mask leaves it out \item[{$\mathit{heat}$}] \texttt{heat} over $\mathcal{G} \times \mathcal{T}$ @@ -93,6 +95,7 @@ \paragraph{Variable domains} \begin{align} +\text{flow} && \mathit{flow}_{u,g,t} & \ge 0 && \forall\, \left( u,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T} \,:\, \mathrm{p}^{\mathrm{max}}_{g} > 0 \\ \text{fuel} && \mathit{fuel}_{g,t} & \ge 0 && \forall\, g \in \mathcal{G},\ t \in \mathcal{T} \\ \text{headroom} && \mathit{headroom} & \ge 0 && \text{where } \mathrm{budget} \text{ is defined} \\ \text{heat} && \mathit{heat}_{g,t} & \ge 0 && \forall\, g \in \mathcal{G},\ t \in \mathcal{T} \\ diff --git a/tests/typesetting/golden/markdown.out b/tests/typesetting/golden/markdown.out index 4451ad904..82aa1f790 100644 --- a/tests/typesetting/golden/markdown.out +++ b/tests/typesetting/golden/markdown.out @@ -7,12 +7,12 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 | Symbol | Meaning | |---|---| | $`\mathcal{B}`$ | index $`b`$ — `bp` | -| $`\mathcal{U}`$ | index $`u`$ — `bus` with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}`$, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | +| $`\mathcal{U}`$ | index $`u`$ — `bus` with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}`$, `bus_zone` is `absent` where the data has no row, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | | $`\mathcal{G}`$ | index $`g`$ — `generator` with $`\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}`$, `connection` is `absent` where the data has no row | | $`\mathcal{S}`$ | index $`s`$ — `season` with $`\mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | | $`\mathcal{T}`$ | index $`t`$ — `snapshot` (`int` coordinates) with $`\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | | $`\mathcal{E}`$ | index $`e`$ — `technology` with $`\mathrm{technology}: \mathcal{G} \to \mathcal{E}`$ | -| $`\mathcal{Z}`$ | index $`z`$ — `zone` (labels `north`, `south`) with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}`$, `bus[area]` is `absent` where the data gives no label | +| $`\mathcal{Z}`$ | index $`z`$ — `zone` (labels `north`, `south`) with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}`$, `bus_zone` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | #### Parameters @@ -29,13 +29,14 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 | $`\mathrm{growth}`$ | `growth` (scalar) | | $`\mathrm{is\_flexible}`$ | `is_flexible` over $`\mathcal{G}`$, `false` where the data has no row | | $`\mathrm{lead}`$ | `lead` over $`\mathcal{G}`$ | -| $`\mathrm{link\_eff}`$ | `link_eff` over $`\mathcal{U} \times \mathcal{G}`$, only where $`\left( u,\ g \right) \in \mathrm{connection}`$, $`1`$ where the data has no row | +| $`\mathrm{link\_eff}`$ | `link_eff` over $`\left( u,\ g \right) \in \mathrm{connection}`$, $`1`$ where the data has no row | | $`\mathrm{load}`$ | `load` over $`\mathcal{U} \times \mathcal{T}`$ | | $`\mathrm{min\_up}`$ | `min_up` over $`\mathcal{G}`$ | | $`\mathrm{p}^{\mathrm{max}}`$ | `p_max` over $`\mathcal{G}`$, $`\infty`$ where the data has no row | | $`\mathrm{p}^{\mathrm{min}}`$ | `p_min` over $`\mathcal{G}`$ | | $`\mathrm{tech\_cap}`$ | `tech_cap` over $`\mathcal{U} \times \mathcal{E}`$, `absent` where the data has no row | | $`\mathrm{zone\_cap}`$ | `zone_cap` over $`\mathcal{Z}`$ | +| $`\mathrm{zone\_share}`$ | `zone_share` over $`\left( u,\ z \right) \in \mathrm{bus\_zone}`$, $`1`$ where the data has no row | #### Conditions @@ -48,6 +49,7 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 | Symbol | Meaning | |---|---| +| $`\mathit{flow}`$ | `flow` over $`\left( u,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T}`$ | | $`\mathit{fuel}`$ | `fuel` over $`\mathcal{G} \times \mathcal{T}`$ | | $`\mathit{headroom}`$ | `headroom` (scalar), $`0`$ where the mask leaves it out | | $`\mathit{heat}`$ | `heat` over $`\mathcal{G} \times \mathcal{T}`$ | @@ -102,6 +104,12 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 #### Variable domains +**`flow`** + +```math +\mathit{flow}_{u,g,t} \ge 0 \qquad \forall\, \left( u,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T} \,:\, \mathrm{p}^{\mathrm{max}}_{g} > 0 +``` + **`fuel`** ```math diff --git a/tests/typesetting/golden/model.yaml b/tests/typesetting/golden/model.yaml index a98643e17..5e169278e 100644 --- a/tests/typesetting/golden/model.yaml +++ b/tests/typesetting/golden/model.yaml @@ -21,9 +21,10 @@ dimensions: technology: { dtype: str } bp: { dtype: int, ordered: true } # the breakpoints every curve below runs through -relations: +sets: gen_zone: { dims: [generator, snapshot] } # a relation over two dimensions, with a map: a call sums one away and joins on the other connection: { dims: [generator, bus], missing: absent } # a relation with no map: many-to-many, read only by sum with both ends named + bus_zone: { dims: [bus, zone], missing: absent } # the pairs that exist maps: generator: { bus: { dim: bus }, technology: { dim: technology } } # two maps of one subject, read to both at once @@ -43,7 +44,8 @@ parameters: is_flexible: { dims: [generator], dtype: bool, missing: false } zone_cap: { dims: [zone] } tech_cap: { dims: [bus, technology], missing: absent } - link_eff: { dims: [generator, bus], where: connection, missing: 1 } + link_eff: { dims: [connection], missing: 1 } # over the pairs a relation lists; a `where:` may not test them + zone_share: { dims: [bus_zone], missing: 1 } # over the pairs that exist: data refused outside them, with no `where:` of its own area_weight: { dims: [bus], where: "bus[area]" } # a map's label as a condition: the buses that have an area min_up: { dims: [generator], dtype: int } eta: { dims: [generator] } # a Greek name that is *given*, so the rule wins and it prints as the word @@ -81,6 +83,10 @@ variables: reserve: # an empty dims: a scalar entry, whose line carries no quantifier dims: [] bounds: { lower: 0 } + flow: # a domain of the pairs `connection` lists, at every snapshot, and a `where:` inside it + dims: [snapshot, connection] + where: "p_max > 0" + bounds: { lower: 0 } headroom: # scalar too, but masked, so the condition stands with no set beside it; 0, so the legend names the value dims: [] where: "budget" @@ -269,9 +275,8 @@ constraints: relational: # a sum through a relation with no map: the domain is a row of the relation rather than a label dims: [snapshot, bus] expression: sum(p, over=generator, by=connection[bus]) <= load - connected: # a relation as a where: the row of the frame has to be a member of the relation - dims: [snapshot, generator, bus] - where: "connection" + connected: # a relation in dims: the row of the frame has to be a member of the relation + dims: [snapshot, connection] expression: p <= load representative: # a map into its own dimension, read both ways: the frame is unchanged and the index is primed dims: [snapshot] diff --git a/tests/typesetting/golden/typst.out b/tests/typesetting/golden/typst.out index 0722349a2..224b0b798 100644 --- a/tests/typesetting/golden/typst.out +++ b/tests/typesetting/golden/typst.out @@ -5,12 +5,12 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 == Sets / $cal(B)$: index $b$ --- `bp` -/ $cal(U)$: index $u$ --- `bus` with $upright("area"): cal(U) arrow.r cal(Z), upright("zone"): cal(U) arrow.r cal(Z), upright("connection") subset.eq cal(U) times cal(G), upright("bus"): cal(G) arrow.r cal(U)$, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label +/ $cal(U)$: index $u$ --- `bus` with $upright("area"): cal(U) arrow.r cal(Z), upright("zone"): cal(U) arrow.r cal(Z), upright("bus_zone") subset.eq cal(U) times cal(Z), upright("connection") subset.eq cal(U) times cal(G), upright("bus"): cal(G) arrow.r cal(U)$, `bus_zone` is `absent` where the data has no row, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label / $cal(G)$: index $g$ --- `generator` with $upright("connection") subset.eq cal(U) times cal(G), upright("gen_zone") subset.eq cal(G) times cal(T), upright("zone"): upright("gen_zone") arrow.r cal(Z), upright("bus"): cal(G) arrow.r cal(U), upright("technology"): cal(G) arrow.r cal(E)$, `connection` is `absent` where the data has no row / $cal(S)$: index $s$ --- `season` with $upright("season"): cal(T) arrow.r cal(S)$ / $cal(T)$: index $t$ --- `snapshot` (`int` coordinates) with $upright("gen_zone") subset.eq cal(G) times cal(T), upright("zone"): upright("gen_zone") arrow.r cal(Z), upright("rep"): cal(T) arrow.r cal(T), upright("season"): cal(T) arrow.r cal(S)$ / $cal(E)$: index $e$ --- `technology` with $upright("technology"): cal(G) arrow.r cal(E)$ -/ $cal(Z)$: index $z$ --- `zone` (labels `north`, `south`) with $upright("area"): cal(U) arrow.r cal(Z), upright("zone"): cal(U) arrow.r cal(Z), upright("zone"): upright("gen_zone") arrow.r cal(Z)$, `bus[area]` is `absent` where the data gives no label +/ $cal(Z)$: index $z$ --- `zone` (labels `north`, `south`) with $upright("area"): cal(U) arrow.r cal(Z), upright("zone"): cal(U) arrow.r cal(Z), upright("bus_zone") subset.eq cal(U) times cal(Z), upright("zone"): upright("gen_zone") arrow.r cal(Z)$, `bus_zone` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label == Parameters / $upright("area_weight")$: `area_weight` over $cal(U)$, only where $upright("area")(u) upright(" is defined")$ @@ -24,19 +24,21 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 / $upright("growth")$: `growth` (scalar) / $upright("is_flexible")$: `is_flexible` over $cal(G)$, `false` where the data has no row / $upright("lead")$: `lead` over $cal(G)$ -/ $upright("link_eff")$: `link_eff` over $cal(U) times cal(G)$, only where $(u, g) in upright("connection")$, $1$ where the data has no row +/ $upright("link_eff")$: `link_eff` over $(u, g) in upright("connection")$, $1$ where the data has no row / $upright("load")$: `load` over $cal(U) times cal(T)$ / $upright("min_up")$: `min_up` over $cal(G)$ / $upright("p")^(upright("max"))$: `p_max` over $cal(G)$, $infinity$ where the data has no row / $upright("p")^(upright("min"))$: `p_min` over $cal(G)$ / $upright("tech_cap")$: `tech_cap` over $cal(U) times cal(E)$, `absent` where the data has no row / $upright("zone_cap")$: `zone_cap` over $cal(Z)$ +/ $upright("zone_share")$: `zone_share` over $(u, z) in upright("bus_zone")$, $1$ where the data has no row == Conditions / $upright("dispatchable")$: `dispatchable` over $cal(G)$ --- a unit that dispatches and cannot be turned down / $upright("priced_dispatch")$: `priced_dispatch` over $cal(G)$ == Variables +/ $italic("flow")$: `flow` over $(u, g) in upright("connection"), t in cal(T)$ / $italic("fuel")$: `fuel` over $cal(G) times cal(T)$ / $italic("headroom")$: `headroom` (scalar), $0$ where the mask leaves it out / $italic("heat")$: `heat` over $cal(G) times cal(T)$ @@ -76,7 +78,8 @@ $ upright("dispatchable") & upright("dispatchable")_(g) & arrow.l.r.double.long == Variable domains #set math.equation(numbering: "(1)") -$ upright("fuel") & italic("fuel")_(g,t) & >= 0 & forall g in cal(G), t in cal(T) \ +$ upright("flow") & italic("flow")_(u,g,t) & >= 0 & forall (u, g) in upright("connection"), t in cal(T) colon upright("p")^(upright("max"))_(g) > 0 \ + upright("fuel") & italic("fuel")_(g,t) & >= 0 & forall g in cal(G), t in cal(T) \ upright("headroom") & italic("headroom") & >= 0 & upright("where ") upright("budget") upright(" is defined") \ upright("heat") & italic("heat")_(g,t) & >= 0 & forall g in cal(G), t in cal(T) \ upright("on") & italic("on")_(g,t) & in {0, 1} & forall g in cal(G), t in cal(T) \ diff --git a/tests/typesetting/test_walk.py b/tests/typesetting/test_walk.py index 0c9b3fcc8..64ae4a6ea 100644 --- a/tests/typesetting/test_walk.py +++ b/tests/typesetting/test_walk.py @@ -669,7 +669,7 @@ def test_every_operator_probe_renders(path, name: FormatName, fmt: Format): 'bus': {'dtype': 'str'}, 'technology': {'dtype': 'str'}, }, - 'relations': {'gen_zone': {'dims': ['generator', 'snapshot']}}, + 'sets': {'gen_zone': {'dims': ['generator', 'snapshot']}}, 'maps': { 'gen_zone': {'zone': {'dim': 'zone'}}, 'generator': {'bus': {'dim': 'bus'}, 'technology': {'dim': 'technology'}}, @@ -700,7 +700,7 @@ def test_a_sum_that_reads_no_map_asks_only_that_the_relation_has_the_row(): """ row = _grouped(['snapshot'], 'sum(u, over=generator, by=gen_zone[snapshot]) <= cap') assert r'\sum_{g \in \mathcal{G} \,:\, \left( g,\ t \right) \in \mathrm{gen\_zone}}' in row, ( - 'the condition is that the row is in the relation, and the unread map is written nowhere' + 'the condition is that the row is in the set, and the unread map is written nowhere' ) @@ -911,7 +911,7 @@ def test_an_assumption_prints_under_its_own_heading(name: FormatName, fmt: Forma text = typeset(model, name, legend=False) section = text[text.index('Assumptions') :] assert fmt.subscript(fmt.upright('cost'), ['g']) in section - assert f'{fmt.operators["gt"]} 0' in section, 'the line aligns on the relation, which leads the right side' + assert f'{fmt.operators["gt"]} 0' in section, 'the line aligns on the set, which leads the right side' assert 'Assumptions' not in typeset(DISPATCH_MODEL, name, legend=False), ( 'a model that assumes nothing of its data prints no heading for it' ) diff --git a/tools/notation.py b/tools/notation.py index c47c76ceb..a33b7f068 100644 --- a/tools/notation.py +++ b/tools/notation.py @@ -71,6 +71,7 @@ 'spare': 'Unbounded integer domain', 'reserve': 'Scalar variable', 'headroom': 'Scalar variable with a condition', + 'flow': 'Variable over the rows of a relation', 'weight': 'Variable in a special ordered set', 'fuel': 'Second axis of a curve', 'heat': 'Third axis of a curve', @@ -295,7 +296,7 @@ def legend() -> str: #: What the legend is made of. No equation comes from these, so they are shown #: once, together, above the tables they turn into. -DECLARED = ('dimensions', 'relations', 'maps', 'parameters') +DECLARED = ('dimensions', 'sets', 'maps', 'parameters') def preamble(text: str) -> str: diff --git a/tools/pypsa_split.py b/tools/pypsa_split.py index 20bdbac6f..ec1dfb9d7 100644 --- a/tools/pypsa_split.py +++ b/tools/pypsa_split.py @@ -54,7 +54,7 @@ SOURCE = Path('examples/pypsa.yaml') SECTIONS = ( 'dimensions', - 'relations', + 'sets', 'maps', 'parameters', 'variables', @@ -64,7 +64,7 @@ 'constraints', 'assumptions', ) -FRAME = ('dimensions', 'relations', 'maps') +FRAME = ('dimensions', 'sets', 'maps') IDENT = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') READ = re.compile(r'(? Date: Mon, 16 Nov 2026 16:41:23 +0000 Subject: [PATCH 2/9] docs(changelog): the set line links #1018 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1xxh6NMJN6mrYY34Q --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dc07cdfa..f1078bf08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ it releases that version ([RELEASING.md](https://github.com/energy-models/mathsp ## Upcoming version -- feat(language)!: an entry or a sum may range over a set ([#0000](https://github.com/energy-models/mathspec/pull/0000)) +- feat(language)!: an entry or a sum may range over a set ([#1018](https://github.com/energy-models/mathspec/pull/1018)) - Breaks a file: `relations:` is `sets:`. A set named in the `dims:` of a parameter, a variable or a constraint, or in `over=`, stands for its dimensions at the combinations it lists. - Breaks a file: the `where:` of a parameter, a variable or a constraint no longer takes a set's bare name. Name the set in `dims:`. - feat(language)!: maps give labels, relations give rows ([#1015](https://github.com/energy-models/mathspec/pull/1015)) From b22163832e4eb371833a783b4450d20ce0dcde5c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 17:44:56 +0000 Subject: [PATCH 3/9] fix(language): a misspelled key is not told of `in_` `in:` was answered "Did you mean 'in_'?", the field the load fills from `dims:` and a file may not write. The suggestion now leaves it out. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1xxh6NMJN6mrYY34Q --- src/mathspec/spec.py | 4 +++- tests/test_validation.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/mathspec/spec.py b/src/mathspec/spec.py index 09fc22ebf..afcdc9b80 100644 --- a/src/mathspec/spec.py +++ b/src/mathspec/spec.py @@ -149,10 +149,12 @@ def _reject_unknown_keys(cls, data: object) -> object: return data known = set(_written_keys(cls).values()) unknown = [k for k in data if isinstance(k, str) and k not in known] + written = known - {'in_'} # the load writes `in_` from `dims:`; a file never does if unknown: raise ValueError( '\n'.join( - f"unknown key '{k}' in {cls._label}. {did_you_mean(k, known, label='Valid keys')}" for k in unknown + f"unknown key '{k}' in {cls._label}. {did_you_mean(k, written, label='Valid keys')}" + for k in unknown ) ) return data diff --git a/tests/test_validation.py b/tests/test_validation.py index b2cf193f0..d95b96e7d 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -3796,3 +3796,10 @@ def test_a_divisor_a_shift_fills_with_zero_is_refused(divisor, patch): def test_a_divisor_a_shift_fills_with_another_value_loads(divisor): spec = to_spec(varied(SMALL_MODEL, **{'constraints.cap': {'dims': ['g'], 'expression': f'p / ({divisor}) <= 1'}})) assert 'cap' in spec.program.constraints + + +def test_a_misspelled_key_is_not_told_of_the_field_the_load_writes(): + """`in:` was told "Did you mean 'in_'?": a field the load fills from `dims:`, which a file may not write.""" + message = _refusal(**{'parameters.c.in': ['g']}) + assert "unknown key 'in'" in message + assert 'in_' not in message, 'the field the load writes is no key a file may use' From 12315cfa0d9cb52a19ea5e8a46c1238ebde0695e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 17:50:24 +0000 Subject: [PATCH 4/9] Merge feat/dimension-alias into feat/maps-sets The TIMES examples in this PR's spelling: sets: in place of relations:, act over [prc_ts, rtp_vintyr], and ire_flo, trade_cost and export over the set top_ire. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1xxh6NMJN6mrYY34Q --- docs/examples/times/capacity.md | 16 ++++++++-------- docs/examples/times/trade.md | 24 +++++++++++------------- examples/times/capacity.yaml | 5 ++--- examples/times/trade.yaml | 11 ++++------- 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/docs/examples/times/capacity.md b/docs/examples/times/capacity.md index c8824589d..4b3fc9e29 100644 --- a/docs/examples/times/capacity.md +++ b/docs/examples/times/capacity.md @@ -17,9 +17,10 @@ without storage, with the upper bound only. The GAMS conditions map to the spec as follows: -- `RTP_VINTYR` and `PRC_TS` are [relations](../../reference/language/relations.md), - and `act` exists only at the coordinates that both list. -- `TS_MAP(R,S,TS)` is the relation `ts_map`, which names `timeslice` twice +- `RTP_VINTYR` and `PRC_TS` are [sets](../../reference/language/relations.md), + and `act` names both in `dims:`, so it is over their dimensions and exists + only at the coordinates that both list. +- `TS_MAP(R,S,TS)` is the set `ts_map`, which names `timeslice` twice under two column names. - `COEF_AF(…,'UP')` is the `where:` of `eq_capact_up`. - `NCAP_PASTI$PASTYEAR(V)` reads `0` where the data has no row. @@ -41,7 +42,7 @@ dimensions: process: { description: a technology } timeslice: { description: "a slice of the year, at any level of the timeslice tree" } -relations: +sets: rtp_vintyr: description: the vintages of a process that exist in a period (RTP_VINTYR) dims: [region, vintage, period, process] @@ -69,8 +70,7 @@ variables: cap: { dims: [region, period, process], bounds: { lower: 0 }, description: installed capacity (VAR_CAP) } act: description: activity of a vintage of a process in a timeslice (VAR_ACT) - dims: [region, vintage, period, process, timeslice] - where: rtp_vintyr AND prc_ts + dims: [prc_ts, rtp_vintyr] bounds: { lower: 0 } constraints: @@ -121,7 +121,7 @@ Capacity transfer and capacity–activity from TIMES: EQ\_CPT and the upper form | Symbol | Meaning | |---|---| -| $`\mathit{act}`$ | `act` over $`\mathcal{T} \times \mathcal{P} \times \mathcal{R} \times \mathcal{S} \times \mathcal{V}`$ — activity of a vintage of a process in a timeslice (VAR\_ACT) | +| $`\mathit{act}`$ | `act` over $`\left( p,\ r,\ s \right) \in \mathrm{prc\_ts},\ \left( t,\ p,\ r,\ v \right) \in \mathrm{rtp\_vintyr}`$ — activity of a vintage of a process in a timeslice (VAR\_ACT) | | $`\mathit{cap}`$ | `cap` over $`\mathcal{T} \times \mathcal{P} \times \mathcal{R}`$ — installed capacity (VAR\_CAP) | | $`\mathit{ncap}`$ | `ncap` over $`\mathcal{P} \times \mathcal{R} \times \mathcal{V}`$ — new capacity (VAR\_NCAP) | @@ -130,7 +130,7 @@ Capacity transfer and capacity–activity from TIMES: EQ\_CPT and the upper form **`act`** ```math -\mathit{act}_{t,p,r,s,v} \ge 0 \qquad \forall\, t \in \mathcal{T},\ p \in \mathcal{P},\ r \in \mathcal{R},\ s \in \mathcal{S},\ v \in \mathcal{V} \,:\, \left( t,\ p,\ r,\ v \right) \in \mathrm{rtp\_vintyr} \wedge \left( p,\ r,\ s \right) \in \mathrm{prc\_ts} +\mathit{act}_{t,p,r,s,v} \ge 0 \qquad \forall\, \left( p,\ r,\ s \right) \in \mathrm{prc\_ts},\ \left( t,\ p,\ r,\ v \right) \in \mathrm{rtp\_vintyr} ``` **`cap`** diff --git a/docs/examples/times/trade.md b/docs/examples/times/trade.md index 637308a85..51cc3a512 100644 --- a/docs/examples/times/trade.md +++ b/docs/examples/times/trade.md @@ -13,8 +13,9 @@ which TIMES writes with a second name for each set, `ALIAS(REG,R)` in The spec does the same with an [alias](../../reference/language/dimensions.md#alias): `from_region` has the labels of `region`, and `from_commodity` those of -`commodity`. The relation `top_ire` lists the links that exist, and -`ire_flo`, `trade_cost` and `export` exist only at its rows. The data of +`commodity`. The set `top_ire` lists the links that exist, and +`ire_flo`, `trade_cost` and `export` name it in `dims:`, so they are over its +four dimensions and a data row at a link that does not exist is refused. The data of `ire_flo` has the four columns that the TIMES data already has, and no label is invented. @@ -38,7 +39,7 @@ dimensions: from_region: { alias: region, description: the region an export leaves } from_commodity: { alias: commodity, description: the commodity an export leaves as } -relations: +sets: top_ire: description: the links that exist, from a region and commodity to another (TOP_IRE) dims: [from_region, from_commodity, region, commodity] @@ -47,21 +48,18 @@ relations: parameters: ire_flo: description: the share of an export that arrives as an import (IRE_FLO) - dims: [from_region, from_commodity, region, commodity] - where: top_ire + dims: [top_ire] demand: { dims: [commodity, region], description: what a region needs of a commodity } supply_cost: { dims: [commodity, region], description: the cost of a unit produced in a region } trade_cost: description: the cost of a unit exported along a link - dims: [from_region, from_commodity, region, commodity] - where: top_ire + dims: [top_ire] variables: supply: { dims: [commodity, region], bounds: { lower: 0 }, description: what a region produces of a commodity } export: description: what leaves along a link (VAR_IRE) - dims: [from_region, from_commodity, region, commodity] - where: top_ire + dims: [top_ire] bounds: { lower: 0 } constraints: @@ -96,15 +94,15 @@ Trade between regions from TIMES: IRE\_FLO, the efficiency of a flow from one re | Symbol | Meaning | |---|---| | $`\mathrm{demand}`$ | `demand` over $`\mathcal{C} \times \mathcal{R}`$ — what a region needs of a commodity | -| $`\mathrm{ire\_flo}`$ | `ire_flo` over $`\mathcal{C} \times \mathcal{C} \times \mathcal{R} \times \mathcal{R}`$, only where $`\left( c,\ c',\ r',\ r \right) \in \mathrm{top\_ire}`$ — the share of an export that arrives as an import (IRE\_FLO) | +| $`\mathrm{ire\_flo}`$ | `ire_flo` over $`\left( c,\ c',\ r',\ r \right) \in \mathrm{top\_ire}`$ — the share of an export that arrives as an import (IRE\_FLO) | | $`\mathrm{supply}^{\mathrm{cost}}`$ | `supply_cost` over $`\mathcal{C} \times \mathcal{R}`$ — the cost of a unit produced in a region | -| $`\mathrm{trade\_cost}`$ | `trade_cost` over $`\mathcal{C} \times \mathcal{C} \times \mathcal{R} \times \mathcal{R}`$, only where $`\left( c,\ c',\ r',\ r \right) \in \mathrm{top\_ire}`$ — the cost of a unit exported along a link | +| $`\mathrm{trade\_cost}`$ | `trade_cost` over $`\left( c,\ c',\ r',\ r \right) \in \mathrm{top\_ire}`$ — the cost of a unit exported along a link | #### Variables | Symbol | Meaning | |---|---| -| $`\mathit{export}`$ | `export` over $`\mathcal{C} \times \mathcal{C} \times \mathcal{R} \times \mathcal{R}`$ — what leaves along a link (VAR\_IRE) | +| $`\mathit{export}`$ | `export` over $`\left( c,\ c',\ r',\ r \right) \in \mathrm{top\_ire}`$ — what leaves along a link (VAR\_IRE) | | $`\mathit{supply}`$ | `supply` over $`\mathcal{C} \times \mathcal{R}`$ — what a region produces of a commodity | #### Variable domains @@ -112,7 +110,7 @@ Trade between regions from TIMES: IRE\_FLO, the efficiency of a flow from one re **`export`** ```math -\mathit{export}_{c,c',r',r} \ge 0 \qquad \forall\, c \in \mathcal{C},\ c' \in \mathcal{C},\ r' \in \mathcal{R},\ r \in \mathcal{R} \,:\, \left( c,\ c',\ r',\ r \right) \in \mathrm{top\_ire} +\mathit{export}_{c,c',r',r} \ge 0 \qquad \forall\, \left( c,\ c',\ r',\ r \right) \in \mathrm{top\_ire} ``` **`supply`** diff --git a/examples/times/capacity.yaml b/examples/times/capacity.yaml index f8b544947..53050eeef 100644 --- a/examples/times/capacity.yaml +++ b/examples/times/capacity.yaml @@ -13,7 +13,7 @@ dimensions: process: { description: a technology } timeslice: { description: "a slice of the year, at any level of the timeslice tree" } -relations: +sets: rtp_vintyr: description: the vintages of a process that exist in a period (RTP_VINTYR) dims: [region, vintage, period, process] @@ -41,8 +41,7 @@ variables: cap: { dims: [region, period, process], bounds: { lower: 0 }, description: installed capacity (VAR_CAP) } act: description: activity of a vintage of a process in a timeslice (VAR_ACT) - dims: [region, vintage, period, process, timeslice] - where: rtp_vintyr AND prc_ts + dims: [prc_ts, rtp_vintyr] bounds: { lower: 0 } constraints: diff --git a/examples/times/trade.yaml b/examples/times/trade.yaml index fdbfeee88..089140f1b 100644 --- a/examples/times/trade.yaml +++ b/examples/times/trade.yaml @@ -12,7 +12,7 @@ dimensions: from_region: { alias: region, description: the region an export leaves } from_commodity: { alias: commodity, description: the commodity an export leaves as } -relations: +sets: top_ire: description: the links that exist, from a region and commodity to another (TOP_IRE) dims: [from_region, from_commodity, region, commodity] @@ -21,21 +21,18 @@ relations: parameters: ire_flo: description: the share of an export that arrives as an import (IRE_FLO) - dims: [from_region, from_commodity, region, commodity] - where: top_ire + dims: [top_ire] demand: { dims: [commodity, region], description: what a region needs of a commodity } supply_cost: { dims: [commodity, region], description: the cost of a unit produced in a region } trade_cost: description: the cost of a unit exported along a link - dims: [from_region, from_commodity, region, commodity] - where: top_ire + dims: [top_ire] variables: supply: { dims: [commodity, region], bounds: { lower: 0 }, description: what a region produces of a commodity } export: description: what leaves along a link (VAR_IRE) - dims: [from_region, from_commodity, region, commodity] - where: top_ire + dims: [top_ire] bounds: { lower: 0 } constraints: From 0d31d8e55d33901ad4dc3a8d026f2bc061c778e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 19:51:18 +0000 Subject: [PATCH 5/9] Merge feat/dimension-alias into feat/maps-sets sparse.yaml in this PR's spelling: sets:, and every entry over a set names it in dims:. Restating balance found that over= a set beside by= was refused; a set in over= now stands for its dimensions there too, with the failing test first. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1xxh6NMJN6mrYY34Q --- docs/examples/sparse.md | 34 ++++++++++++++-------------- examples/sparse.yaml | 14 ++++++------ src/mathspec/_expression_resolver.py | 2 +- tests/test_validation.py | 17 ++++++++++++++ 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/docs/examples/sparse.md b/docs/examples/sparse.md index 0581da6cd..147e91f00 100644 --- a/docs/examples/sparse.md +++ b/docs/examples/sparse.md @@ -17,9 +17,9 @@ the sets in one way: - `balance` sums over every row of `connection`, grouped by the bus each port feeds. -`efficiency` and `flow` exist only where `connection` lists a port, so -`missing: 1` fills only those ports, and a data row for a port that -`connection` does not list is dropped. +`efficiency` and `flow` name the set `connection` in `dims:`, so they are over +`unit` and `port` where it lists a port, `missing: 1` fills only those ports, +and a data row for a port that `connection` does not list is refused. [#1019](https://github.com/energy-models/mathspec/issues/1019) compares how three designs for sparse sets write this spec. @@ -35,7 +35,7 @@ dimensions: bus: { description: a node of the network } period: { dtype: int, ordered: true, description: a step of the horizon } -relations: +sets: connection: description: the ports a unit has dims: [unit, port] @@ -50,19 +50,19 @@ maps: bus: { dim: bus, description: the bus a port feeds } parameters: - efficiency: { dims: [unit, port, period], where: connection, missing: 1, description: share of the output a port delivers } + efficiency: { dims: [connection, period], missing: 1, description: share of the output a port delivers } capacity: { dims: [unit], description: the most a unit produces in a period } ramp: { dims: [unit], description: the most a unit's output rises from one period to the next } load: { dims: [bus, period], description: demand to be met } variables: - output: { dims: [unit, period], where: active, bounds: { lower: 0 }, description: what a unit produces } - flow: { dims: [unit, port, period], where: connection, bounds: { lower: 0 }, description: what a port takes of it } + output: { dims: [active], bounds: { lower: 0 }, description: what a unit produces } + flow: { dims: [connection, period], bounds: { lower: 0 }, description: what a port takes of it } constraints: port_limit: description: a port takes no more than the unit's capacity - dims: [unit, port, period] + dims: [connection, period] expression: flow <= capacity split: description: the ports of a unit take what it produces @@ -70,12 +70,12 @@ constraints: expression: sum(flow, over=port) == output ramp_up: description: output rises by at most the ramp from one active period to the next - dims: [unit, period] + dims: [active] expression: output - shift(output, along=period, offset=1) <= ramp balance: description: what the ports deliver to a bus meets its load dims: [bus, period] - expression: sum(efficiency * flow, over=[unit, port], by=connection[bus]) >= load + expression: sum(efficiency * flow, over=connection, by=connection[bus]) >= load objective: sense: minimize @@ -98,7 +98,7 @@ Units that feed buses through the ports they have, and run only in the periods t | Symbol | Meaning | |---|---| | $`\mathrm{capacity}`$ | `capacity` over $`\mathcal{U}`$ — the most a unit produces in a period | -| $`\mathrm{efficiency}`$ | `efficiency` over $`\mathcal{T} \times \mathcal{O} \times \mathcal{U}`$, only where $`\left( o,\ u \right) \in \mathrm{connection}`$, $`1`$ where the data has no row — share of the output a port delivers | +| $`\mathrm{efficiency}`$ | `efficiency` over $`\left( o,\ u \right) \in \mathrm{connection},\ t \in \mathcal{T}`$, $`1`$ where the data has no row — share of the output a port delivers | | $`\mathrm{load}`$ | `load` over $`\mathcal{B} \times \mathcal{T}`$ — demand to be met | | $`\mathrm{ramp}`$ | `ramp` over $`\mathcal{U}`$ — the most a unit's output rises from one period to the next | @@ -106,21 +106,21 @@ Units that feed buses through the ports they have, and run only in the periods t | Symbol | Meaning | |---|---| -| $`\mathit{flow}`$ | `flow` over $`\mathcal{T} \times \mathcal{O} \times \mathcal{U}`$ — what a port takes of it | -| $`\mathit{output}`$ | `output` over $`\mathcal{T} \times \mathcal{U}`$ — what a unit produces | +| $`\mathit{flow}`$ | `flow` over $`\left( o,\ u \right) \in \mathrm{connection},\ t \in \mathcal{T}`$ — what a port takes of it | +| $`\mathit{output}`$ | `output` over $`\left( t,\ u \right) \in \mathrm{active}`$ — what a unit produces | #### Variable domains **`flow`** ```math -\mathit{flow}_{t,o,u} \ge 0 \qquad \forall\, t \in \mathcal{T},\ o \in \mathcal{O},\ u \in \mathcal{U} \,:\, \left( o,\ u \right) \in \mathrm{connection} +\mathit{flow}_{t,o,u} \ge 0 \qquad \forall\, \left( o,\ u \right) \in \mathrm{connection},\ t \in \mathcal{T} ``` **`output`** ```math -\mathit{output}_{t,u} \ge 0 \qquad \forall\, t \in \mathcal{T},\ u \in \mathcal{U} \,:\, \left( t,\ u \right) \in \mathrm{active} +\mathit{output}_{t,u} \ge 0 \qquad \forall\, \left( t,\ u \right) \in \mathrm{active} ``` #### Objective @@ -140,13 +140,13 @@ Units that feed buses through the ports they have, and run only in the periods t **`port_limit`** ```math -\mathit{flow}_{t,o,u} \le \mathrm{capacity}_{u} \qquad \forall\, t \in \mathcal{T},\ o \in \mathcal{O},\ u \in \mathcal{U} +\mathit{flow}_{t,o,u} \le \mathrm{capacity}_{u} \qquad \forall\, t \in \mathcal{T},\ o \in \mathcal{O},\ u \in \mathcal{U} \,:\, \left( o,\ u \right) \in \mathrm{connection} ``` **`ramp_up`** ```math -\mathit{output}_{t,u} - \mathit{output}_{t - 1,u} \le \mathrm{ramp}_{u} \qquad \forall\, t \in \mathcal{T},\ u \in \mathcal{U} +\mathit{output}_{t,u} - \mathit{output}_{t - 1,u} \le \mathrm{ramp}_{u} \qquad \forall\, t \in \mathcal{T},\ u \in \mathcal{U} \,:\, \left( t,\ u \right) \in \mathrm{active} ``` **`split`** diff --git a/examples/sparse.yaml b/examples/sparse.yaml index 3bdd33b1d..358b60747 100644 --- a/examples/sparse.yaml +++ b/examples/sparse.yaml @@ -12,7 +12,7 @@ dimensions: bus: { description: a node of the network } period: { dtype: int, ordered: true, description: a step of the horizon } -relations: +sets: connection: description: the ports a unit has dims: [unit, port] @@ -27,19 +27,19 @@ maps: bus: { dim: bus, description: the bus a port feeds } parameters: - efficiency: { dims: [unit, port, period], where: connection, missing: 1, description: share of the output a port delivers } + efficiency: { dims: [connection, period], missing: 1, description: share of the output a port delivers } capacity: { dims: [unit], description: the most a unit produces in a period } ramp: { dims: [unit], description: the most a unit's output rises from one period to the next } load: { dims: [bus, period], description: demand to be met } variables: - output: { dims: [unit, period], where: active, bounds: { lower: 0 }, description: what a unit produces } - flow: { dims: [unit, port, period], where: connection, bounds: { lower: 0 }, description: what a port takes of it } + output: { dims: [active], bounds: { lower: 0 }, description: what a unit produces } + flow: { dims: [connection, period], bounds: { lower: 0 }, description: what a port takes of it } constraints: port_limit: description: a port takes no more than the unit's capacity - dims: [unit, port, period] + dims: [connection, period] expression: flow <= capacity split: description: the ports of a unit take what it produces @@ -47,12 +47,12 @@ constraints: expression: sum(flow, over=port) == output ramp_up: description: output rises by at most the ramp from one active period to the next - dims: [unit, period] + dims: [active] expression: output - shift(output, along=period, offset=1) <= ramp balance: description: what the ports deliver to a bus meets its load dims: [bus, period] - expression: sum(efficiency * flow, over=[unit, port], by=connection[bus]) >= load + expression: sum(efficiency * flow, over=connection, by=connection[bus]) >= load objective: sense: minimize diff --git a/src/mathspec/_expression_resolver.py b/src/mathspec/_expression_resolver.py index ecc49316a..c3f6861e4 100644 --- a/src/mathspec/_expression_resolver.py +++ b/src/mathspec/_expression_resolver.py @@ -502,7 +502,7 @@ def _over(self, value: ArithmeticNode, operator: str, *, relation: bool) -> tupl f"'{value.relation}', or a dimension." ) return None - names = names_in(value) if relation else _members_of_sets(names_in(value), self.ns) + names = _members_of_sets(names_in(value), self.ns) if not names: self.errors.append( f'{self.context}: {operator}(over=...) is not a name. Write a declared dimension, or a list of them.' diff --git a/tests/test_validation.py b/tests/test_validation.py index d95b96e7d..00bc71fe9 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -3803,3 +3803,20 @@ def test_a_misspelled_key_is_not_told_of_the_field_the_load_writes(): message = _refusal(**{'parameters.c.in': ['g']}) assert "unknown key 'in'" in message assert 'in_' not in message, 'the field the load writes is no key a file may use' + + +def test_a_sum_over_a_set_groups_by_a_map_of_the_set(): + """`sum(q, over=link, by=link[zone])` was refused: beside `by=`, `over=` did not read the set as its dimensions.""" + spec = to_spec( + varied( + SMALL_MODEL, + **_LINKED, + **{ + 'dimensions.z': {}, + 'maps.link': {'zone': {'dim': 'z'}}, + 'variables.y': {'dims': ['link']}, + 'constraints.c': {'dims': ['z'], 'expression': 'sum(y, over=link, by=link[zone]) >= 0'}, + }, + ) + ) + assert spec.program.constraints['c'].dims == ('z',), 'the set is summed away, grouped by its map' From 23ffc3720459a96602da2f8d0a32fd2c965f343e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 19:20:50 +0000 Subject: [PATCH 6/9] feat(language)!: a multi-dim is a dimension declared with over: The `sets:` section goes. A multi-dim is an entry under `dimensions:` with `over:`, the spelling datarecord uses, and `maps:` keys it as it keys a dimension. In `dims:` and `over=` it stands for its members, and two multi-dims in one `dims:` join on the members they share. A multi-dim has no `missing:`: a combination with no row is not one of its labels. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1Y7i5TWDbh9SKx86i --- docs/about/relations-as-linear-maps.md | 5 +- docs/about/sql-and-dataframes.md | 2 +- docs/examples/connection.md | 7 +- docs/examples/operators.md | 8 +- docs/examples/sparse.md | 24 +- docs/examples/times/capacity.md | 22 +- docs/examples/times/trade.md | 9 +- docs/howto/declare-a-column.md | 10 +- docs/howto/index.md | 2 +- docs/reference/composition.md | 62 ++--- docs/reference/glossary.md | 12 +- docs/reference/index.md | 2 +- docs/reference/language/absence.md | 25 +- docs/reference/language/dimensions.md | 11 +- docs/reference/language/expressions.md | 64 ++--- docs/reference/language/file.md | 31 ++- docs/reference/language/operators.md | 12 +- .../parameters-variables-constraints.md | 77 +++--- docs/reference/language/relations.md | 205 ++++++++-------- docs/reference/notation.md | 18 +- docs/reference/reading.md | 29 +-- examples/connection.yaml | 5 +- examples/operators/sum_by_column_lists.yaml | 4 +- examples/operators/sum_by_columns.yaml | 4 +- examples/pypsa.yaml | 47 ++-- examples/pypsa/generator_maintenance.yaml | 5 +- examples/pypsa/link_maintenance.yaml | 5 +- examples/pypsa/process_maintenance.yaml | 5 +- examples/sparse.yaml | 12 +- examples/times/capacity.yaml | 11 +- examples/times/trade.yaml | 5 +- schema/mathspec.schema.json | 112 ++++----- src/mathspec/_composition.py | 24 +- src/mathspec/_condition_parser.py | 2 +- src/mathspec/_dimensions.py | 2 +- src/mathspec/_exclusivity.py | 4 +- src/mathspec/_expression_resolver.py | 13 +- src/mathspec/_renaming.py | 8 +- src/mathspec/_validation.py | 69 +++--- src/mathspec/spec.py | 132 +++++++---- tests/test_alias.py | 4 +- tests/test_canonical.py | 2 +- tests/test_composition.py | 40 ++-- tests/test_dimensions.py | 16 +- tests/test_expansion.py | 4 +- tests/test_lowering.py | 13 +- tests/test_piecewise.py | 13 +- tests/test_pypsa_references.py | 2 +- tests/test_renaming.py | 26 ++- tests/test_schema.py | 13 +- tests/test_validation.py | 219 +++++++++--------- tests/typesetting/golden/latex.out | 4 +- tests/typesetting/golden/markdown.out | 4 +- tests/typesetting/golden/model.yaml | 10 +- tests/typesetting/golden/typst.out | 4 +- tests/typesetting/test_walk.py | 2 +- tools/notation.py | 6 +- tools/pypsa_split.py | 3 +- 58 files changed, 737 insertions(+), 754 deletions(-) diff --git a/docs/about/relations-as-linear-maps.md b/docs/about/relations-as-linear-maps.md index a86386835..8a6ad1823 100644 --- a/docs/about/relations-as-linear-maps.md +++ b/docs/about/relations-as-linear-maps.md @@ -19,8 +19,7 @@ dimensions: snapshot: { dtype: int } generator: { dtype: str } zone: { dtype: str } -sets: - zone_of: { dims: [generator, snapshot], missing: absent } + zone_of: { over: [generator, snapshot] } maps: zone_of: { zone: { dim: zone } } parameters: @@ -70,7 +69,7 @@ generators, snapshots and zones, `zone_of[zone]` in the spec above is a partial $`f: \mathcal{G} \times \mathcal{T} \to \mathcal{Z}`$, because the relation `zone_of` declares `missing: absent`. -A [relation](../reference/language/relations.md#sets) with no map is a +A [relation](../reference/language/relations.md#multi-dims) with no map is a subset with no function behind it. So one generator may sit in several rows, each with its own bus. By default a relation also lists every key tuple, so one that does not must declare `missing: absent`. diff --git a/docs/about/sql-and-dataframes.md b/docs/about/sql-and-dataframes.md index 8089cc696..c168b1d05 100644 --- a/docs/about/sql-and-dataframes.md +++ b/docs/about/sql-and-dataframes.md @@ -70,7 +70,7 @@ are tables of the rows that a tool builds. `dispatch` has one decision variable per coordinate of `(snapshot, generator)`, and `power_balance` has one row per coordinate of `(snapshot, bus)`. -A [set](../reference/language/relations.md#sets) is a table whose +A [multi-dim](../reference/language/relations.md#multi-dims) is a table whose columns are all in its primary key, so it lists which combinations of labels exist. A [map](../reference/language/relations.md#maps) is a column of the table of its subject. `generator: { bus: { dim: bus } }` under `maps:` is a table diff --git a/docs/examples/connection.md b/docs/examples/connection.md index 3a2d8631b..8520d964d 100644 --- a/docs/examples/connection.md +++ b/docs/examples/connection.md @@ -9,7 +9,7 @@ Units with ports is a specification of an optimisation model ([a spec](../reference/glossary.md#spec)) in which each unit attaches to buses through numbered ports. Each port number matches one of the `bus0`, `bus1`, … columns of an open-source power system modelling framework -([PyPSA](https://pypsa.org)). The [set](../reference/language/relations.md) +([PyPSA](https://pypsa.org)). The [multi-dim](../reference/language/relations.md#multi-dims) `connection` lists the bus and the role of each port. Its key is the unit and the port, not the bus, so two ports of one unit may attach to the same bus. @@ -31,12 +31,9 @@ dimensions: unit: { description: "converters, such as an electrolyser or a heat pump" } port: { dtype: int, description: "an attachment of a unit, numbered as PyPSA's bus0, bus1, ..." } role: { description: input or output } - -sets: connection: description: the bus each port of a unit attaches to, and whether the port draws or delivers - dims: [unit, port] - missing: absent + over: [unit, port] maps: connection: { bus: { dim: bus }, role: { dim: role } } diff --git a/docs/examples/operators.md b/docs/examples/operators.md index a1905e087..07c3ec106 100644 --- a/docs/examples/operators.md +++ b/docs/examples/operators.md @@ -158,9 +158,7 @@ dimensions: generator: { dtype: str } period: { dtype: int } zone: { dtype: str } - -sets: - zone_of: { dims: [generator, period] } + zone_of: { over: [generator, period] } maps: zone_of: { zone: { dim: zone } } @@ -197,9 +195,7 @@ dimensions: period: { dtype: int } bus: { dtype: str } technology: { dtype: str } - -sets: - slot_of: { dims: [generator, period] } + slot_of: { over: [generator, period] } maps: slot_of: { bus: { dim: bus }, technology: { dim: technology } } diff --git a/docs/examples/sparse.md b/docs/examples/sparse.md index 147e91f00..bfdff28ab 100644 --- a/docs/examples/sparse.md +++ b/docs/examples/sparse.md @@ -7,18 +7,20 @@ SPDX-License-Identifier: CC-BY-4.0 Units feed buses through the ports they have, and each runs only in the periods it is active. `connection` lists the ports of each unit and the bus -each feeds, and `active` lists the periods of each unit. Each constraint reads -the sets in one way: +each feeds, and `active` lists the periods of each unit. Both are +[multi-dims](../reference/language/relations.md#multi-dims), declared under +`dimensions:` with `over:`. Each constraint reads them in one way: - `port_limit` reads `capacity`, a parameter of each unit, at each port. - `split` sums the flows of a unit over its ports, one column of `connection`. -- `ramp_up` steps `output` along `period` inside the periods of `active`. +- `ramp_up` is over `active`, and steps `output` along `period` inside the + periods of `active`. - `balance` sums over every row of `connection`, grouped by the bus each port feeds. -`efficiency` and `flow` name the set `connection` in `dims:`, so they are over -`unit` and `port` where it lists a port, `missing: 1` fills only those ports, +`efficiency` and `flow` name `connection` in `dims:`, so they are over `unit` +and `port` where it lists a port, `missing: 1` fills only those ports, and a data row for a port that `connection` does not list is refused. [#1019](https://github.com/energy-models/mathspec/issues/1019) compares how three designs for sparse sets write this spec. @@ -34,16 +36,8 @@ dimensions: port: { description: an output of a unit } bus: { description: a node of the network } period: { dtype: int, ordered: true, description: a step of the horizon } - -sets: - connection: - description: the ports a unit has - dims: [unit, port] - missing: absent - active: - description: the periods a unit runs in - dims: [unit, period] - missing: absent + connection: { over: [unit, port], description: the ports a unit has } + active: { over: [unit, period], description: the periods a unit runs in } maps: connection: diff --git a/docs/examples/times/capacity.md b/docs/examples/times/capacity.md index 4b3fc9e29..dc4589c6f 100644 --- a/docs/examples/times/capacity.md +++ b/docs/examples/times/capacity.md @@ -17,11 +17,12 @@ without storage, with the upper bound only. The GAMS conditions map to the spec as follows: -- `RTP_VINTYR` and `PRC_TS` are [sets](../../reference/language/relations.md), - and `act` names both in `dims:`, so it is over their dimensions and exists - only at the coordinates that both list. -- `TS_MAP(R,S,TS)` is the set `ts_map`, which names `timeslice` twice - under two column names. +- `RTP_VINTYR` and `PRC_TS` are + [multi-dims](../../reference/language/relations.md#multi-dims), and `act` + names both in `dims:`. They join on the members they share, `region` and + `process`, so `act` exists only at the coordinates that both list. +- `TS_MAP(R,S,TS)` is the multi-dim `ts_map`, which names `timeslice` twice + under two member names. - `COEF_AF(…,'UP')` is the `where:` of `eq_capact_up`. - `NCAP_PASTI$PASTYEAR(V)` reads `0` where the data has no row. @@ -41,20 +42,15 @@ dimensions: vintage: { dtype: int, description: the period a capacity was built in } process: { description: a technology } timeslice: { description: "a slice of the year, at any level of the timeslice tree" } - -sets: rtp_vintyr: description: the vintages of a process that exist in a period (RTP_VINTYR) - dims: [region, vintage, period, process] - missing: absent + over: [region, vintage, period, process] prc_ts: description: the timeslices a process operates at (PRC_TS) - dims: [region, process, timeslice] - missing: absent + over: [region, process, timeslice] ts_map: description: each timeslice and the timeslices at or below it (TS_MAP) - dims: { region: region, s: timeslice, ts: timeslice } - missing: absent + over: { region: region, s: timeslice, ts: timeslice } parameters: coef_cpt: { dims: [region, vintage, period, process], description: the share of a vintage that still exists in a period } diff --git a/docs/examples/times/trade.md b/docs/examples/times/trade.md index da63a52ac..081fa3f2f 100644 --- a/docs/examples/times/trade.md +++ b/docs/examples/times/trade.md @@ -13,9 +13,9 @@ which TIMES writes with a second name for each set, `ALIAS(REG,R)` in The spec does the same with an [alias](../../reference/language/dimensions.md#alias): `from_region` has the labels of `region`, and `from_commodity` those of -`commodity`. The set `top_ire` lists the processes that carry each link, as +`commodity`. The multi-dim `top_ire` lists the processes that carry each link, as TIMES' `TOP_IRE` does, and `ire_flo`, `trade_cost` and `export` name it in -`dims:`, so they are over its five dimensions and a data row outside its rows +`dims:`, so they are over its five members and a data row outside its rows is refused. The data of `ire_flo` has the five columns that the TIMES data already has, and no label is invented. @@ -39,12 +39,9 @@ dimensions: process: { description: a technology that carries trade } from_region: { alias: region, description: the region an export leaves } from_commodity: { alias: commodity, description: the commodity an export leaves as } - -sets: top_ire: description: the processes that carry each link from a region and commodity to another (TOP_IRE) - dims: [from_region, from_commodity, region, commodity, process] - missing: absent + over: [from_region, from_commodity, region, commodity, process] parameters: ire_flo: diff --git a/docs/howto/declare-a-column.md b/docs/howto/declare-a-column.md index c60088ec1..cbeae5e1f 100644 --- a/docs/howto/declare-a-column.md +++ b/docs/howto/declare-a-column.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: mathspec contributors SPDX-License-Identifier: CC-BY-4.0 --> -# Choose between a dimension, a set, a map and a parameter +# Choose between a dimension, a multi-dim, a map and a parameter This guide shows how to choose the declaration for a [column](../reference/glossary.md#column) of your data, in @@ -20,7 +20,7 @@ Use the first row that fits what you do with the column in the spec: | --------------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | | index parameters, variables or constraints over the rows the column lists | `generator`, `snapshot` | a [`dimension`](../reference/language/dimensions.md) | | look up a label of another dimension, one per row | the bus of each generator | a [`map`](../reference/language/relations.md#maps) of `generator` into `bus`, a dimension | -| list which pairs of two dimensions exist, with no value | `connection`, the buses a generator may connect to | a [`set`](../reference/language/relations.md#sets) with [`missing: absent`](../reference/language/relations.md#missing) | +| list which pairs of two dimensions exist, with no value | `connection`, the buses a generator may connect to | a [multi-dim](../reference/language/relations.md#multi-dims), a dimension with `over:` | | use its numbers in an expression or a bound | `cost` per generator, `availability` per generator and snapshot | a [`parameter`](../reference/language/parameters-variables-constraints.md#parameters) over the dimensions that index it | | test its values in a [condition](../reference/language/expressions.md#conditions) | `fuel`, `is_committable` | a `str` or `bool` parameter | @@ -33,9 +33,7 @@ dimensions: generator: { dtype: str } snapshot: { dtype: int } bus: { dtype: str } - -sets: - connection: { dims: [generator, bus], missing: absent } + connection: { over: [generator, bus] } maps: generator: { bus: { dim: bus } } @@ -105,6 +103,6 @@ spec and has no remark that is not a refusal - [Check a spec without data](check.md), for how to read a refusal or advice. - [Dimensions](../reference/language/dimensions.md), - [sets and maps](../reference/language/relations.md) and + [multi-dims and maps](../reference/language/relations.md) and [parameters](../reference/language/parameters-variables-constraints.md#parameters), for every key of each declaration. diff --git a/docs/howto/index.md b/docs/howto/index.md index 2da60b11f..61bd2f037 100644 --- a/docs/howto/index.md +++ b/docs/howto/index.md @@ -28,7 +28,7 @@ To learn the language from the start, follow the
-- **[Choose between a dimension, a set, a map and a parameter](declare-a-column.md)** +- **[Choose between a dimension, a multi-dim, a map and a parameter](declare-a-column.md)** for a column of your data. - **[State a rule that differs by case](cases.md)**, such as committable and non-committable generators, in one spec. diff --git a/docs/reference/composition.md b/docs/reference/composition.md index 214f0377b..acf5f552b 100644 --- a/docs/reference/composition.md +++ b/docs/reference/composition.md @@ -176,21 +176,21 @@ The objective is the one that `network.yaml` sets. ### Each kind of entry Two fragments may share a [dimension](language/dimensions.md), a -[set](language/relations.md#sets), a [map](language/relations.md#maps), a given entry or a +[multi-dim](language/relations.md#multi-dims), a [map](language/relations.md#maps), a given entry or a sum. -| The entry | What `merge` does | -| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | -| a dimension, a set or a map | accepts it from every fragment that declares it, where every fragment declares it the same way | -| a `description` on a shared dimension, set or map | keeps the first wording in the list | -| `ordered: true` on a shared dimension | makes the dimension ordered if one fragment sets it | -| `missing:` on a shared set or map | requires the same value from every fragment that declares it, where `missing: refused` and no `missing:` are one value | -| an entry under `given:` | checks it against the fragment that declares the name, then folds it in ([given entries](#given-entries)) | -| an expression with `adds_to:` | adds it as a term to the sum in its `adds_to:` ([terms and sums](#terms-and-sums)) | -| `objective` | takes it from the one fragment that sets it, and refuses a second | -| `version` | requires the same value from every fragment | -| `description` at the top of a fragment | drops it | -| any other entry | takes it from the one fragment that declares it, and refuses a second | +| The entry | What `merge` does | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| a dimension, a multi-dim or a map | accepts it from every fragment that declares it, where every fragment declares it the same way | +| a `description` on a shared dimension, multi-dim or map | keeps the first wording in the list | +| `ordered: true` on a shared dimension | makes the dimension ordered if one fragment sets it | +| `missing:` on a shared map | requires the same value from every fragment that declares it, where `missing: refused` and no `missing:` are one value | +| an entry under `given:` | checks it against the fragment that declares the name, then folds it in ([given entries](#given-entries)) | +| an expression with `adds_to:` | adds it as a term to the sum in its `adds_to:` ([terms and sums](#terms-and-sums)) | +| `objective` | takes it from the one fragment that sets it, and refuses a second | +| `version` | requires the same value from every fragment | +| `description` at the top of a fragment | drops it | +| any other entry | takes it from the one fragment that declares it, and refuses a second | ### Terms and sums @@ -432,10 +432,10 @@ patched `dispatch` has no row at `old`. | a whole entry under a new name | adds it | | a named expression as a [bare string](language/named.md#expressions) | replaces only its body, `expression:` or `cases:`, and keeps its `dims:`, description and `adds_to:`, so the new body may have no dimension outside those `dims:` | | `null` on an entry, a field or a section | applies the rule in [`null` in a patch](#null-in-a-patch) | -| a dimension, a set or a map | adds it, or accepts it restated as the base declares it, field by field, where a field left out equals its default | -| `description:` on a dimension, a set or a map of the base | replaces the description of the base, alone or with other fields, and `null` removes it | +| a dimension, a multi-dim or a map | adds it, or accepts it restated as the base declares it, field by field, where a field left out equals its default | +| `description:` on a dimension, a multi-dim or a map of the base | replaces the description of the base, alone or with other fields, and `null` removes it | | `ordered:` on a restated dimension | makes the dimension ordered for `true`, and refuses `false` over an ordered dimension | -| `missing:` on a set or a map, alone or with other fields | changes whether every combination needs a row, or every row a label, as on a parameter | +| `missing:` on a map, alone or with other fields | changes whether every row needs a label, as on a parameter | | an entry under one kind of `given:` | edits, adds or removes it like any entry, and keeps the other kinds | | `version`, `description` | replaces the value of the base | | a field that an earlier patch writes | replaces that value | @@ -518,19 +518,19 @@ A second call with the inverse map returns a spec with the ### What a rename changes -| The spec has | What `rename` does | -| -------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| a name as the key of an entry, in `dims:`, under `maps:`, in `adds_to:` or in `given:` | replaces the name | -| a name in an expression, `where:`, bound, case or assumption | replaces the name, and keeps the spaces and line breaks around it | -| a map or a set column named after its dimension, such as `bus` in `unit[bus]` | replaces it with the dimension | -| a map or a set column named after something else, such as `bus0` in `{bus0: bus}` | keeps it, and refuses a map that changes a name to or from it | -| a longer name that contains a renamed name, such as `Unit_p_nom` for `Unit_p` | keeps the longer name | -| a keyword of an operator, such as `over=` | keeps the keyword | -| a quoted label, a label under `one_of:`, or a number such as `1e6` | keeps the label or the number | -| the name of a case or of a piecewise link | keeps the name, because it is a label of rows | -| an argument of a macro, and the template text that uses it | keeps the argument | -| a fixed value of a field, such as `domain: integer` | keeps the value | -| a `description` | keeps the text as written | +| The spec has | What `rename` does | +| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| a name as the key of an entry, in `dims:`, under `maps:`, in `adds_to:` or in `given:` | replaces the name | +| a name in an expression, `where:`, bound, case or assumption | replaces the name, and keeps the spaces and line breaks around it | +| a map or a multi-dim member named after its dimension, such as `bus` in `unit[bus]` | replaces it with the dimension | +| a map or a multi-dim member named after something else, such as `bus0` in `{bus0: bus}` | keeps it, and refuses a map that changes a name to or from it | +| a longer name that contains a renamed name, such as `Unit_p_nom` for `Unit_p` | keeps the longer name | +| a keyword of an operator, such as `over=` | keeps the keyword | +| a quoted label, a label under `one_of:`, or a number such as `1e6` | keeps the label or the number | +| the name of a case or of a piecewise link | keeps the name, because it is a label of rows | +| an argument of a macro, and the template text that uses it | keeps the argument | +| a fixed value of a field, such as `domain: integer` | keeps the value | +| a `description` | keeps the text as written | ## Refusals @@ -601,9 +601,9 @@ patch 'project.yaml' edits the constraint 'power_balnce', which its base does no ### A dimension restated differently -Over a dimension, a set or a map that its base declares, a patch may +Over a dimension, a multi-dim or a map that its base declares, a patch may change only `description`, a dimension's `ordered`, and the `missing` of a -set or a map. A patch +map. A patch that writes any other field restates the entry. Every other field must then be as the base declares it, and a field the patch leaves out equals its default. diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index bd4cb4978..00330dd4e 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -61,7 +61,7 @@ line](cli.md)). ## Column A column is one variable at one coordinate, so a variable indexed over -`generator` has one column per generator. In a data table or a set, a +`generator` has one column per generator. In a data table or a multi-dim, a column is one named field of the table. ## Compile @@ -91,7 +91,7 @@ and a constraint has one row at each coordinate. ## Data preparation Before the data is attached, data preparation is the work of turning raw data -into the tables that a spec declares as parameters and sets. You compute +into the tables that a spec declares as parameters and multi-dims. You compute in data preparation anything that the mathspec language cannot state ([what counts as data preparation](../about/limits.md#data-preparation)). @@ -175,14 +175,14 @@ expression before any tool reads it. A file declares its own macros under ## Map -A map gives each row of a dimension or a set (its +A map gives each row of a dimension or a multi-dim (its [subject](#subject)) one label of a dimension, such as the bus of each generator. A condition or an operator reads it as `subject[map]`, as in `generator[bus]` ([maps](language/relations.md#maps)). ## Missing row -A missing row is a coordinate for which the table of a parameter or a set +A missing row is a coordinate for which the table of a parameter or a multi-dim has no row. The `missing:` key of the entry sets how a tool treats that coordinate ([`missing` on a parameter](language/absence.md#missing-on-a-parameter)). @@ -239,7 +239,7 @@ program in this sense. ## Row A row is one constraint at one coordinate, so a constraint indexed over -`snapshot` has one row per snapshot. In a data table or a set, a row is +`snapshot` has one row per snapshot. In a data table or a multi-dim, a row is one line of the table ([missing row](#missing-row)). ## Rung @@ -279,7 +279,7 @@ most two consecutive members are non-zero ([sos](language/piecewise.md#sos)). ## Subject -The subject of a map is a dimension or a set, such as `generator` in +The subject of a map is a dimension or a multi-dim, such as `generator` in `generator[bus]` ([maps](language/relations.md#maps)). ## Surface diff --git a/docs/reference/index.md b/docs/reference/index.md index 30c982823..fa76f111b 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -25,7 +25,7 @@ optimisation model ([a spec](glossary.md#spec)). - **[Dimensions](language/dimensions.md)** lists the keys of a named set of labels (a dimension). - **[Sets and maps](language/relations.md)** lists the keys of a set of - combinations of labels (a set) and of the label each row has (a map), + combinations of labels (a multi-dim) and of the label each row has (a map), and how an expression uses them. - **[Expressions](language/expressions.md)** defines the grammar of arithmetic and of conditions, and how dimensions combine. diff --git a/docs/reference/language/absence.md b/docs/reference/language/absence.md index 462fef295..c754e11b3 100644 --- a/docs/reference/language/absence.md +++ b/docs/reference/language/absence.md @@ -53,18 +53,17 @@ contain. Nine constructs make a coordinate absent: -| Construct | What is absent | -| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `where:` on a variable | the variable, at the coordinates the condition excludes ([`where`](#where)) | -| `where:` on a parameter | the parameter, at the coordinates the condition excludes ([`where` on a parameter](parameters-variables-constraints.md#where)) | -| `where:` on a constraint | the row ([`where`](#where)) | -| `shift(x, along=d, offset=n)` without `edge=` | the expression, at the edge coordinate ([shift](operators.md#shift)) | -| a key with no row in a `missing: absent` set | the set, at that key ([sets](relations.md#missing)) | -| a row with no label in a `missing: absent` map | the map, at that row ([maps](relations.md#missing)) | -| a key with no row in a set in `dims:` | the parameter or variable, at that key ([a set in `dims`](parameters-variables-constraints.md#a-set-in-dims)) | -| a missing row of a `missing: absent` parameter | the parameter, at that coordinate ([`missing`](#missing)) | -| `within=` at a coordinate in no group | the expression, at that coordinate ([`within=`](operators.md#within)) | -| a quotient whose divisor is zero | the quotient, at that coordinate ([reported values](#reported-values)) | +| Construct | What is absent | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `where:` on a variable | the variable, at the coordinates the condition excludes ([`where`](#where)) | +| `where:` on a parameter | the parameter, at the coordinates the condition excludes ([`where` on a parameter](parameters-variables-constraints.md#where)) | +| `where:` on a constraint | the row ([`where`](#where)) | +| `shift(x, along=d, offset=n)` without `edge=` | the expression, at the edge coordinate ([shift](operators.md#shift)) | +| a row with no label in a `missing: absent` map | the map, at that row ([maps](relations.md#missing)) | +| a key with no row in a multi-dim | the multi-dim, the parameter or the variable over it, at that key ([a multi-dim in `dims`](parameters-variables-constraints.md#a-multi-dim-in-dims)) | +| a missing row of a `missing: absent` parameter | the parameter, at that coordinate ([`missing`](#missing)) | +| `within=` at a coordinate in no group | the expression, at that coordinate ([`within=`](operators.md#within)) | +| a quotient whose divisor is zero | the quotient, at that coordinate ([reported values](#reported-values)) | No value shifts in at the edge, so a `shift` without `edge=` has no row there: @@ -229,7 +228,7 @@ as [missing rows](../reading.md#missing-rows) describes. ### `missing` on a variable -Outside its `where:` and its [set](parameters-variables-constraints.md#a-set-in-a-variables-dims), +Outside its `where:` and its [multi-dim](parameters-variables-constraints.md#a-multi-dim-in-a-variables-dims), a variable is absent by default. Its `missing:` key can set a number there instead, such as `0`. Under `absent`, a row that needs that coordinate is absent. A file declares `missing: 0` for a quantity that is zero diff --git a/docs/reference/language/dimensions.md b/docs/reference/language/dimensions.md index 8039463ff..2ab0c7382 100644 --- a/docs/reference/language/dimensions.md +++ b/docs/reference/language/dimensions.md @@ -102,7 +102,7 @@ Variable 'p': 'gnrtr' is not a label of 'carrier', which allows only ['ccgt', 'o mathspec lists the allowed labels and suggests two rewrites. With data attached, a tool refuses a label -outside the list: in the table of the dimension, in every column of a set +outside the list: in the table of the dimension, in every member of a multi-dim over it, and in the column of every map into it. Only a dimension takes `one_of`, and a `one_of` key on a parameter is refused @@ -168,8 +168,7 @@ each pair of regions that trade: dimensions: region: { dtype: str } from_region: { alias: region } -sets: - trade: { dims: [from_region, region], missing: absent } + trade: { over: [from_region, region] } parameters: ire_flo: { dims: [trade] } variables: @@ -183,13 +182,13 @@ constraints: `from_region` has the labels, the `dtype` and the order of `region`, so it takes no `dtype:`, `one_of:` or `ordered:` of its own. `ire_flo` is over the -[set](relations.md) `trade`, so over `from_region` and `region` at the pairs it +[multi-dim](relations.md) `trade`, so over `from_region` and `region` at the pairs it lists. The data of `ire_flo` has the columns `from_region`, `region` and `value`, and the data has no table for `from_region`. An alias has the map `from_region[region]`, which sends each label to itself, so `at(exp, by=from_region[region])` reads `exp` at the region a row comes from. The typesetter prints the alias as the index of -`region`, primed, over the same set: +`region`, primed, over the same multi-dim: ```math \mathit{imp}_{r} = \sum_{r' \in \mathcal{R}} \mathrm{ire\_flo}_{r',r} \cdot \mathit{exp}_{r'} \qquad \forall\, r \in \mathcal{R} @@ -200,5 +199,5 @@ An alias names a declared dimension that is no alias itself. A second alias of !!! info "See also" - - [choose between a dimension, a set, a map and a parameter](../../howto/declare-a-column.md) + - [choose between a dimension, a multi-dim, a map and a parameter](../../howto/declare-a-column.md) for a column of data. diff --git a/docs/reference/language/expressions.md b/docs/reference/language/expressions.md index 303641b5a..1794d4c5a 100644 --- a/docs/reference/language/expressions.md +++ b/docs/reference/language/expressions.md @@ -251,18 +251,18 @@ and never to a constraint of the same name. Before any data is attached, [`to_spec`](../api.md#mathspec.to_spec) works out the dimensions of every expression. Each part of an expression (a node) has a set of dimensions: -| Node | Dimensions of the result | Refused when | -| -------------------------------- | ------------------------------ | --------------------------------------------------------------------------------------------- | -| number | `{}` | | -| parameter / variable | its `dims` | | -| `-x`, `+x` | `dims(x)` | | -| `a + b`, `a * b`, `a / b` | `dims(a) ∪ dims(b)` | | -| `sum(x)` | `{}` | `dims(x)` is already empty | -| `sum(x, over=d)` | `dims(x) − {d}` | `d ∉ dims(x)` | -| `sum(x, over=d, by=s[c])` | `(dims(x) − joined) ∪ grouped` | a refusal under [how sets and maps are used](relations.md#how-sets-and-maps-are-used) applies | -| `at(x, by=s[m])` | `(dims(x) − joined) ∪ grouped` | the same refusals apply | -| `shift(x, along=d, offset=n)` | `dims(x)` | `d ∉ dims(x)` | -| `sum_back(x, along=d, window=n)` | `dims(x)` | `d ∉ dims(x)` | +| Node | Dimensions of the result | Refused when | +| -------------------------------- | ------------------------------ | --------------------------------------------------------------------------------------------------------- | +| number | `{}` | | +| parameter / variable | its `dims` | | +| `-x`, `+x` | `dims(x)` | | +| `a + b`, `a * b`, `a / b` | `dims(a) ∪ dims(b)` | | +| `sum(x)` | `{}` | `dims(x)` is already empty | +| `sum(x, over=d)` | `dims(x) − {d}` | `d ∉ dims(x)` | +| `sum(x, over=d, by=s[c])` | `(dims(x) − joined) ∪ grouped` | a refusal under [how multi-dims and maps are used](relations.md#how-multi-dims-and-maps-are-used) applies | +| `at(x, by=s[m])` | `(dims(x) − joined) ∪ grouped` | the same refusals apply | +| `shift(x, along=d, offset=n)` | `dims(x)` | `d ∉ dims(x)` | +| `sum_back(x, along=d, window=n)` | `dims(x)` | `d ∉ dims(x)` | mathspec refuses `sum(p, over=t)` with `p` over `g` alone, and prints: @@ -322,26 +322,26 @@ QUOTED ::= "'" chars "'" | '"' chars '"' A name written alone is a bare name, and what it tests depends on the kind of its entry. -| Written as | Kind of name | Meaning | -| ---------------------------------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` (bare) | parameter | For a number, it is true where the data has a row, whatever the `missing:`. For a `bool`, it is its value, and the `missing:` sets the value at a missing row ([`missing`](parameters-variables-constraints.md#missing)). `0.0` is a row, and `inflow != 0` tests for non-zero | -| `name` (bare) | variable | It is true where the variable exists | -| `name` (bare) | set | It is true where the set has a row. A `missing: absent` set has no row for some combinations ([`missing`](relations.md#missing)), and the bare name selects the combinations it lists. The `where:` of a parameter, a variable or a constraint cannot hold it, because naming the set in [`dims:`](parameters-variables-constraints.md#a-set-in-dims) states those rows | -| `subject[map]` (bare) | map | It is true where the map gives the row of its subject a label. A `missing: absent` map gives some rows no label ([`missing`](relations.md#missing)) | -| `name` (bare) | dimension | mathspec refuses it at load | -| `name` (bare) | named condition | It tests the [named condition](named.md#conditions). A given condition is true or false at each coordinate, as a `bool` parameter is | -| `name OP value` | parameter | It compares element by element. A missing row with no `missing:` value compares false | -| `name OP value` | dimension | It filters the frame's own coordinate column | -| `subject[map] OP value` | map | It compares the label that the map gives each row of its subject. A comparison reads one map. Where the map gives no label, the comparison is false for every operator, `!=` included, so `NOT` of it is true there | -| `s[a] OP t[b]` | two maps | mathspec accepts it where both subjects are over the same dimensions and both maps refer to one dimension. `line[bus0] != line[bus1]` excludes a self-loop. Where either map gives no label, the comparison is false | -| `expression OP expression` | arithmetic over parameters | It compares coordinate by coordinate, over every dimension either side is over ([arithmetic in a comparison](#arithmetic-in-a-comparison)). A side with no value at a coordinate compares false | -| `position(name) OP i` | dimension | It tests where the row sits along the dimension's own order. `0` is first, and a negative number counts from the end | -| `position(name, within=s[m])` | dimension | It tests the same, counted within each group the map makes | -| `count(condition, over=name) OP i` | condition | It compares how many coordinates along the dimension the condition admits ([`count`](#count)) | -| `shift(condition, along=name, offset=i)` | condition | It is the condition `i` coordinates back, and false at the first `i` coordinates | -| `at(condition, by=s[m])` | condition | It is the condition through the map ([`at` in a condition](#at-in-a-condition)), and false where the map gives no label | -| `AND` `OR` `NOT` | | They are case-insensitive. `NOT` applies before `AND`, and `AND` before `OR` | -| `True` / `False` | | `True` is the same as no `where`. `False` leaves an entry with no rows | +| Written as | Kind of name | Meaning | +| ---------------------------------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` (bare) | parameter | For a number, it is true where the data has a row, whatever the `missing:`. For a `bool`, it is its value, and the `missing:` sets the value at a missing row ([`missing`](parameters-variables-constraints.md#missing)). `0.0` is a row, and `inflow != 0` tests for non-zero | +| `name` (bare) | variable | It is true where the variable exists | +| `name` (bare) | set | It is true where the set has a row. A `missing: absent` set has no row for some combinations ([`missing`](relations.md#missing)), and the bare name selects the combinations it lists. The `where:` of a parameter, a variable or a constraint cannot hold it, because naming the set in [`dims:`](parameters-variables-constraints.md#a-multi-dim-in-dims) states those rows | +| `subject[map]` (bare) | map | It is true where the map gives the row of its subject a label. A `missing: absent` map gives some rows no label ([`missing`](relations.md#missing)) | +| `name` (bare) | dimension | mathspec refuses it at load | +| `name` (bare) | named condition | It tests the [named condition](named.md#conditions). A given condition is true or false at each coordinate, as a `bool` parameter is | +| `name OP value` | parameter | It compares element by element. A missing row with no `missing:` value compares false | +| `name OP value` | dimension | It filters the frame's own coordinate column | +| `subject[map] OP value` | map | It compares the label that the map gives each row of its subject. A comparison reads one map. Where the map gives no label, the comparison is false for every operator, `!=` included, so `NOT` of it is true there | +| `s[a] OP t[b]` | two maps | mathspec accepts it where both subjects are over the same dimensions and both maps refer to one dimension. `line[bus0] != line[bus1]` excludes a self-loop. Where either map gives no label, the comparison is false | +| `expression OP expression` | arithmetic over parameters | It compares coordinate by coordinate, over every dimension either side is over ([arithmetic in a comparison](#arithmetic-in-a-comparison)). A side with no value at a coordinate compares false | +| `position(name) OP i` | dimension | It tests where the row sits along the dimension's own order. `0` is first, and a negative number counts from the end | +| `position(name, within=s[m])` | dimension | It tests the same, counted within each group the map makes | +| `count(condition, over=name) OP i` | condition | It compares how many coordinates along the dimension the condition admits ([`count`](#count)) | +| `shift(condition, along=name, offset=i)` | condition | It is the condition `i` coordinates back, and false at the first `i` coordinates | +| `at(condition, by=s[m])` | condition | It is the condition through the map ([`at` in a condition](#at-in-a-condition)), and false where the map gives no label | +| `AND` `OR` `NOT` | | They are case-insensitive. `NOT` applies before `AND`, and `AND` before `OR` | +| `True` / `False` | | `True` is the same as no `where`. `False` leaves an entry with no rows | With a dimension `bp` declared, mathspec prints for `where: "bp"`: diff --git a/docs/reference/language/file.md b/docs/reference/language/file.md index 9f3e3b5fb..62ab9b891 100644 --- a/docs/reference/language/file.md +++ b/docs/reference/language/file.md @@ -14,22 +14,21 @@ The top level of a spec file has **fourteen section keys**, plus `version` and `description`. mathspec accepts any subset of the fourteen, so a file that declares only `dimensions:` loads. -| Key | What the section declares | -| ------------- | ------------------------------------------------------------------------------------------------------------------ | -| `given` | what this file uses and another file declares ([given](parameters-variables-constraints.md#given)) | -| `dimensions` | the sets of labels that entries are indexed over ([dimensions](dimensions.md)) | -| `sets` | named sets of combinations of labels of two or more dimensions ([sets](relations.md#sets)) | -| `maps` | the label of a dimension that each row of a dimension or a set has ([maps](relations.md#maps)) | -| `parameters` | the data the spec expects ([parameters](parameters-variables-constraints.md#parameters)) | -| `conditions` | named conditions, used by name in a `where:`, a `when:` or a `holds:` ([conditions](named.md#conditions)) | -| `variables` | what the solver decides ([variables](parameters-variables-constraints.md#variables)) | -| `expressions` | named quantities, used again in the math and readable once the solver runs ([named expressions](named.md)) | -| `objective` | what the solver minimises or maximises ([objective](parameters-variables-constraints.md#objective)) | -| `constraints` | the rules that the decisions obey ([constraints](parameters-variables-constraints.md#constraints)) | -| `sos` | sets of variables of which few may be non-zero ([special ordered sets, SOS](piecewise.md#sos)) | -| `piecewise` | piecewise-linear curves ([piecewise](piecewise.md)) | -| `assumptions` | what the spec expects of its data ([assumptions](assumptions.md)) | -| `macros` | templates that take arguments, substituted into an expression before anything reads it ([macros](named.md#macros)) | +| Key | What the section declares | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `given` | what this file uses and another file declares ([given](parameters-variables-constraints.md#given)) | +| `dimensions` | the sets of labels that entries are indexed over ([dimensions](dimensions.md)), and the multi-dims, written with `over:` ([multi-dims](relations.md#multi-dims)) | +| `maps` | the label of a dimension that each row of a dimension or a set has ([maps](relations.md#maps)) | +| `parameters` | the data the spec expects ([parameters](parameters-variables-constraints.md#parameters)) | +| `conditions` | named conditions, used by name in a `where:`, a `when:` or a `holds:` ([conditions](named.md#conditions)) | +| `variables` | what the solver decides ([variables](parameters-variables-constraints.md#variables)) | +| `expressions` | named quantities, used again in the math and readable once the solver runs ([named expressions](named.md)) | +| `objective` | what the solver minimises or maximises ([objective](parameters-variables-constraints.md#objective)) | +| `constraints` | the rules that the decisions obey ([constraints](parameters-variables-constraints.md#constraints)) | +| `sos` | sets of variables of which few may be non-zero ([special ordered sets, SOS](piecewise.md#sos)) | +| `piecewise` | piecewise-linear curves ([piecewise](piecewise.md)) | +| `assumptions` | what the spec expects of its data ([assumptions](assumptions.md)) | +| `macros` | templates that take arguments, substituted into an expression before anything reads it ([macros](named.md#macros)) | The order of the sections, and the order of the entries in a section, change nothing. `to_spec` sorts the entries of each section by name, and the entries diff --git a/docs/reference/language/operators.md b/docs/reference/language/operators.md index 4a70329b6..3ea54323a 100644 --- a/docs/reference/language/operators.md +++ b/docs/reference/language/operators.md @@ -26,7 +26,7 @@ A file cannot add an operator, and only | `sum(array)` | `sum` removes every [dimension](dimensions.md) of `array`. The result is a scalar | | `sum(array, over=dim)` | `sum` removes `dim` | | `sum(array, over=[a, …])` | `sum` removes each dimension in the list | -| `sum(array, over=dim, by=subject[c])` | `sum` removes `dim`, and groups the result by the column `c` of the table of `subject`, a [map](relations.md#maps) or a column of a [set](relations.md#sets). The call joins on the other key columns, so `array` and the result are indexed over them | +| `sum(array, over=dim, by=subject[c])` | `sum` removes `dim`, and groups the result by the column `c` of the table of `subject`, a [map](relations.md#maps) or a member of a [multi-dim](relations.md#multi-dims). The call joins on the other key columns, so `array` and the result are indexed over them | | `sum(array, over=[dim, …], by=subject[c, …])` | The call does the same with several dimensions and several columns of one table, joins on them together, and groups by their product | | `at(array, by=subject[m])` | The result is `array` read at the label that the map `m` gives each row of `subject`, once per row. The result gains the key of `subject` | | `shift(array, along=dim, offset=n)` | The result is the value `n` positions earlier along `dim`, and the vacated edge is [absent](absence.md) | @@ -54,7 +54,7 @@ each dimension once. `sum(x, over=[a, b])` is the same as one sum over both sets. With `by=`, the sum adds up each group that a [map](relations.md#maps) or a -column of a [set](relations.md#sets) makes, and the result is +member of a [multi-dim](relations.md#multi-dims) makes, and the result is indexed over that column. The balance at each bus (a nodal balance) is one `sum(by=)` per kind of component: @@ -110,8 +110,8 @@ sum. !!! info "See also" - - [How sets and maps are used](relations.md#how-sets-and-maps-are-used), - for the dimensions that a call through a set or a map joins on, and + - [How multi-dims and maps are used](relations.md#how-multi-dims-and-maps-are-used), + for the dimensions that a call through a multi-dim or a map joins on, and those its result has. ## `at` @@ -428,8 +428,8 @@ The typesetter prints each form from the smallest spec that uses it. - [Absence and where](absence.md) for what each operator does with an absent input. - - [Sets and maps](relations.md#how-sets-and-maps-are-used) for the - set or the map that `by=` and `within=` take. + - [Multi-dims and maps](relations.md#how-multi-dims-and-maps-are-used) for the + multi-dim or the map that `by=` and `within=` take. - [Macros](named.md#macros) for a composition of operators under one name. - [One construct per spec](../../examples/operators.md) for the smallest spec that uses each form. diff --git a/docs/reference/language/parameters-variables-constraints.md b/docs/reference/language/parameters-variables-constraints.md index 5630ba9f2..a440ce97f 100644 --- a/docs/reference/language/parameters-variables-constraints.md +++ b/docs/reference/language/parameters-variables-constraints.md @@ -316,7 +316,7 @@ A `str` parameter takes any label. A dimension with no row. It takes `refused` (the default), `absent`, or a value of the `dtype` of the parameter. On a parameter with a [`where:`](#where), `missing:` applies only where the condition is true. On a parameter over a -[set](#a-set-in-dims), it applies only where the set has a row. +[multi-dim](#a-multi-dim-in-dims), it applies only where the multi-dim has a row. ```yaml dimensions: @@ -405,8 +405,7 @@ dimensions: bus: { dtype: str } role: { dtype: str } snapshot: { dtype: int } -sets: - connection: { dims: [entity, bus], missing: absent } + connection: { over: [entity, bus] } maps: connection: { role: { dim: role } } parameters: @@ -451,8 +450,8 @@ snapshot of the example, and drops every other row. A tool drops a row of the data outside the condition without refusing the data. An [assumption](assumptions.md) on the parameter does not test that row either, because the parameter has no row there. To have a tool refuse a row at -a pair that is not wired, name the set in `dims:` -([a set in `dims`](#a-set-in-dims)). A set name alone in the `where:` would +a pair that is not wired, name the multi-dim in `dims:` +([a multi-dim in `dims`](#a-multi-dim-in-dims)). A multi-dim name alone in the `where:` would state the same rows a second time, so mathspec refuses it. For `where: connection` on `efficiency`, the message is: @@ -462,14 +461,14 @@ Parameter 'efficiency': `where: connection` tests the rows of 'connection', whic A tool evaluates the condition from the data alone, before it reads the parameter. So the condition may contain [dimensions](dimensions.md), -[sets](relations.md) and parameters with neither a `where:` nor a set in +[multi-dims](relations.md) and parameters with neither a `where:` nor a multi-dim in `dims:`. mathspec refuses four conditions at load: - a condition that contains a variable - a condition that contains the parameter itself -- a condition that contains a parameter with a `where:` or a set in its `dims:` +- a condition that contains a parameter with a `where:` or a multi-dim in its `dims:` - a condition over a dimension that the parameter does not have For a `where:` on `efficiency` that contains the variable `p`, the message @@ -485,9 +484,9 @@ The typeset legend lists the condition beside the dimensions, after `only where`. For `efficiency`, the legend entry ends in `only where` $\mathrm{role}(b,\ e) = \text{'}\mathrm{output}\text{'}$. -### A set in `dims` +### A multi-dim in `dims` -A [set](relations.md) named in `dims:` stands for its dimensions, at the +A [multi-dim](relations.md) named in `dims:` stands for its dimensions, at the combinations it lists. The parameter has coordinates only there, and a tool refuses a row of the data anywhere else. A [`where:`](#where) drops that row instead. In this example, `efficiency` and `p_nom_max` have values only at the @@ -498,8 +497,7 @@ dimensions: entity: { dtype: str } bus: { dtype: str } snapshot: { dtype: int } -sets: - connection: { dims: [entity, bus], missing: absent } + connection: { over: [entity, bus] } parameters: efficiency: { dims: [connection, snapshot], missing: 1 } p_nom_max: { dims: [connection], missing: .inf } @@ -528,12 +526,12 @@ steps: So `missing: 1` fills only the wired pairs, and a data row for an entity at a bus it is not wired to is refused. -An entry over two sets that share a dimension, such as +An entry over two multi-dims that share a dimension, such as `dims: [connection, active]`, has coordinates only where both list them. A -sum over a set, `sum(x, over=connection)`, sums over its dimensions, as +sum over a multi-dim, `sum(x, over=connection)`, sums over its dimensions, as `over=[entity, bus]` does. -A set in `dims:` gives its rows, and its [maps](relations.md#maps) play no +A multi-dim in `dims:` gives its rows, and its [maps](relations.md#maps) play no part. They stay readable as `connection[bus]` in a `where:` and in a call. In this example, `connection` has a row per port of a unit, and its maps `bus` and `role` give each port a bus and a role: @@ -545,8 +543,7 @@ dimensions: bus: { dtype: str } role: { dtype: str } snapshot: { dtype: int } -sets: - connection: { dims: [unit, port], missing: absent } + connection: { over: [unit, port] } maps: connection: { bus: { dim: bus }, role: { dim: role } } parameters: @@ -567,40 +564,37 @@ refuses a data row for a port that `connection` does not list. The `where:` then drops the input ports. `efficiency` is not over `bus` or `role`, the dimensions that the maps refer to. -mathspec refuses three sets in `dims:` at load: +mathspec refuses two multi-dims in `dims:` at load: -- a set named twice -- a set under `missing: refused`, which has a row at every combination and so - leaves nothing out -- a set with two columns over one dimension, because an entry is over each - dimension once +- a multi-dim named twice +- a multi-dim with two members over one dimension, because an entry is over + each dimension once -For `p_nom_max: { dims: [connection] }` with `connection` under -`missing: refused`, the message is: +For `p_nom_max: { dims: [connection, connection] }`, the message is: ```text -Parameter 'p_nom_max': dims: names 'connection', which under missing: refused has a row at every combination, so it leaves nothing out. Write missing: absent on 'connection', or name its dimensions in its place. +Parameter 'p_nom_max': dims: names 'connection' twice. Delete one. ``` -A `where:` on a parameter over a set narrows the coordinates inside the set, -and the tool drops the rows outside it, as in step 2. Inside the set, its bare +A `where:` on a parameter over a multi-dim narrows the coordinates inside the multi-dim, +and the tool drops the rows outside it, as in step 2. Inside the multi-dim, its bare name is true. So mathspec refuses a `where:` that reads it, and names what the `where:` is left with. `where: connection` on `efficiency` in the first example is always true: ```text -Parameter 'efficiency': dims: names 'connection', so 'efficiency' has coordinates only where it has a row, and `where: connection` narrows nothing. Delete the `where:`, or the set from it. +Parameter 'efficiency': dims: names 'connection', so 'efficiency' has coordinates only where it has a row, and `where: connection` narrows nothing. Delete the `where:`, or the multi-dim from it. ``` A `where:` that is always false, such as `where: NOT connection`, leaves the parameter no coordinate, and mathspec suggests a `where:` that holds somewhere -inside the set. For `where: "connection AND active"`, where `active` is a +inside the multi-dim. For `where: "connection AND active"`, where `active` is a `bool` parameter, the message ends in -`` `where: connection AND active` repeats the set. Delete the set from it. `` +`` `where: connection AND active` repeats the multi-dim. Delete the multi-dim from it. `` A test of a map, such as `connection[role] == 'output'`, is not the bare name -of the set, so it loads. +of the multi-dim, so it loads. -The typeset legend prints the set as the domain of the parameter, after +The typeset legend prints the multi-dim as the domain of the parameter, after `over`. For `efficiency` in the first example, the legend entry starts with `efficiency` over $(b, e) \in \mathrm{connection},\ t \in \mathcal{T}$. In the second example, it starts with `efficiency` over @@ -658,10 +652,10 @@ bounds pin a variable to one value ([fix a quantity](../../howto/pin-a-variable.md)), and a pinned variable is still a variable. -### A set in a variable's `dims` +### A multi-dim in a variable's `dims` -A variable names a [set](#a-set-in-dims) in `dims:` as a parameter does: it -exists only where the set has a row. In this example, `flow` exists at the +A variable names a [multi-dim](#a-multi-dim-in-dims) in `dims:` as a parameter does: it +exists only where the multi-dim has a row. In this example, `flow` exists at the pairs that `connection` lists, at every snapshot: ```yaml @@ -669,8 +663,7 @@ dimensions: generator: { dtype: str } bus: { dtype: str } snapshot: { dtype: int } -sets: - connection: { dims: [generator, bus], missing: absent } + connection: { over: [generator, bus] } parameters: load: { dims: [bus, snapshot] } variables: @@ -685,13 +678,13 @@ constraints: $$\mathit{flow}_{b,g,t} \ge 0 \qquad \forall\, \left( b,\ g \right) \in \mathrm{connection},\ t \in \mathcal{T}$$ -A variable has no data, so nothing is refused. mathspec refuses the same sets +A variable has no data, so nothing is refused. mathspec refuses the same multi-dims at load as on a parameter, and a `where:` on the variable narrows inside the -set. A `missing:` number stands for the variable outside its set, as outside a -`where:`. A constraint names a set in `dims:` the same way, and has rows only -where the set has one. +multi-dim. A `missing:` number stands for the variable outside its multi-dim, as outside a +`where:`. A constraint names a multi-dim in `dims:` the same way, and has rows only +where the multi-dim has one. -mathspec refuses a `where:` that tests the rows of a set, as on a parameter, +mathspec refuses a `where:` that tests the rows of a multi-dim, as on a parameter, because `dims:` states them. For `where: connection` on `flow`, the message is: diff --git a/docs/reference/language/relations.md b/docs/reference/language/relations.md index 06c0fcaa9..3fcdca2f0 100644 --- a/docs/reference/language/relations.md +++ b/docs/reference/language/relations.md @@ -3,84 +3,96 @@ SPDX-FileCopyrightText: mathspec contributors SPDX-License-Identifier: CC-BY-4.0 --> -# Sets and maps +# Multi-dims and maps -A **set** is a set of combinations of labels of -[dimensions](dimensions.md), such as the buses that each generator may connect -to. A **map** gives each row of a dimension or a set one label of a -dimension, such as the bus of each generator. The file declares both, and the -rows come with the data. This page lists the keys of `sets:` and `maps:`, -the data they expect, and how each operator uses them. +A **multi-dim** is a dimension whose labels are combinations of labels of other +[dimensions](dimensions.md), its **members**, such as the buses that each +generator may connect to. A **map** gives each row of a dimension or a +multi-dim one label of a dimension, such as the bus of each generator. The file +declares both, and the rows come with the data. This page lists the keys of a +multi-dim and of `maps:`, the data they expect, and how each operator uses +them. -## `sets` +## Multi-dims -Each key under `sets:` declares one set. +A multi-dim is an entry under `dimensions:` that has `over:`. ```yaml dimensions: bus: { dtype: str } generator: { dtype: str } snapshot: { dtype: int } -sets: - connection: { dims: [generator, bus], missing: absent } # the pairs that are wired - neighbour: { dims: { from: snapshot, to: snapshot }, missing: absent } + connection: { over: [generator, bus] } # the pairs that are wired + neighbour: { over: { from: snapshot, to: snapshot } } ``` Each row of `connection` is one pair of a generator and a bus, and a pair with -no row is not in the set. +no row is not a label of `connection`. -A set is what an entry is indexed over and what a sum runs over, as in +A multi-dim is what an entry is indexed over and what a sum runs over, as in $`\sum_{(g, b) \in \mathrm{connection}}`$. So its name may stand in the `dims:` of a parameter, a variable or a constraint, and in `over=`, for its -dimensions at the combinations it lists -([a set in `dims`](parameters-variables-constraints.md#a-set-in-dims)). - -| Key | What it sets | Default | -| ------------- | ------------------------------------------------------------------------ | --------- | -| `dims` | required. The columns of the set, at least two | | -| `missing` | [`refused` or `absent`](#missing): whether every combination needs a row | `refused` | -| `description` | free text | `null` | - -A column is named after its dimension, or after no dimension. Where two -columns share a dimension, each column has its own name in the mapping form, -as in `{from: snapshot, to: snapshot}`. A column named after a dimension that -it is not over is refused, because mathspec reads the name of a column as its -dimension. With `zp: { dims: { period: zone, generator: generator } }`, +members at the combinations it lists +([a multi-dim in `dims`](parameters-variables-constraints.md#a-multi-dim-in-dims)). +Two multi-dims in one `dims:` join on the members they share, at the rows that +both list, so TIMES writes `act: { dims: [prc_ts, rtp_vintyr] }`. + +| Key | What it sets | Default | +| ------------- | ---------------------------------------------------- | ------- | +| `over` | required. The members of the multi-dim, at least two | | +| `description` | free text | `null` | + +A multi-dim has no `missing:`. Its labels are the rows that the data lists, as +the labels of a dimension are, so a combination with no row is not a label. + +A member is named after its dimension, or after no dimension. Where two +members share a dimension, each member has its own name in the mapping form, +as in `{from: snapshot, to: snapshot}`. A member named after a dimension that +it is not over is refused, because mathspec reads the name of a member as its +dimension. With `zp: { over: { period: zone, generator: generator } }`, mathspec prints: ```text -Set 'zp' names column 'period' after dimension 'period', but the column is over 'zone'. Rename the column after what it holds. +Multi-dim 'zp' names member 'period' after dimension 'period', but the member is over 'zone'. Rename the member after what it holds. ``` -A set with one column is refused, because one column holds labels of one +A multi-dim with one member is refused, because one member holds labels of one dimension, and a map or a condition states that. With -`r: { dims: [generator] }`, mathspec prints: +`r: { over: [generator] }`, mathspec prints: ```text -Set 'r' has one column under 'dims:', and a set needs at least two. To give each generator a label, declare a map under maps: generator:. To mark some of them, declare a bool parameter or a condition. +Multi-dim 'r' has one member under 'over:', and a multi-dim needs at least two. To give each generator a label, declare a map under maps: generator:. To mark some of them, declare a bool parameter or a condition. ``` -A set is not one dimension, so its name does not go in `along=`, which steps -along one. A set with two columns over one dimension, such as `neighbour`, -does not go in `dims:` either, because an entry is over each dimension once. -With `flow: { dims: [neighbour] }`, mathspec prints: +A list that names one dimension twice is refused, and the rewrite is an +[alias](dimensions.md#alias). With `link: { over: [bus, bus] }`, mathspec +prints: ```text -Variable 'flow' names the set 'neighbour', which has two columns over 'snapshot', and an entry is over each dimension once. Declare an alias of 'snapshot' under dimensions:, and one column of the set over it. +Multi-dim 'link' names dimension 'bus' twice under 'over:'. Declare an alias of 'bus' under dimensions:, such as bus_to: {alias: bus}, and list it in place of the second. +``` + +A multi-dim is not one dimension, so its name does not go in `along=`, which +steps along one. A multi-dim with two members over one dimension, such as +`neighbour`, does not go in `dims:` either, because an entry is over each +dimension once. With `flow: { dims: [neighbour] }`, mathspec prints: + +```text +Variable 'flow' names the multi-dim 'neighbour', which has two members over 'snapshot', and an entry is over each dimension once. Declare an alias of 'snapshot' under dimensions:, and list it under the over: of 'neighbour' in place of one of the two. ``` With `from_snapshot: { alias: snapshot }` and -`neighbour: { dims: [from_snapshot, snapshot] }`, the set names each +`neighbour: { over: [from_snapshot, snapshot] }`, the multi-dim names each dimension once, and an entry may be over it ([`alias`](dimensions.md#alias)). -The order of the columns under `dims:` changes nothing. The typesetter sorts -the columns by name, so `connection` above prints as +The order of the members under `over:` changes nothing. The typesetter sorts +the members by name, so `connection` above prints as $`\mathrm{connection} \subseteq \mathcal{B} \times \mathcal{G}`$. ## `maps` -Each key under `maps:` names a **subject**, which is a dimension or a set. +Each key under `maps:` names a **subject**, which is a dimension or a multi-dim. Each key under the subject declares one map, which gives every row of the subject one label of the dimension that `dim:` names. @@ -111,8 +123,8 @@ typesetter prints a map as a function from its subject into that dimension, so `generator[bus]` prints as $`\mathrm{bus}: \mathcal{G} \to \mathcal{B}`$. mathspec refuses a map at load in four cases: -- The subject is neither a dimension nor a set. -- `dim:` names a set, or names no dimension. +- The subject is neither a dimension nor a multi-dim. +- `dim:` names a multi-dim, or names no dimension. - The map is named after a dimension that it does not refer to. - The map has the name of a column of its subject. @@ -122,37 +134,36 @@ With `generator: { period: { dim: zone } }`, mathspec prints: maps: 'generator': map 'period' is named after dimension 'period', but refers to 'zone'. Rename the map after what it holds. ``` -A set with two columns over one dimension takes no maps, because a map +A multi-dim with two members over one dimension takes no maps, because a map gives one label per combination of the dimensions of its subject. ### Cardinalities How many labels a row may have (the cardinality) decides whether a fact is a -map or a set: - -| What the file states | Written | Cardinality | -| -------------------------------------------- | --------------------------------------------------------------------------------------------------- | --------------------------- | -| each generator has one bus | `maps: { generator: { bus: { dim: bus } } }` | many-to-one | -| a generator may connect to several buses | `sets: { connection: { dims: [generator, bus], missing: absent } }` | many-to-many | -| a generator has one zone in each period | `sets: { zone_of: { dims: [generator, period] } }` and `maps: { zone_of: { zone: { dim: zone } } }` | many-to-one, from a pair | -| a snapshot has a month, a week and a weekday | `maps: { snapshot: { month: { dim: month }, week: { dim: week }, weekday: { dim: weekday } } }` | three maps of one subject | -| a line has two ends, both buses | `maps: { line: { bus0: { dim: bus }, bus1: { dim: bus } } }` | two maps into one dimension | -| a snapshot has a representative snapshot | `maps: { snapshot: { rep: { dim: snapshot } } }` | many-to-one, onto itself | -| a snapshot has neighbours | `sets: { neighbour: { dims: { from: snapshot, to: snapshot }, missing: absent } }` | many-to-many, onto itself | +map or a multi-dim: + +| What the file states | Written | Cardinality | +| -------------------------------------------- | --------------------------------------------------------------------------------------------------------- | --------------------------- | +| each generator has one bus | `maps: { generator: { bus: { dim: bus } } }` | many-to-one | +| a generator may connect to several buses | `dimensions: { connection: { over: [generator, bus] } }` | many-to-many | +| a generator has one zone in each period | `dimensions: { zone_of: { over: [generator, period] } }` and `maps: { zone_of: { zone: { dim: zone } } }` | many-to-one, from a pair | +| a snapshot has a month, a week and a weekday | `maps: { snapshot: { month: { dim: month }, week: { dim: week }, weekday: { dim: weekday } } }` | three maps of one subject | +| a line has two ends, both buses | `maps: { line: { bus0: { dim: bus }, bus1: { dim: bus } } }` | two maps into one dimension | +| a snapshot has a representative snapshot | `maps: { snapshot: { rep: { dim: snapshot } } }` | many-to-one, onto itself | +| a snapshot has neighbours | `dimensions: { neighbour: { over: { from: snapshot, to: snapshot } } }` | many-to-many, onto itself | ## `missing` -`missing:` sets whether the data may leave something out. On a set, it -sets whether every combination of the dimensions under `dims:` needs a row. On -a map, it sets whether every row of the subject needs a label. +`missing:` on a map sets whether every row of the subject needs a label. A +multi-dim has no `missing:`: a combination with no row is not one of its +labels. ```yaml dimensions: generator: { dtype: str } bus: { dtype: str } line: { dtype: str } -sets: - connection: { dims: [generator, bus], missing: absent } # most pairs are not wired + connection: { over: [generator, bus] } # most pairs are not wired maps: generator: bus: { dim: bus } # refused: every generator is on a bus @@ -167,10 +178,10 @@ maps: At `l3`, the data gives `line[bus1]` no label, so `line[bus1]` is absent there. -| `missing:` | Result | -| --------------------- | ------------------------------------------------------------------------------------------------------------------- | -| `refused`, by default | a tool refuses the data when it is attached, and names the combination or the row | -| `absent` | a combination with no row is not in the set, and a row with no label maps to nothing, which is [absent](absence.md) | +| `missing:` | Result | +| --------------------- | --------------------------------------------------------------------------------- | +| `refused`, by default | a tool refuses the data when it is attached, and names the combination or the row | +| `absent` | a row with no label maps to nothing, which is [absent](absence.md) | No number can replace a missing label, so at load mathspec refuses a value such as `missing: 0`: @@ -179,19 +190,15 @@ such as `missing: 0`: maps.generator.bus.missing: missing: 0 on a map, which takes refused or absent. A row with nothing there is refused, or is not in the data. ``` -Under the default `refused`, a set needs a row for every combination, so -`connection` would need every generator wired to every bus. Most sets are -sparse, so they declare `missing: absent`. - The typeset legend lists `` `connection` is `absent` where the data has no row `` and `` `line[bus1]` is `absent` where the data gives no label `` beside each dimension they are over, and nothing for `refused`. ## The data contract -With data attached, each set and each subject with maps has one table, -named after it. The table of a set has one column per column under -`dims:`. The table of a subject also has one column per map, named after the +With data attached, each multi-dim and each subject with maps has one table, +named after it. The table of a multi-dim has one column per member under +`over:`. The table of a subject also has one column per map, named after the map. So the data for the maps of `line` is a table `line` with the columns `line`, `bus0` and `bus1`. A piece of software that reads a spec ([a tool](../glossary.md#tool)) requires five things of a table: @@ -200,46 +207,45 @@ map. So the data for the maps of `line` is a table `line` with the columns generator with two rows, on two buses, is refused. - Every label in the column of a map is a label of the dimension it refers to, and a label that matches none is refused. -- The table has a row for every combination of its dimensions, unless the - set declares `missing: absent`. The table of a dimension has a row for - every label of the dimension. +- The table of a dimension has a row for every label of the dimension. The + table of a multi-dim has the rows it lists, and no others. - A null is refused, except in the column of a map that declares `missing: absent`. - The order of the rows has no meaning, because the order comes from the [dimension](dimensions.md#ordered). -A set and a map have no `where:`, so their rows and labels come from the +A multi-dim and a map have no `where:`, so their rows and labels come from the data alone. A tool therefore knows them before it evaluates any condition, such as the [`where:` of a parameter](parameters-variables-constraints.md#where). -The rows of a set can also be the coordinates of a parameter or a +The rows of a multi-dim are also the coordinates of a parameter or a variable. An entry with `connection` in its `dims:` has coordinates only at the pairs that `connection` lists, and a tool refuses a row of the data of a parameter at any other pair -([a set in `dims`](parameters-variables-constraints.md#a-set-in-dims)). A map -is no set to be over, because it adds no row to its subject. +([a multi-dim in `dims`](parameters-variables-constraints.md#a-multi-dim-in-dims)). A map +is no multi-dim to be over, because it adds no row to its subject. -An [assumption](assumptions.md) can require rows of a set or labels of a +An [assumption](assumptions.md) can require rows of a multi-dim or labels of a map. For `line[bus1]` [above](#missing), `{holds: "line[bus1]", where: "closed"}` requires an end for every closed line, where `closed` is a `bool` parameter over `line`. -## How sets and maps are used +## How multi-dims and maps are used -A set fixes no direction, so the same table can map generators to buses +A multi-dim fixes no direction, so the same table can map generators to buses or buses to generators. A call **joins** the expression it acts on (its operand) to the table on their shared columns, and then **groups** the rows it has joined. In a call, `s[c]` names the subject `s` and its column `c`, and `s[c, …]` names several columns of one table. A column is a map of `s`, or one -of the columns of a set under `dims:`. The subject appears once, so one +of the members of a multi-dim under `over:`. The subject appears once, so one call uses one table. -| Kind | What it does | Written as | -| ----------------- | ------------------------------------------------------------------------- | -------------------------------------------------- | -| join and group by | rows of the operand match rows of the table, and each group is added up | `sum(x, over=d, by=s[c])` | -| join | each row of the operand matches one row of the table, and takes its label | `at(x, by=s[m])` | -| partition | the dimensions stay, and the rows are grouped | `shift`, `sum_back`, `position` with `within=s[m]` | -| test | a coordinate stays where the set has a row, or where the map has a label | `r` or `s[m]` in a condition | +| Kind | What it does | Written as | +| ----------------- | ------------------------------------------------------------------------------ | -------------------------------------------------- | +| join and group by | rows of the operand match rows of the table, and each group is added up | `sum(x, over=d, by=s[c])` | +| join | each row of the operand matches one row of the table, and takes its label | `at(x, by=s[m])` | +| partition | the dimensions stay, and the rows are grouped | `shift`, `sum_back`, `position` with `within=s[m]` | +| test | a coordinate stays where the multi-dim has a row, or where the map has a label | `r` or `s[m]` in a condition | Four rules apply to every call: @@ -280,8 +286,8 @@ Each column of the table is either **joined on** or not, and either | a key column the call does not list | yes | yes | kept | | a map the call does not list | no | no | not used | -With the set `zone_of` over `[generator, period]` and its map `zone`, the -set `connection` over `[generator, bus]`, and a variable `p` over +With the multi-dim `zone_of` over `[generator, period]` and its map `zone`, the +multi-dim `connection` over `[generator, bus]`, and a variable `p` over `[generator, period]`, three sums have three results: | Call | Joins on | Groups by | Result | @@ -344,7 +350,7 @@ it at `south`. Three rules apply to a lookup: - A lookup reads only maps. A key column in `by=` is refused, and so is a - set with no map, where one combination can have several rows. + multi-dim with no map, where one combination can have several rows. - The result gains every key column that the operand is not already over. A key column over a dimension that the operand is over is joined on too, unless a map in `by=` already refers to that dimension. With `price` over @@ -379,7 +385,7 @@ The first snapshot of each season takes the value of the last snapshot of the same season, and no value crosses from winter to summer. `within=` may list two maps into one dimension, but not a key column. A -set with no map makes no groups, so mathspec refuses it in `within=`: +multi-dim with no map makes no groups, so mathspec refuses it in `within=`: ```text Constraint 'd': shift(within=sp[period]): 'sp' has no map, so it makes no groups. Declare the group as a map of the dimension shift steps along, under maps: snapshot:. @@ -387,7 +393,7 @@ Constraint 'd': shift(within=sp[period]): 'sp' has no map, so it makes no groups ### Tests -A condition tests a set or a map at the coordinates of its own entry. It +A condition tests a multi-dim or a map at the coordinates of its own entry. It compares a map, written `s[m]`, against a label, compares two maps, or tests that a row or a label exists. [Conditions](expressions.md#conditions) lists when two maps may be compared. A comparison reads one map, so @@ -401,18 +407,17 @@ Expected end of text, found '[' (at char 8), (line:1, col:9) ``` A map alone in a condition, such as `line[bus1]`, is true where the map gives -the row a label. A set name alone is true where the set has a row. -On a parameter, a variable or a constraint, the set in `dims:` states those -rows, so mathspec refuses the name in its `where:`. Over the set, a variable is -absent at every pair that the set does not list: +the row a label. A multi-dim name alone is true where the multi-dim has a row. +On a parameter, a variable or a constraint, the multi-dim in `dims:` states those +rows, so mathspec refuses the name in its `where:`. Over the multi-dim, a variable is +absent at every pair that the multi-dim does not list: ```yaml title="flow.yaml" dimensions: generator: { dtype: str } bus: { dtype: str } snapshot: { dtype: int } -sets: - connection: { dims: [generator, bus], missing: absent } # the pairs that are wired + connection: { over: [generator, bus] } # the pairs that are wired parameters: load: { dims: [bus, snapshot] } variables: @@ -455,5 +460,5 @@ $`\mathrm{bus1}(l) \text{ is defined}`$. !!! info "See also" - - [sets as linear maps](../../about/relations-as-linear-maps.md) for the + - [multi-dims as linear maps](../../about/relations-as-linear-maps.md) for the same calls written as matrices diff --git a/docs/reference/notation.md b/docs/reference/notation.md index d9ba357c4..616a89d1b 100644 --- a/docs/reference/notation.md +++ b/docs/reference/notation.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: CC-BY-4.0 This page lists each construct of the language beside the math that [the typesetter](typeset.md) prints for it. Each section has the math of one -kind of construct, and the legend lists the symbol of each dimension, set and parameter. +kind of construct, and the legend lists the symbol of each dimension, multi-dim and parameter. Each construct is printed as its YAML and then its equation. Most of the YAML comes from one test spec, @@ -23,7 +23,7 @@ typesetter changes nothing else. ### Legend -A dimension, a relation, a map and a parameter declare no equation; what they print is the legend every spec opens with. +A dimension, a multi-dim, a map and a parameter declare no equation; what they print is the legend every spec opens with. ```yaml dimensions: @@ -35,11 +35,9 @@ dimensions: season: { dtype: str } technology: { dtype: str } bp: { dtype: int, ordered: true } # the breakpoints every curve below runs through - -sets: - gen_zone: { dims: [generator, snapshot] } # a relation over two dimensions, with a map: a call sums one away and joins on the other - connection: { dims: [generator, bus], missing: absent } # a relation with no map: many-to-many, read only by sum with both ends named - bus_zone: { dims: [bus, zone], missing: absent } # the pairs that exist + gen_zone: { over: [generator, snapshot] } # a multi-dim over two dimensions, with a map: a call sums one away and joins on the other + connection: { over: [generator, bus] } # a multi-dim with no map: many-to-many, read only by sum with both ends named + bus_zone: { over: [bus, zone] } # the pairs that exist maps: generator: { bus: { dim: bus }, technology: { dim: technology } } # two maps of one subject, read to both at once @@ -59,7 +57,7 @@ parameters: is_flexible: { dims: [generator], dtype: bool, missing: false } zone_cap: { dims: [zone] } tech_cap: { dims: [bus, technology], missing: absent } - link_eff: { dims: [connection], missing: 1 } # over the pairs a relation lists; a `where:` may not test them + link_eff: { dims: [connection], missing: 1 } # over the pairs a multi-dim lists; a `where:` may not test them zone_share: { dims: [bus_zone], missing: 1 } # over the pairs that exist: data refused outside them, with no `where:` of its own susceptance: { dims: [from_bus, bus], missing: 0 } # over one set twice, through its alias area_weight: { dims: [bus], where: "bus[area]" } # a map's label as a condition: the buses that have an area @@ -81,9 +79,9 @@ parameters: | $`\mathcal{B}`$ | index $`b`$ — `bp` | | $`\mathcal{U}`$ | index $`u`$ — `bus` with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}`$, `bus_zone` is `absent` where the data has no row, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | | $`\mathcal{U}`$ | index $`u'`$ — `from_bus`, an alias of `bus` | -| $`\mathcal{G}`$ | index $`g`$ — `generator` with $`\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}`$, `connection` is `absent` where the data has no row | +| $`\mathcal{G}`$ | index $`g`$ — `generator` with $`\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}`$, `connection` is `absent` where the data has no row, `gen_zone` is `absent` where the data has no row | | $`\mathcal{S}`$ | index $`s`$ — `season` with $`\mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | -| $`\mathcal{T}`$ | index $`t`$ — `snapshot` (`int` coordinates) with $`\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | +| $`\mathcal{T}`$ | index $`t`$ — `snapshot` (`int` coordinates) with $`\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}`$, `gen_zone` is `absent` where the data has no row | | $`\mathcal{E}`$ | index $`e`$ — `technology` with $`\mathrm{technology}: \mathcal{G} \to \mathcal{E}`$ | | $`\mathcal{Z}`$ | index $`z`$ — `zone` (labels `north`, `south`) with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}`$, `bus_zone` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | diff --git a/docs/reference/reading.md b/docs/reference/reading.md index 280250a63..fa1e94ac0 100644 --- a/docs/reference/reading.md +++ b/docs/reference/reading.md @@ -42,7 +42,7 @@ the file changes nothing: - the entries of each section, and of each kind under `given:` - each `dims:` list -- the columns of each set, and the maps of each subject +- the members of each multi-dim, and the maps of each subject - the cases under `cases:` - the `kwargs:` of a macro - the labels under `one_of:` @@ -164,10 +164,11 @@ the colon is the description that the file sets. `program.parameters[name].missing` is what the file sets under [`missing:`](language/absence.md#missing) for a missing row of that parameter. It is `'refused'`, `'absent'`, or the value that a tool uses for the row. -`program.sets` has one table per set and per subject with +`program.relations` has one table per multi-dim and per subject with [maps](language/relations.md#the-data-contract), named after it, and each map -is one of its `values`. `program.sets[name].missing` is `'refused'` or -`'absent'`, and `partial` lists the maps under `missing: absent`. The program of +is one of its `values`. `program.relations[name].missing` is `'absent'` for a +multi-dim and `'refused'` for a dimension, and `partial` lists the maps under +`missing: absent`. The program of `spec.expand(...)` has the same `missing` values. A [given](language/parameters-variables-constraints.md#given) parameter is a @@ -186,26 +187,26 @@ the parameter has no row. `program.parameters[name].where` is a [`Mask`](#nodes-and-conditions) for the coordinates of the parameter, or `None` when the parameter has every -coordinate of its `dims`. It contains each set named in the `dims:` of the +coordinate of its `dims`. It contains each multi-dim named in the `dims:` of the parameter and its `where:`. With data attached, an engine applies the condition before `missing:`, as [`where` on a parameter](language/parameters-variables-constraints.md#where) describes. The condition contains no variable and no parameter with a `where:` -or a set in its `dims:`. So an engine can evaluate every such condition before +or a multi-dim in its `dims:`. So an engine can evaluate every such condition before it reads any parameter that has one. -`program.parameters[name].in_` lists the sets the file named in the `dims:` +`program.parameters[name].in_` lists the multi-dims the file named in the `dims:` of the parameter, sorted by name. `program.parameters[name].dims` holds their dimensions in their place. With data attached, an engine refuses a row of the parameter at a combination that one of them has no row for, and reads no map. It does this before it applies the `where`, and so before `missing:`, as -[a set in `dims`](language/parameters-variables-constraints.md#a-set-in-dims) describes. An engine +[a multi-dim in `dims`](language/parameters-variables-constraints.md#a-multi-dim-in-dims) describes. An engine drops a row outside the rest of the `where`, and refuses a row outside the -sets. +multi-dims. `program.variables[name].where` is the same kind of `Mask` for the coordinates -of the variable, with the sets of its `dims:` in it, and -`program.variables[name].in_` lists those sets. A variable has no data, so +of the variable, with the multi-dims of its `dims:` in it, and +`program.variables[name].in_` lists those multi-dims. A variable has no data, so an engine builds it where the `where` is true and needs nothing more from `in_`. @@ -239,7 +240,7 @@ An `ExpressionReference` is the node for a use of an `expressions:` entry. Its Every expression node has `dims`, the dimensions that it is indexed over, sorted by name. A sum has the dimensions of its operand, less the ones it sums -over. A `Join` under a sum through a set has the dimensions of that sum. +over. A `Join` under a sum through a multi-dim has the dimensions of that sum. A `Join` also opens an index per dropped column (an `Axis`). An axis is not a dimension and is not in `dims`. An `ExpressionReference` has the dimensions of its body. @@ -281,7 +282,7 @@ both are true at the same coordinates. A comparison of a map against a label is a `RelationComparison`, and a comparison of two maps is a `RelationPairComparison`. The page on [conditions](language/expressions.md#conditions) defines both, and both are false for every operator where the map gives no label. A -set name alone is a `RelationDefined`, and a map alone, `subject[map]`, is a +multi-dim name alone is a `RelationDefined`, and a map alone, `subject[map]`, is a `RelationDefined` with `column` set to the map. Three conditions contain another condition rather than an entry. A @@ -461,7 +462,7 @@ Writing the canonical form out again gives the same text. The canonical form has these rules: - The sections come in one fixed order: `version`, `description`, `given`, - `dimensions`, `sets`, `maps`, `parameters`, `conditions`, `variables`, + `dimensions`, `maps`, `parameters`, `conditions`, `variables`, `expressions`, `objective`, `constraints`, `sos`, `piecewise`, `assumptions`, `macros`. The keys of an entry also come in one order. - Entries are sorted by name within each section, and so is every other list diff --git a/examples/connection.yaml b/examples/connection.yaml index 092b37746..f01766534 100644 --- a/examples/connection.yaml +++ b/examples/connection.yaml @@ -10,12 +10,9 @@ dimensions: unit: { description: "converters, such as an electrolyser or a heat pump" } port: { dtype: int, description: "an attachment of a unit, numbered as PyPSA's bus0, bus1, ..." } role: { description: input or output } - -sets: connection: description: the bus each port of a unit attaches to, and whether the port draws or delivers - dims: [unit, port] - missing: absent + over: [unit, port] maps: connection: { bus: { dim: bus }, role: { dim: role } } diff --git a/examples/operators/sum_by_column_lists.yaml b/examples/operators/sum_by_column_lists.yaml index 907aeaf2d..1605d5f1e 100644 --- a/examples/operators/sum_by_column_lists.yaml +++ b/examples/operators/sum_by_column_lists.yaml @@ -11,9 +11,7 @@ dimensions: period: { dtype: int } bus: { dtype: str } technology: { dtype: str } - -sets: - slot_of: { dims: [generator, period] } + slot_of: { over: [generator, period] } maps: slot_of: { bus: { dim: bus }, technology: { dim: technology } } diff --git a/examples/operators/sum_by_columns.yaml b/examples/operators/sum_by_columns.yaml index 2f83d345b..35bd0d081 100644 --- a/examples/operators/sum_by_columns.yaml +++ b/examples/operators/sum_by_columns.yaml @@ -12,9 +12,7 @@ dimensions: generator: { dtype: str } period: { dtype: int } zone: { dtype: str } - -sets: - zone_of: { dims: [generator, period] } + zone_of: { over: [generator, period] } maps: zone_of: { zone: { dim: zone } } diff --git a/examples/pypsa.yaml b/examples/pypsa.yaml index 6882e6d6c..9401ef6d6 100644 --- a/examples/pypsa.yaml +++ b/examples/pypsa.yaml @@ -88,6 +88,27 @@ dimensions: breakpoints its data has rows for, so curves may differ in length dtype: int ordered: true + Generator_maintenance_cover: + description: >- + the snapshots a maintenance event covers, by the snapshot it starts in, + data prep. In a scenario, they are the start and the snapshots after it, + until their generator weightings reach that scenario's + `maintenance_duration`. A generator that is not maintainable has no rows + over: { scenario: scenario, generator: generator, start: snapshot, covered: snapshot } + Link_maintenance_cover: + description: >- + the snapshots a maintenance event covers, by the snapshot it starts in, + data prep. In a scenario, they are the start and the snapshots after it, + until their generator weightings reach that scenario's + `maintenance_duration`. A link that is not maintainable has no rows + over: { scenario: scenario, link: link, start: snapshot, covered: snapshot } + Process_maintenance_cover: + description: >- + the snapshots a maintenance event covers, by the snapshot it starts in, + data prep. In a scenario, they are the start and the snapshots after it, + until their generator weightings reach that scenario's + `maintenance_duration`. A process that is not maintainable has no rows + over: { scenario: scenario, process: process, start: snapshot, covered: snapshot } parameters: Bus_attached: @@ -1581,32 +1602,6 @@ parameters: dims: [global_constraint, process] missing: absent -sets: - Generator_maintenance_cover: - description: >- - the snapshots a maintenance event covers, by the snapshot it starts in, - data prep. In a scenario, they are the start and the snapshots after it, - until their generator weightings reach that scenario's - `maintenance_duration`. A generator that is not maintainable has no rows - dims: { scenario: scenario, generator: generator, start: snapshot, covered: snapshot } - missing: absent - Link_maintenance_cover: - description: >- - the snapshots a maintenance event covers, by the snapshot it starts in, - data prep. In a scenario, they are the start and the snapshots after it, - until their generator weightings reach that scenario's - `maintenance_duration`. A link that is not maintainable has no rows - dims: { scenario: scenario, link: link, start: snapshot, covered: snapshot } - missing: absent - Process_maintenance_cover: - description: >- - the snapshots a maintenance event covers, by the snapshot it starts in, - data prep. In a scenario, they are the start and the snapshots after it, - until their generator weightings reach that scenario's - `maintenance_duration`. A process that is not maintainable has no rows - dims: { scenario: scenario, process: process, start: snapshot, covered: snapshot } - missing: absent - maps: snapshot: period: diff --git a/examples/pypsa/generator_maintenance.yaml b/examples/pypsa/generator_maintenance.yaml index 919358ed2..cc741651c 100644 --- a/examples/pypsa/generator_maintenance.yaml +++ b/examples/pypsa/generator_maintenance.yaml @@ -27,16 +27,13 @@ dimensions: ordered: true generator: description: generating units, each on one bus - -sets: Generator_maintenance_cover: description: >- the snapshots a maintenance event covers, by the snapshot it starts in, data prep. In a scenario, they are the start and the snapshots after it, until their generator weightings reach that scenario's `maintenance_duration`. A generator that is not maintainable has no rows - dims: { scenario: scenario, generator: generator, start: snapshot, covered: snapshot } - missing: absent + over: { scenario: scenario, generator: generator, start: snapshot, covered: snapshot } parameters: Generator_maintainable: diff --git a/examples/pypsa/link_maintenance.yaml b/examples/pypsa/link_maintenance.yaml index 5dad5d540..1803eae06 100644 --- a/examples/pypsa/link_maintenance.yaml +++ b/examples/pypsa/link_maintenance.yaml @@ -27,16 +27,13 @@ dimensions: ordered: true link: description: controllable connections, each from one bus to the buses it delivers to - -sets: Link_maintenance_cover: description: >- the snapshots a maintenance event covers, by the snapshot it starts in, data prep. In a scenario, they are the start and the snapshots after it, until their generator weightings reach that scenario's `maintenance_duration`. A link that is not maintainable has no rows - dims: { scenario: scenario, link: link, start: snapshot, covered: snapshot } - missing: absent + over: { scenario: scenario, link: link, start: snapshot, covered: snapshot } parameters: Link_maintainable: diff --git a/examples/pypsa/process_maintenance.yaml b/examples/pypsa/process_maintenance.yaml index 6b44c2819..d4a7bbce7 100644 --- a/examples/pypsa/process_maintenance.yaml +++ b/examples/pypsa/process_maintenance.yaml @@ -27,16 +27,13 @@ dimensions: ordered: true process: description: generalised multi-port converters, each with an internal power that every port draws or delivers at its own rate - -sets: Process_maintenance_cover: description: >- the snapshots a maintenance event covers, by the snapshot it starts in, data prep. In a scenario, they are the start and the snapshots after it, until their generator weightings reach that scenario's `maintenance_duration`. A process that is not maintainable has no rows - dims: { scenario: scenario, process: process, start: snapshot, covered: snapshot } - missing: absent + over: { scenario: scenario, process: process, start: snapshot, covered: snapshot } parameters: Process_maintainable: diff --git a/examples/sparse.yaml b/examples/sparse.yaml index 358b60747..f0e206c58 100644 --- a/examples/sparse.yaml +++ b/examples/sparse.yaml @@ -11,16 +11,8 @@ dimensions: port: { description: an output of a unit } bus: { description: a node of the network } period: { dtype: int, ordered: true, description: a step of the horizon } - -sets: - connection: - description: the ports a unit has - dims: [unit, port] - missing: absent - active: - description: the periods a unit runs in - dims: [unit, period] - missing: absent + connection: { over: [unit, port], description: the ports a unit has } + active: { over: [unit, period], description: the periods a unit runs in } maps: connection: diff --git a/examples/times/capacity.yaml b/examples/times/capacity.yaml index 53050eeef..19473b4f8 100644 --- a/examples/times/capacity.yaml +++ b/examples/times/capacity.yaml @@ -12,20 +12,15 @@ dimensions: vintage: { dtype: int, description: the period a capacity was built in } process: { description: a technology } timeslice: { description: "a slice of the year, at any level of the timeslice tree" } - -sets: rtp_vintyr: description: the vintages of a process that exist in a period (RTP_VINTYR) - dims: [region, vintage, period, process] - missing: absent + over: [region, vintage, period, process] prc_ts: description: the timeslices a process operates at (PRC_TS) - dims: [region, process, timeslice] - missing: absent + over: [region, process, timeslice] ts_map: description: each timeslice and the timeslices at or below it (TS_MAP) - dims: { region: region, s: timeslice, ts: timeslice } - missing: absent + over: { region: region, s: timeslice, ts: timeslice } parameters: coef_cpt: { dims: [region, vintage, period, process], description: the share of a vintage that still exists in a period } diff --git a/examples/times/trade.yaml b/examples/times/trade.yaml index 11ad0e180..1771c85d0 100644 --- a/examples/times/trade.yaml +++ b/examples/times/trade.yaml @@ -13,12 +13,9 @@ dimensions: process: { description: a technology that carries trade } from_region: { alias: region, description: the region an export leaves } from_commodity: { alias: commodity, description: the commodity an export leaves as } - -sets: top_ire: description: the processes that carry each link from a region and commodity to another (TOP_IRE) - dims: [from_region, from_commodity, region, commodity, process] - missing: absent + over: [from_region, from_commodity, region, commodity, process] parameters: ire_flo: diff --git a/schema/mathspec.schema.json b/schema/mathspec.schema.json index 079714bed..959b70276 100644 --- a/schema/mathspec.schema.json +++ b/schema/mathspec.schema.json @@ -648,6 +648,49 @@ "title": "MapSpec", "type": "object" }, + "MultiDimSpec": { + "additionalProperties": false, + "description": "A multi-dim: a dimension whose labels are the combinations of its members that exist, one row each.\n\nWritten under ``dimensions:`` with ``over:``, a list of the member\ndimensions, or a mapping of column name to dimension where two columns\nshare one::\n\n dimensions:\n connection: {over: [unit, port]}\n corridor: {over: {from: bus, to: bus}}\n\nIn an entry's ``dims:`` and in ``over=``, a multi-dim stands for its\nmembers, at the rows it lists. Two multi-dims in one ``dims:`` join on the\nmembers they share. Its name also stands in a condition, where it is true\nat the rows it lists, and before a bracket, where it reads one of its maps\n(``maps:``, [`MapSpec`][]). The rows are data, and arrive under the\nmulti-dim's name, one column per member, with a column per map beside\nthem. A combination with no row is not a label of the multi-dim.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "over": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + ], + "title": "Over" + } + }, + "required": [ + "over" + ], + "title": "MultiDimSpec", + "type": "object" + }, "ObjectiveSpec": { "additionalProperties": false, "description": "A declared objective function.", @@ -894,58 +937,6 @@ "title": "PiecewiseSpec", "type": "object" }, - "SetSpec": { - "additionalProperties": false, - "description": "A named relation: the combinations of its dimensions that exist, one row each.\n\n``dims:`` is a list of dimensions, or a mapping of column name to\ndimension where two columns share one. A row is its own identity, so the\nrelation is a subset of the product of its dimensions::\n\n sets:\n connection: {dims: [unit, port], missing: absent}\n corridor: {dims: {from: bus, to: bus}}\n\nA relation is not a dimension. Its name stands under an entry's ``in:``,\nin a condition, where it is true at the rows it lists, and before a\nbracket, where it reads one of its maps (``maps:``, [`MapSpec`][]). The\nrows are data, and arrive under the relation's name, one column per\ncolumn name, with a column per map beside them.", - "properties": { - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Description" - }, - "dims": { - "anyOf": [ - { - "type": "string" - }, - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - } - ], - "title": "Dims" - }, - "missing": { - "default": "refused", - "enum": [ - "refused", - "absent" - ], - "title": "Missing", - "type": "string" - } - }, - "required": [ - "dims" - ], - "title": "SetSpec", - "type": "object" - }, "SosSpec": { "additionalProperties": false, "description": "A special-ordered set over one dimension of one variable.\n\nOne set per coordinate of the variable's ``dims`` minus ``along``; the\nmembers are the variable's *existing* coordinates along ``along``, in that\ndimension's declared order.\n\n``type: 1`` admits at most one nonzero member, ``type: 2`` at most two,\nand those two consecutive. A consumer with the concept takes the set as\none; [`Spec.expand`][] states it as binaries instead, and the rows it\nwrites multiply by the member's own ``bounds``, which is why a member\nneeds both.", @@ -1100,7 +1091,14 @@ }, "dimensions": { "additionalProperties": { - "$ref": "#/$defs/DimensionSpec" + "anyOf": [ + { + "$ref": "#/$defs/DimensionSpec" + }, + { + "$ref": "#/$defs/MultiDimSpec" + } + ] }, "default": {}, "title": "Dimensions", @@ -1164,14 +1162,6 @@ "title": "Piecewise", "type": "object" }, - "sets": { - "additionalProperties": { - "$ref": "#/$defs/SetSpec" - }, - "default": {}, - "title": "Sets", - "type": "object" - }, "sos": { "additionalProperties": { "$ref": "#/$defs/SosSpec" diff --git a/src/mathspec/_composition.py b/src/mathspec/_composition.py index d3397651e..1a23a8205 100644 --- a/src/mathspec/_composition.py +++ b/src/mathspec/_composition.py @@ -108,7 +108,7 @@ from mathspec._yaml import read_spec from mathspec.errors import LanguageError, SchemaError, did_you_mean, schema_error from mathspec.program import variables_of -from mathspec.spec import GivenSpec, MapSpec, Spec +from mathspec.spec import GivenSpec, MapSpec, MultiDimSpec, Spec if TYPE_CHECKING: from collections.abc import Iterable @@ -116,14 +116,13 @@ from mathspec.program import Program from mathspec.spec import DimensionSpec -SHARED_SECTIONS = ('dimensions', 'sets', 'maps') +SHARED_SECTIONS = ('dimensions', 'maps') SETTINGS: dict[str, frozenset[str]] = { 'dimensions': frozenset({'ordered'}), - 'sets': frozenset({'missing'}), 'maps': frozenset({'missing'}), } -"""The fields of a dimension, a relation or a map, other than ``description``, that a patch may change. A patch may +"""The fields of a dimension, a multi-dim or a map, other than ``description``, that a patch may change. A patch may change no other field of one that its base declares. """ @@ -191,9 +190,8 @@ def merge(fragments: Sequence[Source], description: str | None = None) -> Spec: merged: dict[str, object] = {'version': _one_version(read)} if description is not None: merged['description'] = description - for section in ('dimensions', 'sets'): - if agreed := _agreed(read, section, _singular(section), 'give one of them a name of its own'): - merged[section] = agreed + if agreed := _agreed(read, 'dimensions', 'dimension', 'give one of them a name of its own'): + merged['dimensions'] = agreed flat: dict[str, dict[str, object]] = { name: {'maps': _flat_maps(sections.get('maps'))} for name, sections in read.items() } @@ -280,7 +278,7 @@ def _said(read: Mapping[str, dict[str, object]], section: str, key: str) -> obje def _read(owner: type[BaseModel], section: str, entry: object) -> BaseModel: """*entry* as the schema reads it, so a field written at its default and one left out read the same.""" try: - return _entry_class(owner, section).model_validate(entry) + return _entry_class(owner, section, entry).model_validate(entry) except ValidationError as e: raise schema_error(e) from None @@ -292,7 +290,7 @@ def _same(owner: type[BaseModel], section: str, a: object, b: object, *, aside: here, because the spec sorts it as it loads. """ x, y = _read(owner, section, a), _read(owner, section, b) - return all(getattr(x, f) == getattr(y, f) for f in type(x).model_fields if f not in aside) + return type(x) is type(y) and all(getattr(x, f) == getattr(y, f) for f in type(x).model_fields if f not in aside) def _restates( @@ -310,7 +308,7 @@ def _restates( the rewrite. A key the schema does not know stays a schema error, which names the near-miss. """ - known = restated.keys() <= _entry_class(owner, section).model_fields.keys() + known = restated.keys() <= _entry_class(owner, section, restated).model_fields.keys() try: return _same(owner, section, declared, restated, aside=aside) except SchemaError: @@ -774,12 +772,14 @@ def _singular(section: str) -> str: return IRREGULAR.get(section, section[:-1]) -def _entry_class(owner: type[BaseModel], field: str) -> type[BaseModel]: - """The schema's own class for one entry under *field* of *owner*. +def _entry_class(owner: type[BaseModel], field: str, entry: object = None) -> type[BaseModel]: + """The schema's own class for one entry under *field* of *owner*, *entry* being a multi-dim where it has ``over:``. Read off the annotation rather than listed here, so a section added to the schema cannot be laid over by a rule that does not know what it is made of. """ + if owner is Spec and field == 'dimensions' and isinstance(entry, Mapping) and 'over' in entry: + return MultiDimSpec annotation = owner.model_fields[field].annotation inner = [arg for arg in get_args(annotation) if arg is not type(None)] return cast('type[BaseModel]', inner[-1] if inner else annotation) diff --git a/src/mathspec/_condition_parser.py b/src/mathspec/_condition_parser.py index b51e562cb..c8160e4c3 100644 --- a/src/mathspec/_condition_parser.py +++ b/src/mathspec/_condition_parser.py @@ -251,7 +251,7 @@ def _named_rewrite(text: str, loc: int) -> str | None: _DEEP_REWRITE = ( - 'Declare a parameter or set carrying part of the test and name that here, or split the ' + 'Declare a parameter or multi-dim carrying part of the test and name that here, or split the ' 'entry into two, each masked by one half.' ) diff --git a/src/mathspec/_dimensions.py b/src/mathspec/_dimensions.py index b2b561c9f..5d1336bc9 100644 --- a/src/mathspec/_dimensions.py +++ b/src/mathspec/_dimensions.py @@ -171,7 +171,7 @@ def _check_joined(call: str, use: JoinColumns | Partition, inner: frozenset[str] if twice := sorted({d for d in dims if dims.count(d) > 1}): raise DimensionError( f"{context}: {call} joins '{use.name}' on {twice} through more than one column. Join on " - f'different dimensions, or use a set whose columns are over different dimensions.' + f'different dimensions, or use a multi-dim whose members are over different dimensions.' ) diff --git a/src/mathspec/_exclusivity.py b/src/mathspec/_exclusivity.py index 6766fd772..75b0efa2e 100644 --- a/src/mathspec/_exclusivity.py +++ b/src/mathspec/_exclusivity.py @@ -375,7 +375,7 @@ def _subject_of(node: TypedCondition) -> Subject: case TranslatedCondition(): return Subject('expression', 'a condition read at a neighbouring coordinate') case JoinedCondition(): - return Subject('expression', 'a condition read through a set') + return Subject('expression', 'a condition read through a multi-dim') case _: assert_never(node) @@ -598,7 +598,7 @@ def _atom(node: TypedCondition, cell: dict[Subject, Cell], grid: _Grid) -> bool: raise AssertionError(msg) case CountComparison() | TranslatedCondition() | JoinedCondition(): msg = ( - 'a condition read as a count, at a neighbour or through a set is refused as undecidable ' + 'a condition read as a count, at a neighbour or through a multi-dim is refused as undecidable ' 'before any cell is read' ) raise AssertionError(msg) diff --git a/src/mathspec/_expression_resolver.py b/src/mathspec/_expression_resolver.py index c3f6861e4..f61e52c36 100644 --- a/src/mathspec/_expression_resolver.py +++ b/src/mathspec/_expression_resolver.py @@ -208,7 +208,7 @@ def _name(self, node: NameNode) -> Expression | None: return None case 'relation': self.errors.append( - f"{self.context}: '{node.name}' is a set, and a set is not a value. Sum through it " + f"{self.context}: '{node.name}' is a multi-dim, and a multi-dim is not a value. Sum through it " f'as sum(x, over=, by={node.name}[]), test it in a condition, or declare a ' f'parameter to carry numbers.' ) @@ -709,9 +709,9 @@ def not_a_relation(self, name: str, operator: str, key: str) -> str | None: f'under maps: {name}:, or read a subject that has it: {key}=[].' ) kinds = ( - 'a set, a dimension with maps or a formal argument of this macro' + 'a multi-dim, a dimension with maps or a formal argument of this macro' if self.formals - else 'a set or a dimension with maps' + else 'a multi-dim or a dimension with maps' ) return ( f'{context}: {operator}({key}={name}[...]): {name!r} is not {kinds}. ' @@ -741,7 +741,7 @@ def _members_of_sets(names: tuple[str, ...], ns: Namespace) -> tuple[str, ...]: """ out: list[str] = [] for name in names: - relation = None if name in ns.dimensions else ns.spec.sets.get(name) + relation = None if name in ns.dimensions else ns.spec.multi_dims.get(name) dims: list[str] = [d for _, d in relation.pairs] if relation is not None else [] out.extend(dims if dims and len(set(dims)) == len(dims) else [name]) return tuple(out) @@ -750,9 +750,10 @@ def _members_of_sets(names: tuple[str, ...], ns: Namespace) -> tuple[str, ...]: def _undeclared_dim(context: str, operator: str, call: str, name: str, ns: Namespace, formals: frozenset[str]) -> str: if name in ns.relations and name not in ns.dimensions: dims = list(ns.relations[name].dims) - which = 'its dimensions' if call.startswith('over=') else 'one of its dimensions' + which = 'its members' if call.startswith('over=') else 'one of its members' return ( - f"{context}: {operator}({call}) names the set '{name}', and a set is not a dimension. Name {which}, {dims}." + f"{context}: {operator}({call}) names the multi-dim '{name}', which stands for its members only in dims: " + f'and over=. Name {which}, {dims}.' ) return ( f'{context}: {operator}({call}) does not name a declared dimension{_or_a_formal(formals)}. ' diff --git a/src/mathspec/_renaming.py b/src/mathspec/_renaming.py index bfc4e507f..a55b802c5 100644 --- a/src/mathspec/_renaming.py +++ b/src/mathspec/_renaming.py @@ -18,7 +18,7 @@ from mathspec._expression_parser import NAME, REAL from mathspec._validation import to_spec from mathspec.errors import LanguageError, did_you_mean -from mathspec.spec import MacroSpec, Spec +from mathspec.spec import DimensionSpec, MacroSpec, MultiDimSpec, Spec if TYPE_CHECKING: from mathspec._composition import Source @@ -120,8 +120,8 @@ def _checked(names: Mapping[str, str | None], declared: Mapping[str, str], spec: raise LanguageError(msg) changed = {old: new for old, new in names.items() if new is not None and new != old} columns = { - column: f"a column of the set '{relation}'" - for relation, entry in spec.sets.items() + column: f"a member of the multi-dim '{relation}'" + for relation, entry in spec.multi_dims.items() for column, dimension in entry.pairs if column != dimension } | { @@ -176,6 +176,8 @@ def _renamed(cls: type[BaseModel], entry: object, names: Mapping[str, str]) -> o """ if not isinstance(entry, Mapping): return _in_text(entry, names) + if cls is DimensionSpec and 'over' in entry: + cls = MultiDimSpec if cls is MacroSpec: return _macro(entry, names) out: dict[str, object] = {} diff --git a/src/mathspec/_validation.py b/src/mathspec/_validation.py index 92d4d5af7..5bfcd0e09 100644 --- a/src/mathspec/_validation.py +++ b/src/mathspec/_validation.py @@ -123,25 +123,32 @@ def reference_errors(spec: Spec) -> list[str]: def undeclared_dimension(kind: str, name: str, dimension: str, spec: Spec, *, along: bool = False) -> str: """The one wording for an entry naming a dimension the file does not declare, or naming a relation as one.""" - if (relation := spec.sets.get(dimension)) is None: + if (relation := spec.multi_dims.get(dimension)) is None: return f"{kind} '{name}' references undeclared dimension '{dimension}'. Declare it under 'dimensions:'." dims = sorted({d for _, d in relation.pairs}) repeated = sorted({d for _, d in relation.pairs if [c for _, c in relation.pairs].count(d) > 1}) if not along and kind in ('Parameter', 'Variable', 'Constraint') and repeated: return ( - f"{kind} '{name}' names the set '{dimension}', which has two columns over '{repeated[0]}', and an " - f"entry is over each dimension once. Declare an alias of '{repeated[0]}' under dimensions:, and one " - f'column of the set over it.' + f"{kind} '{name}' names the multi-dim '{dimension}', which has two members over '{repeated[0]}', and " + f"an entry is over each dimension once. Declare an alias of '{repeated[0]}' under dimensions:, and " + f"list it under the over: of '{dimension}' in place of one of the two." ) - rewrite = f'Name one of its dimensions, {dims}.' if along else f'Name its dimensions, {dims}.' - return f"{kind} '{name}' names the set '{dimension}', and a set is not a dimension. {rewrite}" + if along: + return ( + f"{kind} '{name}' names the multi-dim '{dimension}', and a step runs along one dimension. Name one of " + f'its members, {dims}.' + ) + return ( + f"{kind} '{name}' names the multi-dim '{dimension}', and the members of a multi-dim are dimensions. " + f'Name its members, {dims}.' + ) def _flat_namespace(spec: Spec) -> list[tuple[str, Iterable[str]]]: """Each kind of entry whose names share the one namespace an expression reads.""" return [ ('dimension', spec.dimensions), - ('set', spec.sets), + ('multi-dim', spec.multi_dims), ('parameter', spec.parameters), ('given parameter', spec.given.parameters), ('variable', spec.variables), @@ -204,29 +211,29 @@ def _frame_dimensions(spec: Spec) -> Iterator[str]: def _relation_targets(spec: Spec) -> Iterator[str]: """A relation has at least two columns, over declared dimensions, each named once.""" - for lname, lk in spec.sets.items(): + for lname, lk in spec.multi_dims.items(): if len(lk.pairs) < 2: dim = lk.pairs[0][1] if lk.pairs else '' yield ( - f"Set '{lname}' has {'one column' if lk.pairs else 'no column'} under 'dims:', and a set " - f'needs at least two. To give each {dim} a label, declare a map under maps: {dim}:. To mark some ' - f'of them, declare a bool parameter or a condition.' + f"Multi-dim '{lname}' has {'one member' if lk.pairs else 'no member'} under 'over:', and a " + f'multi-dim needs at least two. To give each {dim} a label, declare a map under maps: {dim}:. To ' + f'mark some of them, declare a bool parameter or a condition.' ) - if not isinstance(lk.dims, Mapping): + if not isinstance(lk.over, Mapping): yield from ( - f"Set '{lname}' names dimension '{d}' twice under 'dims:'. Give the two columns names: " - f'dims: {{{d}0: {d}, {d}1: {d}}}.' - for d, count in Counter(lk.dims if isinstance(lk.dims, tuple) else (lk.dims,)).items() + f"Multi-dim '{lname}' names dimension '{d}' twice under 'over:'. Declare an alias of '{d}' under " + f'dimensions:, such as {d}_to: {{alias: {d}}}, and list it in place of the second.' + for d, count in Counter(lk.over if isinstance(lk.over, tuple) else (lk.over,)).items() if count > 1 ) yield from ( - undeclared_dimension('Set', lname, d, spec) + undeclared_dimension('Multi-dim', lname, d, spec) for d in dict.fromkeys(d for _, d in lk.pairs) if d not in spec.dimensions ) yield from ( - f"Set '{lname}' names column '{role}' after dimension '{role}', but the column is over " - f"'{dim}'. Rename the column after what it holds." + f"Multi-dim '{lname}' names member '{role}' after dimension '{role}', but the member is over " + f"'{dim}'. Rename the member after what it holds." for role, dim in lk.pairs if role in spec.dimensions and role != dim ) @@ -240,19 +247,19 @@ def _map_targets(spec: Spec) -> Iterator[str]: """ for subject, maps in spec.maps.items(): head = f"maps: '{subject}'" - if subject in spec.sets: - pairs = spec.sets[subject].pairs + if subject in spec.multi_dims: + pairs = spec.multi_dims[subject].pairs if twice := [d for d, count in Counter(d for _, d in pairs).items() if count > 1]: yield ( - f"{head} is a set with two columns over '{twice[0]}', and a map reads one row per " - f'combination of its dimensions. Declare the maps on a set over each dimension once.' + f"{head} is a multi-dim with two members over '{twice[0]}', and a map reads one row per " + f'combination of its members. Declare the maps on a multi-dim over each dimension once.' ) elif subject in spec.dimensions: pairs = ((subject, subject),) else: - subjects = [*spec.dimensions, *spec.sets] + subjects = [*spec.dimensions, *spec.multi_dims] yield ( - f'{head} is neither a dimension nor a set, and a map gives each row of one a label. ' + f'{head} is neither a dimension nor a multi-dim, and a map gives each row of one a label. ' f'{did_you_mean(subject, subjects, label="Subjects")}' ) continue @@ -267,10 +274,10 @@ def _map_targets(spec: Spec) -> Iterator[str]: f"{head}: map '{role}' has the name of a column of '{subject}', which holds the row, not a " f'label of it. Rename the map after what it holds.' ) - if m.dim in spec.sets: + if m.dim in spec.multi_dims: yield ( - f"{head}: map '{role}' refers to the set '{m.dim}', and a map gives a label of one " - f"dimension. Refer to one of its dimensions, or declare '{role}' on the set instead." + f"{head}: map '{role}' refers to the multi-dim '{m.dim}', and a map gives a label of one " + f"dimension. Refer to one of its members, or declare '{role}' on the multi-dim instead." ) elif m.dim not in spec.dimensions: yield ( @@ -284,7 +291,7 @@ def _map_targets(spec: Spec) -> Iterator[str]: def _index_sets(spec: Spec) -> Iterator[str]: - """Each relation an entry names in ``dims:`` is named once and leaves a combination out.""" + """Each multi-dim an entry names in ``dims:`` is named once.""" entries = [ *((f"Parameter '{name}'", p.in_) for name, p in spec.parameters.items()), *((f"Variable '{name}'", v.in_) for name, v in spec.variables.items()), @@ -293,12 +300,6 @@ def _index_sets(spec: Spec) -> Iterator[str]: for context, in_ in entries: head = f'{context}: dims: names' yield from (f"{head} '{r}' twice. Delete one." for r, count in Counter(in_).items() if count > 1) - for rname in dict.fromkeys(in_): - if spec.sets[rname].missing == 'refused': - yield ( - f"{head} '{rname}', which under missing: refused has a row at every combination, so it " - f"leaves nothing out. Write missing: absent on '{rname}', or name its dimensions in its place." - ) def _aliases(spec: Spec) -> Iterator[str]: diff --git a/src/mathspec/spec.py b/src/mathspec/spec.py index afcdc9b80..a249be720 100644 --- a/src/mathspec/spec.py +++ b/src/mathspec/spec.py @@ -76,11 +76,11 @@ 'GivenVariableSpec', 'MacroSpec', 'MapSpec', + 'MultiDimSpec', 'ObjectiveSpec', 'ParameterSpec', 'PiecewiseLinkSpec', 'PiecewiseSpec', - 'SetSpec', 'SosSpec', 'Spec', 'VariableSpec', @@ -149,7 +149,10 @@ def _reject_unknown_keys(cls, data: object) -> object: return data known = set(_written_keys(cls).values()) unknown = [k for k in data if isinstance(k, str) and k not in known] - written = known - {'in_'} # the load writes `in_` from `dims:`; a file never does + written = known - { + 'in_', + 'multi_dims', + } # the load writes these from `dims:` and `dimensions:`; a file never does if unknown: raise ValueError( '\n'.join( @@ -252,41 +255,36 @@ def relation_columns(written: str | Sequence[str] | Mapping[str, str]) -> tuple[ return tuple((d, d) for d in ((written,) if isinstance(written, str) else written)) -class SetSpec(_StrictSpec): - """A named relation: the combinations of its dimensions that exist, one row each. +class MultiDimSpec(_StrictSpec): + """A multi-dim: a dimension whose labels are the combinations of its members that exist, one row each. - ``dims:`` is a list of dimensions, or a mapping of column name to - dimension where two columns share one. A row is its own identity, so the - relation is a subset of the product of its dimensions:: + Written under ``dimensions:`` with ``over:``, a list of the member + dimensions, or a mapping of column name to dimension where two columns + share one:: - sets: - connection: {dims: [unit, port], missing: absent} - corridor: {dims: {from: bus, to: bus}} + dimensions: + connection: {over: [unit, port]} + corridor: {over: {from: bus, to: bus}} - A relation is not a dimension. Its name stands under an entry's ``in:``, - in a condition, where it is true at the rows it lists, and before a - bracket, where it reads one of its maps (``maps:``, [`MapSpec`][]). The - rows are data, and arrive under the relation's name, one column per - column name, with a column per map beside them. + In an entry's ``dims:`` and in ``over=``, a multi-dim stands for its + members, at the rows it lists. Two multi-dims in one ``dims:`` join on the + members they share. Its name also stands in a condition, where it is true + at the rows it lists, and before a bracket, where it reads one of its maps + (``maps:``, [`MapSpec`][]). The rows are data, and arrive under the + multi-dim's name, one column per member, with a column per map beside + them. A combination with no row is not a label of the multi-dim. """ - _label: ClassVar[str] = 'a set entry' + _label: ClassVar[str] = 'a multi-dim entry' - dims: str | Dims | ByName[str] - """The relation's columns: a list of dimensions, each a column named after it, or a mapping of column name to - dimension. - """ - missing: RelationMissing = 'refused' - """Whether every combination of its dimensions is a row. Under ``refused``, a tool refuses data that leaves one out. - Under ``absent``, that combination is not in the relation. - """ + over: str | Dims | ByName[str] + """The members: a list of dimensions, each a column named after it, or a mapping of column name to dimension.""" description: str | None = None - @field_validator('missing', mode='before') - @classmethod - def _refused_or_absent(cls, v: object) -> object: - """A row is data, so no value fills a gap in a relation.""" - return _refused_or_absent(v, 'a set', 'Write absent where a combination may have no row, or leave it out.') + @property + def missing(self) -> RelationMissing: + """A combination with no row is not a label, so a multi-dim's rows are never refused for a gap.""" + return 'absent' @property def pairs(self) -> tuple[tuple[str, str], ...]: @@ -294,7 +292,7 @@ def pairs(self) -> tuple[tuple[str, str], ...]: The program calls the same thing [`columns`][mathspec.program.Relation.columns]. """ - return relation_columns(self.dims) + return relation_columns(self.over) @property def roles(self) -> tuple[str, ...]: @@ -1236,7 +1234,9 @@ class is pydantic's, not a contract this package keeps. given: GivenSpec = GivenSpec() """What this file uses and does not build ([`GivenSpec`][]). It is empty when the file has no ``given:``.""" dimensions: ByName[DimensionSpec] = Frozen({}) - sets: ByName[SetSpec] = Frozen({}) + """Each dimension by name ([`DimensionSpec`][]), and each multi-dim, written with ``over:`` ([`MultiDimSpec`][]).""" + multi_dims: ByName[MultiDimSpec] = Frozen({}) + """The multi-dims, which the file writes under ``dimensions:``; the load moves them here before validation.""" maps: ByName[ByName[MapSpec]] = Frozen({}) """Each subject's maps, by subject, a dimension or a relation, and then by map ([`MapSpec`][]).""" parameters: ByName[ParameterSpec] = Frozen({}) @@ -1323,18 +1323,32 @@ def _drop_absence(self, handler: SerializerFunctionWrapHandler) -> dict[str, obj read in. """ written = _as_set(self, cast('dict[str, object]', handler(self))) - return {'version': self.version, **_sets_written_back(written, self.sets)} + return {'version': self.version, **_multi_dims_written_back(_sets_written_back(written, self.multi_dims))} @model_validator(mode='before') @classmethod def _sets_in_dims(cls, data: object) -> object: - """A relation named in the ``dims:`` of a parameter, a variable or a constraint stands for its dimensions. + """A multi-dim named in the ``dims:`` of a parameter, a variable or a constraint stands for its members. - The entry is over those dimensions, at the rows the relation lists, - so every check after this one sees plain dimensions. ``in_`` keeps - the relations, and the serializer writes them back into ``dims:``. + The multi-dims leave ``dimensions:`` for ``multi_dims`` first, so a + dimension is always an axis. The entry is over the members, at the + rows the multi-dim lists, so every check after this one sees plain + dimensions. ``in_`` keeps the multi-dims, and the serializer writes + them back into ``dims:``. """ - return _over_sets(data) if isinstance(data, dict) else data + return _over_sets(_multi_dims_split(data)) if isinstance(data, dict) else data + + @classmethod + @override + def __get_pydantic_json_schema__(cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler) -> dict[str, object]: + """The schema of the file: a multi-dim is an entry of ``dimensions:``, and ``multi_dims`` is no key of it.""" + generated = handler(core_schema) + resolved = handler.resolve_ref_schema(generated) if set(generated) == {'$ref'} else generated + properties = cast('dict[str, dict[str, object]]', resolved['properties']) + multi = properties.pop('multi_dims')['additionalProperties'] + dimensions = properties['dimensions'] + dimensions['additionalProperties'] = {'anyOf': [dimensions['additionalProperties'], multi]} + return generated def to_dict(self) -> dict[str, object]: """The spec as plain data. ``to_spec(m.to_dict())`` reproduces it.""" @@ -1445,16 +1459,46 @@ def _members(relation: object) -> tuple[str, ...] | None: An entry over a dimension has it once, so a relation with two columns over one dimension stays a name the dimension check refuses. """ - dims = relation.get('dims') if isinstance(relation, Mapping) else getattr(relation, 'dims', None) + dims = relation.get('over') if isinstance(relation, Mapping) else getattr(relation, 'over', None) listed = [dims] if isinstance(dims, str) else list(dims.values()) if isinstance(dims, Mapping) else dims if not isinstance(listed, (list, tuple)) or len(set(listed)) != len(listed): return None return tuple(listed) +def _multi_dims_split(data: dict[str, object]) -> dict[str, object]: + """*data* with each entry of ``dimensions:`` that has ``over:`` moved to ``multi_dims``.""" + if 'multi_dims' in data: + msg = "unknown key 'multi_dims' at the top level. Declare a multi-dim under dimensions:, with over:." + raise ValueError(msg) + dimensions = data.get('dimensions') + if not isinstance(dimensions, Mapping): + return data + multi = {name: entry for name, entry in dimensions.items() if isinstance(entry, Mapping) and 'over' in entry} + if not multi: + return data + plain = {name: entry for name, entry in dimensions.items() if name not in multi} + return {**data, 'dimensions': plain, 'multi_dims': multi} + + +def _multi_dims_written_back(written: dict[str, object]) -> dict[str, object]: + """*written* with ``multi_dims`` folded back into ``dimensions:``, by name, as the file writes them.""" + multi = written.get('multi_dims') + if not isinstance(multi, Mapping): + return written + merged = {**cast('Mapping[str, object]', written.get('dimensions', {})), **multi} + dimensions = {name: merged[name] for name in sorted(merged)} + at = 'dimensions' if 'dimensions' in written else 'multi_dims' + return { + ('dimensions' if key == at else key): (dimensions if key == at else value) + for key, value in written.items() + if key in (at, 'version') or key not in ('dimensions', 'multi_dims') + } + + def _over_sets(data: dict[str, object]) -> dict[str, object]: - """*data* with each relation in an entry's ``dims:`` replaced by its dimensions and recorded under ``in_``.""" - relations = data.get('sets') + """*data* with each multi-dim in an entry's ``dims:`` replaced by its members and recorded under ``in_``.""" + relations = data.get('multi_dims') sets = { name: members for name, entry in (relations.items() if isinstance(relations, Mapping) else ()) @@ -1483,7 +1527,7 @@ def _entry_over_sets(entry: object, sets: Mapping[str, tuple[str, ...]], section return {**entry, 'dims': flat, 'in_': named} -def _sets_written_back(written: dict[str, object], relations: Mapping[str, SetSpec]) -> dict[str, object]: +def _sets_written_back(written: dict[str, object], relations: Mapping[str, MultiDimSpec]) -> dict[str, object]: """*written* with each entry's ``in_`` folded back into its ``dims:``, as the file writes it.""" out = dict(written) for section in _SET_SECTIONS: @@ -1492,7 +1536,7 @@ def _sets_written_back(written: dict[str, object], relations: Mapping[str, SetSp return out -def _entry_written_back(entry: object, relations: Mapping[str, SetSpec]) -> object: +def _entry_written_back(entry: object, relations: Mapping[str, MultiDimSpec]) -> object: """One entry's ``dims:`` with the dimensions of each relation in ``in_`` written as the relation, at its first.""" if not isinstance(entry, Mapping) or not (named := entry.get('in_')): return entry @@ -1524,11 +1568,11 @@ def subject_tables(spec: Spec) -> dict[str, Relation]: """ tables: dict[str, Relation] = {} aliases = {name: d.alias for name, d in spec.dimensions.items() if d.alias in spec.dimensions} - for name in sorted({*spec.sets, *spec.maps, *aliases}): + for name in sorted({*spec.multi_dims, *spec.maps, *aliases}): maps = spec.maps.get(name, Frozen({})) if name in aliases: maps = {aliases[name]: MapSpec(dim=aliases[name]), **maps} - if (relation := spec.sets.get(name)) is not None: + if (relation := spec.multi_dims.get(name)) is not None: key, missing, description = relation.pairs, relation.missing, relation.description elif name in spec.dimensions: key, missing, description = ((name, name),), cast('RelationMissing', 'refused'), None diff --git a/tests/test_alias.py b/tests/test_alias.py index 754d8bd32..a6ba4ecd6 100644 --- a/tests/test_alias.py +++ b/tests/test_alias.py @@ -127,9 +127,7 @@ def test_ire_flo_is_over_the_set_of_links_between_two_regions_and_two_commoditie 'commodity': {}, 'from_region': {'alias': 'region'}, 'from_commodity': {'alias': 'commodity'}, - }, - 'sets': { - 'top_ire': {'dims': ['from_region', 'from_commodity', 'region', 'commodity'], 'missing': 'absent'} + 'top_ire': {'over': ['from_region', 'from_commodity', 'region', 'commodity']}, }, 'parameters': {'ire_flo': {'dims': ['top_ire']}}, 'variables': {'export': {'dims': ['top_ire']}, 'supply': {'dims': ['region', 'commodity']}}, diff --git a/tests/test_canonical.py b/tests/test_canonical.py index 91600e312..161c37d5c 100644 --- a/tests/test_canonical.py +++ b/tests/test_canonical.py @@ -153,7 +153,7 @@ def test_a_map_is_written_back_as_the_mapping_it_was_declared_as(declared): 'description', 'given', 'dimensions', - 'sets', + 'multi_dims', 'maps', 'parameters', 'conditions', diff --git a/tests/test_composition.py b/tests/test_composition.py index fa8aad802..b967ead40 100644 --- a/tests/test_composition.py +++ b/tests/test_composition.py @@ -229,10 +229,9 @@ def _mapped(missing: str | None = None, dim: str = 'bus') -> dict[str, object]: return {'dimensions': {'port': {'dtype': 'str'}, dim: {'dtype': 'str'}}, 'maps': {'port': {'bus': entry}}} -def _wired(missing: str | None = None) -> dict[str, object]: - """A fragment that declares only the relation `wired` over `port` and `bus`, as `_mapped` does a map.""" - entry = {'dims': ['port', 'bus']} | ({} if missing is None else {'missing': missing}) - return {'dimensions': {'port': {'dtype': 'str'}, 'bus': {'dtype': 'str'}}, 'sets': {'wired': entry}} +def _wired(*over: str) -> dict[str, object]: + """A fragment that declares only the multi-dim `wired` over *over*, as `_mapped` does a map.""" + return {'dimensions': {d: {'dtype': 'str'} for d in over} | {'wired': {'over': list(over)}}} #: A fragment that gives `port` a map of its own beside the one `SURFACE` gives it. @@ -276,7 +275,9 @@ def test_two_fragments_that_say_one_missing_agree(fragments, missing): pytest.param([SURFACE, _mapped('absent')], "map 'port[bus]'", id='refused-against-absent'), pytest.param([_mapped('absent'), SURFACE], "map 'port[bus]'", id='absent-against-refused'), pytest.param([SURFACE, _mapped(dim='node')], "map 'port[bus]'", id='two-dimensions-referred-to'), - pytest.param([_wired(), _wired('absent')], "set 'wired'", id='a-set-refused-against-absent'), + pytest.param( + [_wired('port', 'bus'), _wired('port', 'node')], "dimension 'wired'", id='a-multi-dim-over-other-members' + ), ], ) def test_two_fragments_that_say_different_things_about_one_entry_are_refused(fragments, entry): @@ -599,7 +600,7 @@ def test_a_patch_restates_a_dimension_as_the_schema_reads_it(dimension, restated #: `SURFACE` with a relation, `wired`, over the two dimensions `port[bus]` maps between. -WIRED = varied(SURFACE, **{'sets.wired': {'dims': ['port', 'bus']}}) +WIRED = varied(SURFACE, **{'dimensions.wired': {'over': ['port', 'bus']}}) @pytest.mark.parametrize( @@ -613,9 +614,9 @@ def test_a_patch_restates_a_dimension_as_the_schema_reads_it(dimension, restated id='a-dimension', ), pytest.param( - varied(WIRED, **{'sets.wired.description': 'which ports reach which buses'}), - {'sets': {'wired': {'dims': ['port', 'bus'], 'description': 'a port and the buses it reaches'}}}, - lambda spec: spec.sets['wired'].description, + varied(WIRED, **{'dimensions.wired.description': 'which ports reach which buses'}), + {'dimensions': {'wired': {'over': ['port', 'bus'], 'description': 'a port and the buses it reaches'}}}, + lambda spec: spec.multi_dims['wired'].description, 'a port and the buses it reaches', id='a-relation', ), @@ -641,9 +642,9 @@ def test_a_patch_restates_a_dimension_as_the_schema_reads_it(dimension, restated id='null-removes-a-dimension-description', ), pytest.param( - varied(WIRED, **{'sets.wired.description': 'which ports reach which buses'}), - {'sets': {'wired': {'description': None}}}, - lambda spec: spec.sets['wired'].description, + varied(WIRED, **{'dimensions.wired.description': 'which ports reach which buses'}), + {'dimensions': {'wired': {'description': None}}}, + lambda spec: spec.multi_dims['wired'].description, None, id='null-removes-a-relation-description', ), @@ -725,8 +726,10 @@ def test_a_patch_that_rewrites_a_dimension_is_refused(patch, says): @pytest.mark.parametrize( ('patch', 'kind', 'name'), [ - pytest.param({'sets': {'wired': {}}}, 'set', 'wired', id='a-set-with-no-field'), - pytest.param({'sets': {'wired': {'dims': ['port']}}}, 'set', 'wired', id='a-set-over-one-dimension'), + pytest.param({'dimensions': {'wired': {}}}, 'dimension', 'wired', id='a-multi-dim-with-no-field'), + pytest.param( + {'dimensions': {'wired': {'over': ['port']}}}, 'dimension', 'wired', id='a-multi-dim-over-one-dimension' + ), pytest.param({'maps': {'port': {'bus': {}}}}, 'map', 'port[bus]', id='a-map-with-no-field'), ], ) @@ -745,8 +748,8 @@ def test_a_patch_that_restates_an_entry_in_part_is_refused_with_the_rewrite(patc ('patch', 'says'), [ pytest.param( - {'sets': {'wired': {'dmis': ['port', 'bus']}}}, - "unknown key 'dmis' in a set entry. Did you mean 'dims'?", + {'dimensions': {'wired': {'over': ['port', 'bus'], 'ovr': ['port', 'bus']}}}, + "unknown key 'ovr' in a multi-dim entry. Did you mean 'over'?", id='a-relation', ), pytest.param( @@ -801,10 +804,6 @@ def test_a_patch_changes_what_a_row_the_map_gives_no_label_means(base, map_, mis assert override(base, [{'maps': {'port': {'bus': map_}}}]).maps['port']['bus'].missing == missing -def test_a_patch_changes_what_a_combination_the_relation_leaves_out_means(): - assert override(WIRED, [{'sets': {'wired': {'missing': 'absent'}}}]).sets['wired'].missing == 'absent' - - @pytest.mark.parametrize( ('base', 'written', 'missing'), [ @@ -825,7 +824,6 @@ def test_a_patch_changes_what_a_missing_row_of_a_parameter_means(base, written, ('section', 'name', 'declared'), [ pytest.param('parameters', 'c', {'dims': ['g']}, id='a-parameter'), - pytest.param('sets', 'gh', {'dims': ['g', 'h']}, id='a-relation'), pytest.param('maps', 'g.h', {'dim': 'h'}, id='a-map'), ], ) diff --git a/tests/test_dimensions.py b/tests/test_dimensions.py index ada336922..25fef0e3b 100644 --- a/tests/test_dimensions.py +++ b/tests/test_dimensions.py @@ -66,8 +66,8 @@ 'generator': {'dtype': 'str', 'ordered': True}, 'bus': {'dtype': 'str'}, 'zone': {'dtype': 'str'}, + 'gen_zone': {'over': ['generator', 'snapshot']}, }, - 'sets': {'gen_zone': {'dims': ['generator', 'snapshot']}}, 'maps': { 'generator': {'bus': {'dim': 'bus'}, 'zone': {'dim': 'zone'}, 'b0': {'dim': 'bus'}, 'b1': {'dim': 'bus'}}, 'snapshot': {'bus': {'dim': 'bus'}, 'rep': {'dim': 'snapshot'}}, @@ -253,7 +253,7 @@ def _dims_with(expr: str, **overrides) -> frozenset[str]: ) def test_a_bare_relation_is_summed_between_its_key_columns(expr, expected): """A bare relation holds no value column, so the column a sum groups by is a key column.""" - assert _dims_with(expr, **{'sets.connection': {'dims': ['generator', 'bus']}}) == expected + assert _dims_with(expr, **{'dimensions.connection': {'over': ['generator', 'bus']}}) == expected def test_a_value_column_added_to_a_relation_changes_no_call(): @@ -433,7 +433,7 @@ def test_an_ill_dimensioned_expression_is_rejected(expr, error, match): [ pytest.param( 'sum(p, over=k, by=diag[z])', - {'dims': {'k': 'generator', 'j': 'generator', 'z': 'zone'}}, + {'over': {'k': 'generator', 'j': 'generator', 'z': 'zone'}}, id='a-sum-summing-away-a-column-over-a-dimension-it-also-joins-on', ), ], @@ -441,7 +441,7 @@ def test_an_ill_dimensioned_expression_is_rejected(expr, error, match): def test_two_joined_columns_do_not_share_a_dimension(expr, diag): """The operand carries one coordinate per dimension, so the `over=` column and an unnamed key column cannot share one.""" with pytest.raises(DimensionError, match=r"joins 'diag' on \[.*\] through more than one column"): - _dims_with(expr, **{'sets.diag': diag}) + _dims_with(expr, **{'dimensions.diag': diag}) @pytest.mark.parametrize( @@ -695,7 +695,7 @@ def test_a_condition_names_the_entries_its_leaves_test(namespace, condition, exp ) def test_names_read_takes_both_sides_of_a_map_pair(condition, expected): """The one leaf that can name two entries — two maps compared row by row on the key their subjects share.""" - home_of = {'sets.home_of': {'dims': ['generator', 'snapshot']}, 'maps.home_of': {'zone': {'dim': 'zone'}}} + home_of = {'dimensions.home_of': {'over': ['generator', 'snapshot']}, 'maps.home_of': {'zone': {'dim': 'zone'}}} where = where_of(condition, Namespace(_spec(**home_of)), 'test') assert where is not None, 'a condition the connectives cannot settle survives the fold' @@ -974,11 +974,11 @@ def test_every_dims_of_the_program_is_sorted_by_name(path): ) def test_a_condition_on_a_relation_states_its_key_dims_sorted_by_name(entry): """A condition on a relation stated the dims in the order the relation lists its key.""" - keyed_backwards = {'dims': ['snapshot', 'generator'], 'missing': 'absent'} + keyed_backwards = {'over': ['snapshot', 'generator']} spec = _spec( **{ - 'sets.zone_of': keyed_backwards, - 'sets.home_of': keyed_backwards, + 'dimensions.zone_of': keyed_backwards, + 'dimensions.home_of': keyed_backwards, 'maps.zone_of': {'zone': {'dim': 'zone'}}, 'maps.home_of': {'zone': {'dim': 'zone'}}, } diff --git a/tests/test_expansion.py b/tests/test_expansion.py index 9c569ff55..0a2a18b19 100644 --- a/tests/test_expansion.py +++ b/tests/test_expansion.py @@ -290,7 +290,7 @@ def test_macro_collisions_rejected(patch, match): ), pytest.param( {'grouped': {'args': ['x'], 'template': 'sum(x, over=g, by=nope[h])'}}, - r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a set", + r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a multi-dim", id='a-typo-in-a-relation-kwarg', ), pytest.param( @@ -300,7 +300,7 @@ def test_macro_collisions_rejected(patch, match): ), pytest.param( {'grouped': {'args': ['x', 'a', 'b'], 'template': 'sum(x, over=a, by=nope[b])'}}, - r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a set, a dimension with maps or a formal argument of this macro", + r"Macro 'grouped'.*sum\(by=nope\[\.\.\.\]\): 'nope' is not a multi-dim, a dimension with maps or a formal argument of this macro", id='a-typo-in-a-relation-beside-formal-columns', ), pytest.param( diff --git a/tests/test_lowering.py b/tests/test_lowering.py index 6ecd0e037..f0c99ac1a 100644 --- a/tests/test_lowering.py +++ b/tests/test_lowering.py @@ -719,8 +719,12 @@ def test_a_relation_lowers_with_the_join_each_call_names(): """Every node reading a relation carries its columns, its key and the join, so a consumer joins on the right columns.""" program = to_spec( { - 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'generator': {'ordered': True}, 'zone': {}}, - 'sets': {'zone_of': {'dims': ['generator', 'snapshot'], 'missing': 'absent'}}, + 'dimensions': { + 'snapshot': {'dtype': 'int', 'ordered': True}, + 'generator': {'ordered': True}, + 'zone': {}, + 'zone_of': {'over': ['generator', 'snapshot']}, + }, 'maps': {'zone_of': {'zone': {'dim': 'zone'}}}, 'parameters': {'price': {'dims': ['snapshot', 'zone']}}, 'variables': { @@ -812,7 +816,7 @@ def test_a_relation_lowers_with_the_join_each_call_names(): def test_a_condition_tests_a_row_or_a_label_by_how_it_is_written(variable, entry, atom): """A relation's bare name asks for a row, and `subject[map]` alone asks whether the map gives the row a label.""" patch = { - 'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'dimensions.link': {'over': ['g', 'h']}, 'maps.g.h': {'dim': 'h', 'missing': 'absent'}, **{f'variables.{variable}.{key}': value for key, value in entry.items()}, } @@ -896,8 +900,7 @@ def test_a_relation_is_declared_as_the_file_declares_it(): program = to_spec( varied( TINY, - dimensions={'g': {}, 'bus': {}, 'season': {}}, - sets={'link': {'dims': ['g', 'bus'], 'missing': 'absent'}}, + dimensions={'g': {}, 'bus': {}, 'season': {}, 'link': {'over': ['g', 'bus']}}, maps={'g': {'season': {'dim': 'season'}, 'bus': {'dim': 'bus', 'missing': 'absent'}}}, ) ).program diff --git a/tests/test_piecewise.py b/tests/test_piecewise.py index d3516cc4e..c6613461c 100644 --- a/tests/test_piecewise.py +++ b/tests/test_piecewise.py @@ -796,7 +796,7 @@ def test_a_where_joins_both_gate_rows(gate: dict[str, Any]): **{ 'variables.u': {'dims': ['snapshot', 'generator'], 'domain': 'binary', **gate}, 'parameters.committable': {'dims': ['generator'], 'dtype': 'bool'}, - 'sets.running': {'dims': ['generator', 'snapshot'], 'missing': 'absent'}, + 'dimensions.running': {'over': ['generator', 'snapshot']}, 'piecewise.cost_curve.activity': 'u', }, ) @@ -983,7 +983,7 @@ def _walk(**written: object) -> dict[str, object]: def _joined(dims: list[str] | dict[str, str]) -> dict[str, object]: """The `power` link walked through `generator_of`, a relation over *dims* that maps each row to a generator.""" - return {'sets.generator_of': {'dims': dims}, 'maps.generator_of.generator': {'dim': 'generator'}} | _walk( + return {'dimensions.generator_of': {'over': dims}, 'maps.generator_of.generator': {'dim': 'generator'}} | _walk( by='generator_of[generator]' ) @@ -1408,7 +1408,7 @@ def test_a_gate_read_through_a_relation_prints_as_the_read(): ), pytest.param( {'piecewise.curve.activity': 'at(running, by=bp_p[converter])'}, - r"piecewise 'curve' activity: at\(by=bp_p\[\.\.\.\]\): 'bp_p' is not a set", + r"piecewise 'curve' activity: at\(by=bp_p\[\.\.\.\]\): 'bp_p' is not a multi-dim", id='a-walk-through-something-no-relation', ), pytest.param( @@ -1525,8 +1525,7 @@ def test_a_walked_row_exists_only_where_its_relation_has_a_row(patch, where): #: fluxopt's investment cost curve (fluxopt/fluxopt#26): each effect of a sized flow is read off its curve. INVESTED = { 'dimensions': {name: {'dtype': 'str'} for name in ('cost_curve', 'effect', 'flow')} - | {'bp': {'dtype': 'int', 'ordered': True}}, - 'sets': {'cost_of': {'dims': ['flow', 'effect']}}, + | {'bp': {'dtype': 'int', 'ordered': True}, 'cost_of': {'over': ['flow', 'effect']}}, 'maps': {'cost_of': {'cost_curve': {'dim': 'cost_curve'}}}, 'parameters': { 'has_cost_curve': {'dims': ['cost_curve'], 'dtype': 'bool'}, @@ -1611,7 +1610,9 @@ def test_a_mask_over_a_dim_the_walk_joins_on_reaches_the_walked_row_as_written() }, ) ) - assert expanded.constraints['coupling_power'].where == 'season' + assert expanded.constraints['coupling_power'].where == '(generator_of[generator]) AND (season)', ( + 'the mask is kept, beside the rows a multi-dim lists, which are all a multi-dim has' + ) @pytest.mark.parametrize( diff --git a/tests/test_pypsa_references.py b/tests/test_pypsa_references.py index 498872a4a..59fd5a883 100644 --- a/tests/test_pypsa_references.py +++ b/tests/test_pypsa_references.py @@ -180,7 +180,7 @@ def test_a_patch_keeps_the_pypsa_name_of_what_it_changes(page: str): """A patch rewrites entries of the base; a rewritten name keeps its PyPSA name and its dtype, or it has drifted.""" own = SPECS[page] drifted = [] - for section in ('parameters', 'sets', 'variables', 'constraints'): + for section in ('parameters', 'multi_dims', 'variables', 'constraints'): theirs, ours = getattr(BASE, section), getattr(own, section) for name in set(theirs) & set(ours): if (theirs[name].description or '').split(' — ')[0] != (ours[name].description or '').split(' — ')[0]: diff --git a/tests/test_renaming.py b/tests/test_renaming.py index 98eadceb1..7cf2f7d0e 100644 --- a/tests/test_renaming.py +++ b/tests/test_renaming.py @@ -24,8 +24,12 @@ #: One class written over placeholder names: the `unit` dimension and the #: `unit_` entries. Every place a name can stand appears once. UNIT = { - 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'unit': {'dtype': 'str'}, 'bus': {'dtype': 'str'}}, - 'sets': {'unit_link': {'dims': ['bus', 'unit'], 'missing': 'absent'}}, + 'dimensions': { + 'snapshot': {'dtype': 'int', 'ordered': True}, + 'unit': {'dtype': 'str'}, + 'bus': {'dtype': 'str'}, + 'unit_link': {'over': ['bus', 'unit']}, + }, 'maps': {'unit': {'bus': {'dim': 'bus'}, 'bus0': {'dim': 'bus'}}}, 'given': {'expressions': {'total_cost': {'dims': []}}}, 'parameters': { @@ -61,7 +65,8 @@ #: The placeholder names, as a class takes them. PLACEHOLDERS = ( - *(name for section in ('sets', 'parameters', 'conditions', 'variables', 'expressions') for name in UNIT[section]), + 'unit_link', + *(name for section in ('parameters', 'conditions', 'variables', 'expressions') for name in UNIT[section]), 'unit_ramp', 'unit_bounded', 'unit_twice', @@ -87,12 +92,10 @@ def _as(prefix: str, dim: str) -> dict[str, str]: @pytest.mark.parametrize( ('where', 'expected'), [ - pytest.param(lambda s: list(s['dimensions']), ['bus', 'generator', 'snapshot'], id='dimension'), pytest.param( - lambda s: s['sets']['Generator_link'], - {'dims': ['bus', 'generator'], 'missing': 'absent'}, - id='relation', + lambda s: list(s['dimensions']), ['Generator_link', 'bus', 'generator', 'snapshot'], id='dimension' ), + pytest.param(lambda s: s['dimensions']['Generator_link'], {'over': ['bus', 'generator']}, id='multi-dim'), pytest.param(lambda s: s['maps'], {'generator': {'bus': {'dim': 'bus'}, 'bus0': {'dim': 'bus'}}}, id='map'), pytest.param(lambda s: s['parameters']['Generator_cost']['dims'], ['generator', 'snapshot'], id='dims'), pytest.param(lambda s: s['variables']['Generator_p']['where'], 'Generator_on', id='where'), @@ -344,14 +347,17 @@ def test_a_map_the_spec_cannot_take_is_refused(names, message): ('old', 'message'), [ pytest.param('bus0', "'bus0' is also a map of 'line'", id='a-map'), - pytest.param('from_bus', "'from_bus' is also a column of the set 'corridor'", id='a-relation-column'), + pytest.param('from_bus', "'from_bus' is also a member of the multi-dim 'corridor'", id='a-relation-column'), ], ) def test_a_name_that_is_also_a_map_or_a_relation_column_is_refused(old, message): """`line[bus0]` reads a map, and a rename of the parameter `bus0` would change it too.""" spec = { - 'dimensions': {'line': {'dtype': 'str'}, 'bus': {'dtype': 'str'}}, - 'sets': {'corridor': {'dims': {'from_bus': 'bus', 'to_bus': 'bus'}}}, + 'dimensions': { + 'line': {'dtype': 'str'}, + 'bus': {'dtype': 'str'}, + 'corridor': {'over': {'from_bus': 'bus', 'to_bus': 'bus'}}, + }, 'maps': {'line': {'bus0': {'dim': 'bus'}, 'bus1': {'dim': 'bus'}}}, 'parameters': {'bus0': {'dims': ['bus']}, 'from_bus': {'dims': ['bus']}}, 'variables': {'flow': {'dims': ['line']}}, diff --git a/tests/test_schema.py b/tests/test_schema.py index 1e14e1a27..201bfa69d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -59,7 +59,6 @@ def test_no_definition_refers_only_to_itself(): pytest.param('ObjectiveSpec', 'sense', spec.ObjectiveSense, id='sense'), pytest.param('VariableSpec', 'domain', spec.VariableDomain, id='domain'), pytest.param('VariableSpec', 'missing', Literal['absent'], id='variable-missing'), - pytest.param('SetSpec', 'missing', spec.RelationMissing, id='relation-missing'), pytest.param('ParameterSpec', 'missing', spec.RelationMissing, id='parameter-missing'), pytest.param('ParameterSpec', 'dtype', spec.ParameterDtype, id='parameter-dtype'), pytest.param('DimensionSpec', 'dtype', spec.DimensionDtype, id='dimension-dtype'), @@ -80,12 +79,11 @@ def test_a_closed_vocabulary_is_published_as_an_enum(entry, field, alias): ('entry', 'default'), [ pytest.param('ParameterSpec', 'refused', id='parameter'), - pytest.param('SetSpec', 'refused', id='relation'), pytest.param('VariableSpec', 'absent', id='variable'), ], ) def test_the_schema_publishes_the_reading_a_file_gets_by_leaving_missing_out(entry, default): - """The parameter and the relation hold `None` until read, and the schema published that `null`, not `refused`.""" + """The parameter holds `None` until read, and the schema published that `null`, not `refused`.""" published = json.loads(schema.PATH.read_text())['$defs'][entry]['properties']['missing'] assert published['default'] == default @@ -95,3 +93,12 @@ def test_the_piecewise_method_vocabulary_has_one_home(): assert set(get_args(spec.PiecewiseMethod)) == set(spec.PIECEWISE_METHODS), ( 'the typed methods and the emitting ones disagree, so a method is accepted that emits nothing or the reverse' ) + + +def test_a_multi_dim_is_published_as_an_entry_of_dimensions(): + """A multi-dim is written under `dimensions:`, so the schema offers `over:` there and no `multi_dims:` key.""" + published = json.loads(schema.PATH.read_text()) + entry = published['properties']['dimensions']['additionalProperties'] + refs = [branch['$ref'] for branch in entry['anyOf']] + assert refs == ['#/$defs/DimensionSpec', '#/$defs/MultiDimSpec'], 'a dimension entry is a dimension or a multi-dim' + assert 'multi_dims' not in published['properties'], 'the load fills multi_dims; a file never writes it' diff --git a/tests/test_validation.py b/tests/test_validation.py index 00bc71fe9..c0ec8e99c 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -88,7 +88,7 @@ class TestValidateExpressions: ), pytest.param( { - 'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'dimensions.link': {'over': ['g', 'h']}, 'constraints': {'cap': {'dims': ['g'], 'where': 'lnk', 'expression': 'p <= c'}}, }, ("'lnk' not found", "'link']"), @@ -329,7 +329,7 @@ class TestDimensionKwargs: ), pytest.param( 'sum(p, over=generator, by=generatr[zone]) == load', - ("'generatr' is not a set or a dimension with maps", "Did you mean 'generator'?"), + ("'generatr' is not a multi-dim or a dimension with maps", "Did you mean 'generator'?"), id='by-relation-typo', ), pytest.param( @@ -779,7 +779,7 @@ def test_it_resolves(self, mask: str, position: int, by: str | None): ( 'position(snapshot, within=load[snapshot]) == 0', [ - "position(within=load[...]): 'load' is not a set or a dimension with maps", + "position(within=load[...]): 'load' is not a multi-dim or a dimension with maps", 'Write within=[]', ], ), @@ -788,7 +788,7 @@ def test_it_resolves(self, mask: str, position: int, by: str | None): ["'period' has no column of its own over 'snapshot'", "Its own columns are ['period']"], ), ], - ids=['a parameter', 'undeclared', 'within= is not a set', 'within= is over another dim'], + ids=['a parameter', 'undeclared', 'within= is not a multi-dim', 'within= is over another dim'], ) def test_it_refuses(self, mask: str, fragments: list[str]): with pytest.raises(LanguageError) as excinfo: @@ -1053,8 +1053,8 @@ def test_a_read_names_several_columns_as_an_expression_does(self, where, dims, r """ spec = to_spec( { - 'dimensions': {name: {'dtype': 'str'} for name in ('curve', 'effect', 'flow')}, - 'sets': {'cost_of': {'dims': ['flow', 'effect']}}, + 'dimensions': {name: {'dtype': 'str'} for name in ('curve', 'effect', 'flow')} + | {'cost_of': {'over': ['flow', 'effect']}}, 'maps': { 'cost_of': {'curve': {'dim': 'curve'}}, 'curve': {'flow': {'dim': 'flow'}, 'effect': {'dim': 'effect'}}, @@ -1198,7 +1198,7 @@ def test_a_read_names_several_columns_as_an_expression_does(self, where, dims, r ), pytest.param( 'at(r, by=nope[h])', - ("at(by=nope[...]): 'nope' is not a set or a dimension with maps",), + ("at(by=nope[...]): 'nope' is not a multi-dim or a dimension with maps",), id='a-read-through-no-relation', ), pytest.param( @@ -1435,41 +1435,41 @@ class TestRulesDecidedWithoutData: id='sos-whose-expansion-collides-with-an-entry', ), pytest.param( - {'sets.link': {'dims': ['g', 'h'], 'dtype': 'str'}}, - ("unknown key 'dtype' in a set entry. Valid keys: description, dims, missing.",), + {'dimensions.link': {'over': ['g', 'h'], 'dtype': 'str'}}, + ("unknown key 'dtype' in a multi-dim entry. Valid keys: description, over.",), id='relation-with-a-dtype-of-its-own', ), pytest.param( - {'sets.link': {'dims': ['g']}}, + {'dimensions.link': {'over': ['g']}}, ( - "Set 'link' has one column under 'dims:', and a set needs at least two", + "Multi-dim 'link' has one member under 'over:', and a multi-dim needs at least two", 'To give each g a label, declare a map under maps: g:.', ), id='relation-with-one-column', ), pytest.param( - {'sets.link': {'dims': ['g', 'z']}}, - ("Set 'link' references undeclared dimension 'z'",), + {'dimensions.link': {'over': ['g', 'z']}}, + ("Multi-dim 'link' references undeclared dimension 'z'",), id='relation-over-undeclared', ), pytest.param( - {'sets.odd': {'dims': {'h': 'g', 'x': 'h'}}}, - ("names column 'h' after dimension 'h', but the column is over 'g'",), + {'dimensions.odd': {'over': {'h': 'g', 'x': 'h'}}}, + ("names member 'h' after dimension 'h', but the member is over 'g'",), id='relation-column-named-after-a-dimension-it-is-not-over', ), pytest.param( - {'sets.link': {'dims': ['g', 'g']}}, - ("names dimension 'g' twice under 'dims:'", 'dims: {g0: g, g1: g}'), + {'dimensions.link': {'over': ['g', 'g']}}, + ("names dimension 'g' twice under 'over:'", "Declare an alias of 'g' under dimensions:"), id='relation-naming-a-dim-twice-without-names', ), pytest.param( - {'sets.link': {'dims': []}}, - ("Set 'link' has no column under 'dims:', and a set needs at least two",), + {'dimensions.link': {'over': []}}, + ("Multi-dim 'link' has no member under 'over:', and a multi-dim needs at least two",), id='relation-with-no-columns', ), pytest.param( {'maps.nope': {'h': {'dim': 'h'}}}, - ("maps: 'nope' is neither a dimension nor a set", 'Subjects: g, h.'), + ("maps: 'nope' is neither a dimension nor a multi-dim", 'Subjects: g, h.'), id='map-of-no-subject', ), pytest.param( @@ -1478,13 +1478,13 @@ class TestRulesDecidedWithoutData: id='map-whose-name-is-not-a-name', ), pytest.param( - {'sets.link': {'dims': ['g', 'h']}, 'maps.link': {'g': {'dim': 'g'}}}, + {'dimensions.link': {'over': ['g', 'h']}, 'maps.link': {'g': {'dim': 'g'}}}, ("maps: 'link': map 'g' has the name of a column of 'link'", 'Rename the map after what it holds'), id='map-named-after-a-column-of-its-subject', ), pytest.param( - {'sets.link': {'dims': ['g', 'h']}, 'maps.g': {'x': {'dim': 'link'}}}, - ("maps: 'g': map 'x' refers to the set 'link', and a map gives a label of one dimension",), + {'dimensions.link': {'over': ['g', 'h']}, 'maps.g': {'x': {'dim': 'link'}}}, + ("maps: 'g': map 'x' refers to the multi-dim 'link', and a map gives a label of one dimension",), id='map-referring-to-a-relation', ), pytest.param( @@ -1498,8 +1498,8 @@ class TestRulesDecidedWithoutData: id='map-named-after-another-dimension', ), pytest.param( - {'sets.pair': {'dims': {'g0': 'g', 'g1': 'g'}}, 'maps.pair': {'x': {'dim': 'h'}}}, - ("maps: 'pair' is a set with two columns over 'g'", 'Declare the maps on a set over each'), + {'dimensions.pair': {'over': {'g0': 'g', 'g1': 'g'}}, 'maps.pair': {'x': {'dim': 'h'}}}, + ("maps: 'pair' is a multi-dim with two members over 'g'", 'Declare the maps on a multi-dim over each'), id='map-on-a-relation-with-two-columns-over-one-dimension', ), pytest.param( @@ -1525,7 +1525,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'sets.bare': {'dims': {'a': 'g', 'b': 'g', 'm': 'z'}}, + 'dimensions.bare': {'over': {'a': 'g', 'b': 'g', 'm': 'z'}}, 'objective': {'expression': 'sum(sum(p, over=g, by=bare[m]))'}, }, ( @@ -1537,7 +1537,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'sets.gz': {'dims': ['g', 'z']}, + 'dimensions.gz': {'over': ['g', 'z']}, 'objective': {'expression': 'sum(sum(q, over=h, by=gz[z]))'}, }, ("over=h names no column of 'gz', so the sum reads nothing through 'gz'", "its columns are ['g', 'z']"), @@ -1607,7 +1607,7 @@ class TestRulesDecidedWithoutData: id='a-selection-in-over-without-by', ), pytest.param( - {'sets.rel': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(sum(p, by=rel[h]))'}}, + {'dimensions.rel': {'over': ['g', 'h']}, 'objective': {'expression': 'sum(sum(p, by=rel[h]))'}}, ('sum() through a set does not name over=', 'names in over= the columns it sums away'), id='a-sum-through-a-relation-names-what-leaves', ), @@ -1632,7 +1632,7 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'sets.rel': {'dims': ['g', 'h']}, + 'dimensions.rel': {'over': ['g', 'h']}, 'objective': {'expression': 'sum(at(r, by=rel[h]))'}, }, ("'rel' has no map for at() to read. Sum through it instead, or declare a map under maps: rel:.",), @@ -1649,14 +1649,14 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'sets.rel': {'dims': ['g', 'h']}, + 'dimensions.rel': {'over': ['g', 'h']}, 'objective': {'expression': 'sum(shift(p, along=g, offset=1, edge=0, within=rel[h]))'}, }, ("'rel' has no map, so it makes no groups", 'under maps: g:'), id='a-partition-through-a-relation-with-no-map', ), pytest.param( - {'sets.rel': {'dims': ['g', 'h']}, 'variables.q.where': "rel[h] == 'x'"}, + {'dimensions.rel': {'over': ['g', 'h']}, 'variables.q.where': "rel[h] == 'x'"}, ( "compares the set 'rel', which has no map, so its rows hold no label", "Test the bare name 'rel' for whether a row exists, or declare a map under maps: rel:.", @@ -1670,25 +1670,20 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'sets.pair': {'dims': {'g0': 'g', 'g1': 'g'}}, + 'dimensions.pair': {'over': {'g0': 'g', 'g1': 'g'}}, 'variables.q.where': 'pair', }, ("'pair' has two columns over one dimension (['g0', 'g1']), so a bare name is ambiguous",), id='where-bare-name-of-a-relation-with-two-columns-over-one-dim', ), pytest.param( - {'sets.h': {'dims': ['g', 'h']}}, - ("Set 'h' collides with the dimension",), - id='relation-named-after-a-dimension', - ), - pytest.param( - {'sets.link': {'dims': ['g', 'h'], 'rows': {'g1': 'h1'}}}, - ("unknown key 'rows' in a set entry", 'Valid keys'), + {'dimensions.link': {'over': ['g', 'h'], 'rows': {'g1': 'h1'}}}, + ("unknown key 'rows' in a multi-dim entry", 'Valid keys'), id='a-relation-declaring-its-rows', ), pytest.param( - {'sets.link': {'dim': ['g', 'h']}}, - ("unknown key 'dim' in a set entry", "Did you mean 'dims'?"), + {'dimensions.link': {'over': ['g', 'h'], 'ovr': ['g', 'h']}}, + ("unknown key 'ovr' in a multi-dim entry", "Did you mean 'over'?"), id='a-relation-with-a-misspelt-key', ), pytest.param({'variables.p.missing': 0}, ('missing: 0 needs a `where:`',), id='zero-without-a-mask'), @@ -1767,8 +1762,8 @@ class TestRulesDecidedWithoutData: id='a-dimension-as-a-value', ), pytest.param( - {'sets.link': {'dims': ['g', 'h']}, 'objective': {'expression': 'sum(link + p, over=g)'}}, - ("'link' is a set, and a set is not a value",), + {'dimensions.link': {'over': ['g', 'h']}, 'objective': {'expression': 'sum(link + p, over=g)'}}, + ("'link' is a multi-dim, and a multi-dim is not a value",), id='a-relation-as-a-value', ), pytest.param( @@ -1816,7 +1811,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'sets.bare': {'dims': {'k': 'g', 'j0': 'h', 'j1': 'h', 'm': 'z'}}, + 'dimensions.bare': {'over': {'k': 'g', 'j0': 'h', 'j1': 'h', 'm': 'z'}}, 'objective': {'expression': 'sum(sum(q, over=k, by=bare[m]))'}, }, ("joins 'bare' on ['h'] through more than one column",), @@ -1825,7 +1820,7 @@ class TestRulesDecidedWithoutData: pytest.param( { 'dimensions.z': {}, - 'sets.gz': {'dims': ['g', 'z']}, + 'dimensions.gz': {'over': ['g', 'z']}, 'maps.gz': {'h': {'dim': 'h'}}, 'variables.q.where': 'g[h] != gz[h]', }, @@ -1838,7 +1833,7 @@ class TestRulesDecidedWithoutData: id='where-against-a-variable', ), pytest.param( - {'sets.link': {'dims': ['g', 'h']}, 'variables.p.where': 'c > link'}, + {'dimensions.link': {'over': ['g', 'h']}, 'variables.p.where': 'c > link'}, ("compares 'c' against 'link', which is no label", 'Compare against a literal'), id='where-against-a-relation', ), @@ -1921,7 +1916,7 @@ class TestRulesDecidedWithoutData: ), pytest.param( { - 'sets.link': {'dims': ['g', 'h']}, + 'dimensions.link': {'over': ['g', 'h']}, 'maps.link': {'y': {'dim': 'h'}, 'z': {'dim': 'h'}}, 'variables.q.where': "link == 'x'", }, @@ -1986,7 +1981,7 @@ def test_the_rewrite_a_sum_grouped_by_the_whole_key_names_is_one_the_language_ta ('patch', 'where'), [ pytest.param( - {'sets.link': {'dims': ['g', 'h']}, 'maps.link': {'y': {'dim': 'h'}, 'z': {'dim': 'h'}}}, + {'dimensions.link': {'over': ['g', 'h']}, 'maps.link': {'y': {'dim': 'h'}, 'z': {'dim': 'h'}}}, "link == 'x'", id='a-relation-with-two-maps-read-bare', ), @@ -2004,7 +1999,7 @@ def test_the_map_a_bare_comparison_is_told_to_read_is_one_the_language_takes(sel _spec(**{'variables.q.where': where.replace('link', named.group(1), 1)}, **patch) -_LINKED = {'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}} +_LINKED = {'dimensions.link': {'over': ['g', 'h']}} @pytest.mark.parametrize( @@ -2012,17 +2007,18 @@ def test_the_map_a_bare_comparison_is_told_to_read_is_one_the_language_takes(sel [ pytest.param( { - 'sets.next': {'dims': {'from': 'g', 'to': 'g'}, 'missing': 'absent'}, + 'dimensions.next': {'over': {'from': 'g', 'to': 'g'}}, 'variables.x': {'dims': ['next']}, }, - "Variable 'x' names the set 'next', which has two columns over 'g', and an entry is over each " - "dimension once. Declare an alias of 'g' under dimensions:, and one column of the set over it.", + "Variable 'x' names the multi-dim 'next', which has two members over 'g', and an entry is over each " + "dimension once. Declare an alias of 'g' under dimensions:, and list it under the over: of 'next' in " + 'place of one of the two.', id='dims-of-a-relation-over-one-dimension-twice', ), pytest.param( {'constraints.c': {'dims': ['g'], 'expression': 'p <= shift(p, along=link, offset=1)'}}, - "Constraint 'c': shift(along=link) names the set 'link', and a set is not a dimension. Name " - "one of its dimensions, ['g', 'h'].", + "Constraint 'c': shift(along=link) names the multi-dim 'link', which stands for its members only in " + "dims: and over=. Name one of its members, ['g', 'h'].", id='along', ), ], @@ -2060,8 +2056,7 @@ def test_a_sum_over_a_relation_sums_over_its_dimensions(): #: `p` over the pairs of `g` and `h`, with two relations that list some of them. ON_A_RELATION: dict[str, Any] = { - 'dimensions': {'g': {}, 'h': {}}, - 'sets': {'pair': {'dims': ['g', 'h'], 'missing': 'absent'}, 'link': {'dims': ['g', 'h'], 'missing': 'absent'}}, + 'dimensions': {'g': {}, 'h': {}, 'pair': {'over': ['g', 'h']}, 'link': {'over': ['g', 'h']}}, 'parameters': {'flag': {'dims': ['g', 'h'], 'dtype': 'bool'}}, 'variables': {'p': {'dims': ['g', 'h'], 'where': 'pair'}}, } @@ -2112,11 +2107,6 @@ def test_a_sum_over_a_relation_sums_over_its_dimensions(): id='under-an-or', ), pytest.param({'variables.p.where': 'NOT pair'}, "rows of 'pair' under OR or NOT", id='under-a-not'), - pytest.param( - {'sets.pair.missing': 'refused'}, - "under missing: refused 'pair' has a row everywhere. Delete it from the `where:`.", - id='a-relation-with-a-row-everywhere', - ), ], ) def test_a_where_that_tests_the_rows_of_a_relation_is_refused(patch: dict[str, Any], message: str): @@ -2351,7 +2341,10 @@ def _cased(cases: dict[str, Any] | None = None, **entry: Any) -> dict[str, Any]: #: columns written as a mapping, and the maps of a subject. _PROBED = { **DISPATCH_MODEL, - 'sets': {'pair': {'dims': {'from_gen': 'generator', 'to_gen': 'generator'}}}, + 'dimensions': { + **DISPATCH_MODEL['dimensions'], + 'pair': {'over': {'from_gen': 'generator', 'to_gen': 'generator'}}, + }, 'maps': {'generator': {'peer': {'dim': 'generator'}}}, 'expressions': { 'price': { @@ -2628,7 +2621,6 @@ class TestAnEntryIsNamed: 'section', [ 'dimensions', - 'sets', 'parameters', 'variables', 'expressions', @@ -2641,7 +2633,6 @@ class TestAnEntryIsNamed: def test_a_name_no_expression_could_write_is_refused(self, section: str, name: str): entries: dict[str, Any] = { 'dimensions': {'dtype': 'str'}, - 'sets': {'dims': ['g', 'h']}, 'parameters': {'dims': ['g']}, 'variables': {'dims': ['g']}, 'expressions': {'expression': 'c'}, @@ -2940,7 +2931,7 @@ def test_a_values_table_under_a_ragged_where_is_not_refused(parameter, declared) pytest.param( { 'parameters.n': {'dims': ['link'], 'dtype': 'int'}, - 'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'dimensions.link': {'over': ['g', 'h']}, 'variables.p.dims': ['g', 'h'], }, id='an-int-on-a-relation', @@ -2955,7 +2946,7 @@ def test_a_bare_parameter_whose_answer_needs_the_data_loads(patch): #: A parameter over the pairs of `g` and `h` that a relation lists, with a fill. _MASKED = { - 'sets.pair': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'dimensions.pair': {'over': ['g', 'h']}, 'maps.pair.kind': {'dim': 'h', 'missing': 'absent'}, 'parameters.w': {'dims': ['g', 'h'], 'where': 'pair[kind]', 'missing': 1}, } @@ -2963,18 +2954,18 @@ def test_a_bare_parameter_whose_answer_needs_the_data_loads(patch): #: The same parameter, over the rows of the relation rather than masked by it. _IN = { - 'sets.pair': {'dims': ['g', 'h'], 'missing': 'absent'}, + 'dimensions.pair': {'over': ['g', 'h']}, 'parameters.w': {'dims': ['pair'], 'missing': 1}, } -_LINK = {'sets.link': {'dims': ['g', 'h'], 'missing': 'absent'}} +_LINK = {'dimensions.link': {'over': ['g', 'h']}} #: Each kind of entry that names a relation in `dims:`, as the label its refusals open with and `w` of that kind over `pair`. _IN_KINDS = { 'parameters': ('Parameter', _IN), 'variables': ( 'Variable', - {'sets.pair': _IN['sets.pair'], 'variables.w': {'dims': ['pair']}}, + {'dimensions.pair': _IN['dimensions.pair'], 'variables.w': {'dims': ['pair']}}, ), } @@ -3001,7 +2992,7 @@ def _in(kind: str, *relations: str, **entry: Any) -> dict[str, Any]: id='a-parameter-over-one-more-dimension', ), pytest.param( - {'sets.pair': {'dims': {'a': 'g', 'b': 'h'}, 'missing': 'absent'}}, + {'dimensions.pair': {'over': {'a': 'g', 'b': 'h'}}}, {'w': ('pair',)}, id='columns-named-apart-from-their-dimensions', ), @@ -3143,18 +3134,11 @@ def test_a_parameter_mask_that_nothing_can_evaluate_is_refused(patch, fragment): ), pytest.param( ['next'], - {'sets.next': {'dims': {'from': 'g', 'to': 'g'}, 'missing': 'absent'}}, - "{entry} 'w' names the set 'next', which has two columns over 'g', and an entry is over " + {'dimensions.next': {'over': {'from': 'g', 'to': 'g'}}}, + "{entry} 'w' names the multi-dim 'next', which has two members over 'g', and an entry is over " 'each dimension once.', id='a-relation-with-two-columns-over-one-dimension', ), - pytest.param( - ['every'], - {'sets.every': {'dims': ['g', 'h']}}, - "{entry} 'w': dims: names 'every', which under missing: refused has a row at every combination, so it " - "leaves nothing out. Write missing: absent on 'every', or name its dimensions in its place.", - id='a-relation-with-a-row-everywhere', - ), ], ) def test_a_relation_in_dims_that_says_nothing_is_refused( @@ -3162,7 +3146,7 @@ def test_a_relation_in_dims_that_says_nothing_is_refused( ): """Each refusal names the rewrite, alike on a parameter and on a variable.""" entry, _ = _IN_KINDS[kind] - declared = {key: value for key, value in patch.items() if key.startswith(('sets.', 'maps.'))} + declared = {key: value for key, value in patch.items() if key.startswith(('dimensions.', 'maps.'))} fields = {key: value for key, value in patch.items() if key not in declared} assert fragment.format(entry=entry) in _refusal(**_in(kind, *sets, **fields), **declared) @@ -3188,7 +3172,7 @@ def test_a_masked_parameter_is_not_read_as_its_fill_outside_the_mask(): assert "cases 'high' and 'low' both claim the value where cost has no row" in _refusal(**masked) indexed = { **patch, - 'sets.pair': _IN['sets.pair'], + 'dimensions.pair': _IN['dimensions.pair'], 'parameters.cost': {**patch['parameters.cost'], 'dims': ['pair']}, } assert "cases 'high' and 'low' both claim the value where cost has no row" in _refusal(**indexed), ( @@ -3224,17 +3208,54 @@ def test_a_given_parameter_has_no_missing(): @pytest.mark.parametrize( - ('declared', 'read'), + ('dims', 'over', 'in_'), + [ + pytest.param(['pair'], ('g', 'h'), ('pair',), id='one-multi-dim'), + pytest.param(['pair', 'tagged'], ('g', 'h', 'u'), ('pair', 'tagged'), id='two-joined-on-a-shared-member'), + pytest.param(['pair', 'g'], ('g', 'h'), ('pair',), id='a-member-named-again'), + ], +) +def test_a_multi_dim_in_dims_stands_for_its_members_at_its_rows(dims, over, in_): + """Two multi-dims in one `dims:` join on the member they share, at the rows both list, as TIMES' `act` does.""" + patch = { + 'dimensions.u': {}, + 'dimensions.pair': {'over': ['g', 'h']}, + 'dimensions.tagged': {'over': ['g', 'u']}, + 'variables.w': {'dims': dims}, + } + w = _spec(**patch).program.variables['w'] + assert (tuple(sorted(w.dims)), w.in_) == (over, in_), 'the members once each, and every multi-dim named' + + +@pytest.mark.parametrize( + ('patch', 'message'), [ - pytest.param({'dims': ['g', 'h']}, 'refused', id='left-out-is-refused'), - pytest.param({'dims': ['g', 'h'], 'missing': 'refused'}, 'refused', id='refused'), - pytest.param({'dims': ['g', 'h'], 'missing': 'absent'}, 'absent', id='absent'), + pytest.param( + {'multi_dims': {'pair': {'over': ['g', 'h']}}}, + "unknown key 'multi_dims' at the top level. Declare a multi-dim under dimensions:, with over:.", + id='the-key-the-load-fills', + ), + pytest.param( + {'sets': {'pair': {'dims': ['g', 'h']}}}, "unknown key 'sets' in the top level of the file", id='sets' + ), + pytest.param( + {'dimensions.pair': {'over': ['g', 'h'], 'ordered': True}}, + "unknown key 'ordered' in a multi-dim entry", + id='a-multi-dim-with-a-key-of-a-dimension', + ), ], ) -def test_a_relation_says_whether_every_combination_needs_a_row(declared, read): - """A relation takes `missing:`, and reads `refused` where it is left out.""" - spec = to_spec(varied(SMALL_MODEL, **{'sets.pair': declared})) - assert spec.program.relations['pair'].missing == read +def test_a_multi_dim_is_declared_under_dimensions_alone(patch: dict[str, Any], message: str): + """`sets:` is gone, and `multi_dims` is where the load keeps what `dimensions:` declares with `over:`.""" + assert message in _refusal(**patch) + + +def test_a_multi_dim_has_the_combinations_its_rows_list(): + """A combination with no row is not a label of a multi-dim, so it takes no `missing:` and reads `absent`.""" + spec = to_spec(varied(SMALL_MODEL, **{'dimensions.pair': {'over': ['g', 'h']}})) + assert spec.program.relations['pair'].missing == 'absent' + message = _refusal(**{'dimensions.pair': {'over': ['g', 'h'], 'missing': 'refused'}}) + assert "unknown key 'missing' in a multi-dim entry" in message @pytest.mark.parametrize( @@ -3255,26 +3276,6 @@ def test_a_map_says_whether_every_row_needs_a_label(declared, partial): @pytest.mark.parametrize( ('patch', 'fragment'), [ - pytest.param( - {'sets.link': {'dims': ['g', 'h'], 'missing': 0}}, - 'missing: 0 on a set, which takes refused or absent', - id='a-number-on-a-relation', - ), - pytest.param( - {'sets.link': {'dims': ['g', 'h'], 'missing': 'h1'}}, - 'missing: h1 on a set', - id='a-label-on-a-relation', - ), - pytest.param( - {'sets.link': {'dims': ['g', 'h'], 'missing': True}}, - 'missing: true on a set', - id='a-flag-on-a-relation', - ), - pytest.param( - {'sets.link': {'dims': ['g', 'h'], 'missing': None}}, - 'missing: null on a set says nothing', - id='null-on-a-relation', - ), pytest.param( {'maps.g.h': {'dim': 'h', 'missing': 0}}, 'maps.g.h.missing: missing: 0 on a map, which takes refused or absent', diff --git a/tests/typesetting/golden/latex.out b/tests/typesetting/golden/latex.out index b293bfa23..aaacadd7e 100644 --- a/tests/typesetting/golden/latex.out +++ b/tests/typesetting/golden/latex.out @@ -12,9 +12,9 @@ \item[{$\mathcal{B}$}] index $b$ --- \texttt{bp} \item[{$\mathcal{U}$}] index $u$ --- \texttt{bus} with $\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}$, \texttt{bus\_zone} is \texttt{absent} where the data has no row, \texttt{connection} is \texttt{absent} where the data has no row, \texttt{bus[area]} is \texttt{absent} where the data gives no label \item[{$\mathcal{U}$}] index $u'$ --- \texttt{from\_bus}, an alias of \texttt{bus} -\item[{$\mathcal{G}$}] index $g$ --- \texttt{generator} with $\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}$, \texttt{connection} is \texttt{absent} where the data has no row +\item[{$\mathcal{G}$}] index $g$ --- \texttt{generator} with $\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}$, \texttt{connection} is \texttt{absent} where the data has no row, \texttt{gen\_zone} is \texttt{absent} where the data has no row \item[{$\mathcal{S}$}] index $s$ --- \texttt{season} with $\mathrm{season}: \mathcal{T} \to \mathcal{S}$ -\item[{$\mathcal{T}$}] index $t$ --- \texttt{snapshot} (\texttt{int} coordinates) with $\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}$ +\item[{$\mathcal{T}$}] index $t$ --- \texttt{snapshot} (\texttt{int} coordinates) with $\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}$, \texttt{gen\_zone} is \texttt{absent} where the data has no row \item[{$\mathcal{E}$}] index $e$ --- \texttt{technology} with $\mathrm{technology}: \mathcal{G} \to \mathcal{E}$ \item[{$\mathcal{Z}$}] index $z$ --- \texttt{zone} (labels \texttt{north}, \texttt{south}) with $\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}$, \texttt{bus\_zone} is \texttt{absent} where the data has no row, \texttt{bus[area]} is \texttt{absent} where the data gives no label \end{description} diff --git a/tests/typesetting/golden/markdown.out b/tests/typesetting/golden/markdown.out index 98154e400..7be09c552 100644 --- a/tests/typesetting/golden/markdown.out +++ b/tests/typesetting/golden/markdown.out @@ -9,9 +9,9 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 | $`\mathcal{B}`$ | index $`b`$ — `bp` | | $`\mathcal{U}`$ | index $`u`$ — `bus` with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U}`$, `bus_zone` is `absent` where the data has no row, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | | $`\mathcal{U}`$ | index $`u'`$ — `from_bus`, an alias of `bus` | -| $`\mathcal{G}`$ | index $`g`$ — `generator` with $`\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}`$, `connection` is `absent` where the data has no row | +| $`\mathcal{G}`$ | index $`g`$ — `generator` with $`\mathrm{connection} \subseteq \mathcal{U} \times \mathcal{G},\ \mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{bus}: \mathcal{G} \to \mathcal{U},\ \mathrm{technology}: \mathcal{G} \to \mathcal{E}`$, `connection` is `absent` where the data has no row, `gen_zone` is `absent` where the data has no row | | $`\mathcal{S}`$ | index $`s`$ — `season` with $`\mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | -| $`\mathcal{T}`$ | index $`t`$ — `snapshot` (`int` coordinates) with $`\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}`$ | +| $`\mathcal{T}`$ | index $`t`$ — `snapshot` (`int` coordinates) with $`\mathrm{gen\_zone} \subseteq \mathcal{G} \times \mathcal{T},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z},\ \mathrm{rep}: \mathcal{T} \to \mathcal{T},\ \mathrm{season}: \mathcal{T} \to \mathcal{S}`$, `gen_zone` is `absent` where the data has no row | | $`\mathcal{E}`$ | index $`e`$ — `technology` with $`\mathrm{technology}: \mathcal{G} \to \mathcal{E}`$ | | $`\mathcal{Z}`$ | index $`z`$ — `zone` (labels `north`, `south`) with $`\mathrm{area}: \mathcal{U} \to \mathcal{Z},\ \mathrm{zone}: \mathcal{U} \to \mathcal{Z},\ \mathrm{bus\_zone} \subseteq \mathcal{U} \times \mathcal{Z},\ \mathrm{zone}: \mathrm{gen\_zone} \to \mathcal{Z}`$, `bus_zone` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label | diff --git a/tests/typesetting/golden/model.yaml b/tests/typesetting/golden/model.yaml index 1fe390925..f868b3085 100644 --- a/tests/typesetting/golden/model.yaml +++ b/tests/typesetting/golden/model.yaml @@ -21,11 +21,9 @@ dimensions: season: { dtype: str } technology: { dtype: str } bp: { dtype: int, ordered: true } # the breakpoints every curve below runs through - -sets: - gen_zone: { dims: [generator, snapshot] } # a relation over two dimensions, with a map: a call sums one away and joins on the other - connection: { dims: [generator, bus], missing: absent } # a relation with no map: many-to-many, read only by sum with both ends named - bus_zone: { dims: [bus, zone], missing: absent } # the pairs that exist + gen_zone: { over: [generator, snapshot] } # a multi-dim over two dimensions, with a map: a call sums one away and joins on the other + connection: { over: [generator, bus] } # a multi-dim with no map: many-to-many, read only by sum with both ends named + bus_zone: { over: [bus, zone] } # the pairs that exist maps: generator: { bus: { dim: bus }, technology: { dim: technology } } # two maps of one subject, read to both at once @@ -45,7 +43,7 @@ parameters: is_flexible: { dims: [generator], dtype: bool, missing: false } zone_cap: { dims: [zone] } tech_cap: { dims: [bus, technology], missing: absent } - link_eff: { dims: [connection], missing: 1 } # over the pairs a relation lists; a `where:` may not test them + link_eff: { dims: [connection], missing: 1 } # over the pairs a multi-dim lists; a `where:` may not test them zone_share: { dims: [bus_zone], missing: 1 } # over the pairs that exist: data refused outside them, with no `where:` of its own susceptance: { dims: [from_bus, bus], missing: 0 } # over one set twice, through its alias area_weight: { dims: [bus], where: "bus[area]" } # a map's label as a condition: the buses that have an area diff --git a/tests/typesetting/golden/typst.out b/tests/typesetting/golden/typst.out index 99bc3ea02..068d77f4d 100644 --- a/tests/typesetting/golden/typst.out +++ b/tests/typesetting/golden/typst.out @@ -7,9 +7,9 @@ every character a notation escapes, set as text: link\_to, 100% & \#1 costs \$5 / $cal(B)$: index $b$ --- `bp` / $cal(U)$: index $u$ --- `bus` with $upright("area"): cal(U) arrow.r cal(Z), upright("zone"): cal(U) arrow.r cal(Z), upright("bus_zone") subset.eq cal(U) times cal(Z), upright("connection") subset.eq cal(U) times cal(G), upright("bus"): cal(G) arrow.r cal(U)$, `bus_zone` is `absent` where the data has no row, `connection` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label / $cal(U)$: index $u'$ --- `from_bus`, an alias of `bus` -/ $cal(G)$: index $g$ --- `generator` with $upright("connection") subset.eq cal(U) times cal(G), upright("gen_zone") subset.eq cal(G) times cal(T), upright("zone"): upright("gen_zone") arrow.r cal(Z), upright("bus"): cal(G) arrow.r cal(U), upright("technology"): cal(G) arrow.r cal(E)$, `connection` is `absent` where the data has no row +/ $cal(G)$: index $g$ --- `generator` with $upright("connection") subset.eq cal(U) times cal(G), upright("gen_zone") subset.eq cal(G) times cal(T), upright("zone"): upright("gen_zone") arrow.r cal(Z), upright("bus"): cal(G) arrow.r cal(U), upright("technology"): cal(G) arrow.r cal(E)$, `connection` is `absent` where the data has no row, `gen_zone` is `absent` where the data has no row / $cal(S)$: index $s$ --- `season` with $upright("season"): cal(T) arrow.r cal(S)$ -/ $cal(T)$: index $t$ --- `snapshot` (`int` coordinates) with $upright("gen_zone") subset.eq cal(G) times cal(T), upright("zone"): upright("gen_zone") arrow.r cal(Z), upright("rep"): cal(T) arrow.r cal(T), upright("season"): cal(T) arrow.r cal(S)$ +/ $cal(T)$: index $t$ --- `snapshot` (`int` coordinates) with $upright("gen_zone") subset.eq cal(G) times cal(T), upright("zone"): upright("gen_zone") arrow.r cal(Z), upright("rep"): cal(T) arrow.r cal(T), upright("season"): cal(T) arrow.r cal(S)$, `gen_zone` is `absent` where the data has no row / $cal(E)$: index $e$ --- `technology` with $upright("technology"): cal(G) arrow.r cal(E)$ / $cal(Z)$: index $z$ --- `zone` (labels `north`, `south`) with $upright("area"): cal(U) arrow.r cal(Z), upright("zone"): cal(U) arrow.r cal(Z), upright("bus_zone") subset.eq cal(U) times cal(Z), upright("zone"): upright("gen_zone") arrow.r cal(Z)$, `bus_zone` is `absent` where the data has no row, `bus[area]` is `absent` where the data gives no label diff --git a/tests/typesetting/test_walk.py b/tests/typesetting/test_walk.py index 64ae4a6ea..c946a43d3 100644 --- a/tests/typesetting/test_walk.py +++ b/tests/typesetting/test_walk.py @@ -668,8 +668,8 @@ def test_every_operator_probe_renders(path, name: FormatName, fmt: Format): 'zone': {'dtype': 'str'}, 'bus': {'dtype': 'str'}, 'technology': {'dtype': 'str'}, + 'gen_zone': {'over': ['generator', 'snapshot']}, }, - 'sets': {'gen_zone': {'dims': ['generator', 'snapshot']}}, 'maps': { 'gen_zone': {'zone': {'dim': 'zone'}}, 'generator': {'bus': {'dim': 'bus'}, 'technology': {'dim': 'technology'}}, diff --git a/tools/notation.py b/tools/notation.py index b7cbaea80..7dff9d589 100644 --- a/tools/notation.py +++ b/tools/notation.py @@ -298,11 +298,11 @@ def legend() -> str: #: What the legend is made of. No equation comes from these, so they are shown #: once, together, above the tables they turn into. -DECLARED = ('dimensions', 'sets', 'maps', 'parameters') +DECLARED = ('dimensions', 'maps', 'parameters') def preamble(text: str) -> str: - """The fixture's ``dimensions``/``relations``/``maps``/``parameters`` sections, verbatim.""" + """The fixture's ``dimensions``/``maps``/``parameters`` sections, verbatim.""" sections = [] for name in DECLARED: body = text[text.index(f'\n{name}:') + 1 :] @@ -316,7 +316,7 @@ def block() -> str: rendered = to_markdown(MODEL, numbered=False) parts = [ '### Legend', - 'A dimension, a relation, a map and a parameter declare no equation; what they ' + 'A dimension, a multi-dim, a map and a parameter declare no equation; what they ' 'print is the legend every spec opens with.', f'```yaml\n{preamble(MODEL.read_text())}\n```', legend(), diff --git a/tools/pypsa_split.py b/tools/pypsa_split.py index ec1dfb9d7..3506bb29c 100644 --- a/tools/pypsa_split.py +++ b/tools/pypsa_split.py @@ -54,7 +54,6 @@ SOURCE = Path('examples/pypsa.yaml') SECTIONS = ( 'dimensions', - 'sets', 'maps', 'parameters', 'variables', @@ -64,7 +63,7 @@ 'constraints', 'assumptions', ) -FRAME = ('dimensions', 'sets', 'maps') +FRAME = ('dimensions', 'maps') IDENT = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') READ = re.compile(r'(? Date: Tue, 17 Nov 2026 19:21:52 +0000 Subject: [PATCH 7/9] docs(changelog): name the comparison PR Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1Y7i5TWDbh9SKx86i --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 488ec5852..3f2a74b64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,9 @@ it releases that version ([RELEASING.md](https://github.com/energy-models/mathsp ## Upcoming version -- feat(language)!: an entry or a sum may range over a set ([#1018](https://github.com/energy-models/mathspec/pull/1018)) - - Breaks a file: `relations:` is `sets:`. A set named in the `dims:` of a parameter, a variable or a constraint, or in `over=`, stands for its dimensions at the combinations it lists. - - Breaks a file: the `where:` of a parameter, a variable or a constraint no longer takes a set's bare name. Name the set in `dims:`. +- feat(language)!: a multi-dim is a dimension with over: ([#8](https://github.com/fluxopt/mathspec/pull/8)) + - Breaks a file: `relations:` goes. Declare each relation under `dimensions:` with `over:` in place of `dims:`, and delete its `missing:`. A multi-dim named in the `dims:` of a parameter, a variable or a constraint, or in `over=`, stands for its members at the combinations it lists. + - Breaks a file: the `where:` of a parameter, a variable or a constraint no longer takes a multi-dim's bare name. Name the multi-dim in `dims:`. - feat(language): a dimension may alias another's labels ([#1020](https://github.com/energy-models/mathspec/pull/1020)) - feat(language)!: maps give labels, relations give rows ([#1015](https://github.com/energy-models/mathspec/pull/1015)) - Breaks a file: `relations:` takes `dims:`, `missing:` and `description` only. Each value column becomes a map under `maps:`, keyed by its subject, with `dim:` naming its dimension and always written, and is read as `subject[map]`, such as `generator[bus]`. From 72385d2ff7fabdfc354cb320d3f67025c668eb83 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 19:53:25 +0000 Subject: [PATCH 8/9] feat(language): dims: is the domain of every entry A multi-dim in the `dims:` of a named expression or a `given:` entry now stands for its members, as it does on a parameter, a variable and a constraint, so every rule that compares `dims:` compares the members. A multi-dim may have one member, a subset of that dimension. Its bare name is a test in a constraint's `where:`, a named condition and a `when:`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA1Y7i5TWDbh9SKx86i --- CHANGELOG.md | 2 +- docs/examples/candidates.md | 148 ++++++++++++++++++ docs/examples/index.md | 4 + docs/reference/composition.md | 4 +- docs/reference/language/named.md | 4 +- .../parameters-variables-constraints.md | 27 +++- docs/reference/language/relations.md | 20 ++- examples/candidates.yaml | 51 ++++++ mkdocs.yml | 1 + src/mathspec/_lowering.py | 13 +- src/mathspec/_typesetting/walk.py | 5 +- src/mathspec/_validation.py | 8 +- src/mathspec/spec.py | 28 +++- tests/test_validation.py | 111 +++++++++++-- tools/gallery.py | 1 + 15 files changed, 386 insertions(+), 41 deletions(-) create mode 100644 docs/examples/candidates.md create mode 100644 examples/candidates.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f2a74b64..0d7827989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ it releases that version ([RELEASING.md](https://github.com/energy-models/mathsp - feat(language)!: a multi-dim is a dimension with over: ([#8](https://github.com/fluxopt/mathspec/pull/8)) - Breaks a file: `relations:` goes. Declare each relation under `dimensions:` with `over:` in place of `dims:`, and delete its `missing:`. A multi-dim named in the `dims:` of a parameter, a variable or a constraint, or in `over=`, stands for its members at the combinations it lists. - - Breaks a file: the `where:` of a parameter, a variable or a constraint no longer takes a multi-dim's bare name. Name the multi-dim in `dims:`. + - Breaks a file: the `where:` of a parameter or a variable no longer takes a multi-dim's bare name. Name the multi-dim in `dims:`. - feat(language): a dimension may alias another's labels ([#1020](https://github.com/energy-models/mathspec/pull/1020)) - feat(language)!: maps give labels, relations give rows ([#1015](https://github.com/energy-models/mathspec/pull/1015)) - Breaks a file: `relations:` takes `dims:`, `missing:` and `description` only. Each value column becomes a map under `maps:`, keyed by its subject, with `dim:` naming its dimension and always written, and is read as `subject[map]`, such as `generator[bus]`. diff --git a/docs/examples/candidates.md b/docs/examples/candidates.md new file mode 100644 index 000000000..d006d839e --- /dev/null +++ b/docs/examples/candidates.md @@ -0,0 +1,148 @@ + + +# A multi-dim as a test and a subset + +Two uses of a [multi-dim](../reference/language/relations.md#multi-dims) that +`dims:` cannot state: + +- `port_cost` is over every port, and its case `new` tests `candidate`. The + entry covers both branches, so `candidate` cannot stand in its `dims:`. +- `extendable` is over the one member `unit`, so it lists some units. + `build`, `unit_limit` and `build_limit` are over `[extendable, period]`, so + they have coordinates only for the units it lists. + + +```yaml +description: >- + Units whose candidate ports cost more to run, and the units that may be + extended: a multi-dim as a condition, and a multi-dim over one member. + +dimensions: + unit: { description: a converter } + port: { description: an output of a unit } + period: { dtype: int, ordered: true, description: a step of the horizon } + candidate: { over: [unit, port], description: the ports a unit may add } + extendable: { over: [unit], description: the units whose capacity may grow } + +parameters: + run_cost: { dims: [unit, port, period], description: the cost of a unit of flow through a port that exists } + new_cost: { dims: [unit, port, period], description: the cost of a unit of flow through a candidate port } + capacity: { dims: [unit], description: the most a unit produces in a period } + max_build: { dims: [extendable], description: the most a unit's capacity may grow } + load: { dims: [period], description: demand to be met } + +variables: + flow: { dims: [unit, port, period], bounds: { lower: 0 }, description: what a port delivers } + build: { dims: [extendable, period], bounds: { lower: 0 }, description: the capacity added to a unit } + +expressions: + port_cost: + description: a candidate port costs its own price, every other port the running price + dims: [unit, port, period] + cases: + new: { when: candidate, expression: new_cost * flow } + otherwise: run_cost * flow + +constraints: + unit_limit: + description: the ports of a unit deliver no more than its capacity and what it adds + dims: [extendable, period] + expression: sum(flow, over=port) <= capacity + build + build_limit: + description: a unit adds no more than its limit + dims: [extendable, period] + expression: build <= max_build + balance: + description: the ports meet the load + dims: [period] + expression: sum(flow, over=[unit, port]) >= load + +objective: + sense: minimize + expression: sum(port_cost) + sum(build) +``` + +Units whose candidate ports cost more to run, and the units that may be extended: a multi-dim as a condition, and a multi-dim over one member. + +#### Sets + +| Symbol | Meaning | +|---|---| +| $`\mathcal{P}`$ | index $`p`$ — `period` — a step of the horizon | +| $`\mathcal{O}`$ | index $`o`$ — `port` with $`\mathrm{candidate} \subseteq \mathcal{O} \times \mathcal{U}`$, `candidate` is `absent` where the data has no row — an output of a unit | +| $`\mathcal{U}`$ | index $`u`$ — `unit` with $`\mathrm{candidate} \subseteq \mathcal{O} \times \mathcal{U},\ \mathrm{extendable} \subseteq \mathcal{U}`$, `candidate` is `absent` where the data has no row, `extendable` is `absent` where the data has no row — a converter | + +#### Parameters + +| Symbol | Meaning | +|---|---| +| $`\mathrm{capacity}`$ | `capacity` over $`\mathcal{U}`$ — the most a unit produces in a period | +| $`\mathrm{load}`$ | `load` over $`\mathcal{P}`$ — demand to be met | +| $`\mathrm{max\_build}`$ | `max_build` over $`u \in \mathrm{extendable}`$ — the most a unit's capacity may grow | +| $`\mathrm{new\_cost}`$ | `new_cost` over $`\mathcal{P} \times \mathcal{O} \times \mathcal{U}`$ — the cost of a unit of flow through a candidate port | +| $`\mathrm{run\_cost}`$ | `run_cost` over $`\mathcal{P} \times \mathcal{O} \times \mathcal{U}`$ — the cost of a unit of flow through a port that exists | + +#### Variables + +| Symbol | Meaning | +|---|---| +| $`\mathit{build}`$ | `build` over $`u \in \mathrm{extendable},\ p \in \mathcal{P}`$ — the capacity added to a unit | +| $`\mathit{flow}`$ | `flow` over $`\mathcal{P} \times \mathcal{O} \times \mathcal{U}`$ — what a port delivers | + +#### Definitions + +| Symbol | Meaning | +|---|---| +| $`\mathit{port\_cost}`$ | `port_cost` over $`\mathcal{P} \times \mathcal{O} \times \mathcal{U}`$ — a candidate port costs its own price, every other port the running price | + +#### Variable domains + +**`build`** + +```math +\mathit{build}_{p,u} \ge 0 \qquad \forall\, u \in \mathrm{extendable},\ p \in \mathcal{P} +``` + +**`flow`** + +```math +\mathit{flow}_{p,o,u} \ge 0 \qquad \forall\, p \in \mathcal{P},\ o \in \mathcal{O},\ u \in \mathcal{U} +``` + +#### Definitions + +**`port_cost`** + +```math +\mathit{port\_cost}_{p,o,u} = \begin{cases} \mathrm{new\_cost}_{p,o,u} \cdot \mathit{flow}_{p,o,u} & \text{if } \left( o,\ u \right) \in \mathrm{candidate} \\ \mathrm{run\_cost}_{p,o,u} \cdot \mathit{flow}_{p,o,u} & \text{otherwise} \end{cases} \qquad \forall\, p \in \mathcal{P},\ o \in \mathcal{O},\ u \in \mathcal{U} +``` + +#### Objective + +```math +\min \sum_{p \in \mathcal{P},\ o \in \mathcal{O},\ u \in \mathcal{U}} \mathit{port\_cost}_{p,o,u} + \sum_{p \in \mathcal{P},\ u \in \mathcal{U}} \mathit{build}_{p,u} +``` + +#### Subject to + +**`balance`** + +```math +\sum_{o \in \mathcal{O},\ u \in \mathcal{U}} \mathit{flow}_{p,o,u} \ge \mathrm{load}_{p} \qquad \forall\, p \in \mathcal{P} +``` + +**`build_limit`** + +```math +\mathit{build}_{p,u} \le \mathrm{max\_build}_{u} \qquad \forall\, p \in \mathcal{P},\ u \in \mathcal{U} \,:\, u \in \mathrm{extendable} +``` + +**`unit_limit`** + +```math +\sum_{o \in \mathcal{O}} \mathit{flow}_{p,o,u} \le \mathrm{capacity}_{u} + \mathit{build}_{p,u} \qquad \forall\, p \in \mathcal{P},\ u \in \mathcal{U} \,:\, u \in \mathrm{extendable} +``` + diff --git a/docs/examples/index.md b/docs/examples/index.md index 07c73a881..bfa5dd105 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -30,6 +30,10 @@ like yours, and what each line of the file means. Units on the ports they have and in the periods they run: a parameter per unit, a sum over the ports, a step in time and a sum over every port. [Open →](sparse.md) +- **A multi-dim as a test and a subset** + A case tests whether a port is a candidate, and a multi-dim over one + member lists the units that may grow. + [Open →](candidates.md) - **One construct per spec** Each form of the four operators and `dual` has its smallest file, beside its typeset equation. diff --git a/docs/reference/composition.md b/docs/reference/composition.md index acf5f552b..d528861ac 100644 --- a/docs/reference/composition.md +++ b/docs/reference/composition.md @@ -278,7 +278,9 @@ A given entry is a name that a file lists under [`given:`](language/parameters-variables-constraints.md#given). `merge` folds each given entry into the entry of the fragment that declares the name. The fragment that lists the name may state less than the file that declares it, -such as the dimensions of a variable with no `domain`. +such as the dimensions of a variable with no `domain`. A multi-dim in the +`dims:` of either counts as its members +([`dims` is the domain](language/parameters-variables-constraints.md#dims-is-the-domain)). | The given entry | What `merge` does | | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/reference/language/named.md b/docs/reference/language/named.md index 7be6c9547..6ef45b594 100644 --- a/docs/reference/language/named.md +++ b/docs/reference/language/named.md @@ -412,7 +412,9 @@ at load: nearest name, such as `Did you mean 'injection'?`. - A term does not use the sum that it adds to, directly or through another name. -- A term is indexed only over dimensions that the given entry lists. +- A term is indexed only over dimensions that the given entry lists. A + multi-dim in either `dims:` counts as its members + ([`dims` is the domain](parameters-variables-constraints.md#dims-is-the-domain)). - Like every expression that uses the sum, a term contains a product of at most [two variables](expressions.md#where-a-product-of-two-variables-is-allowed). diff --git a/docs/reference/language/parameters-variables-constraints.md b/docs/reference/language/parameters-variables-constraints.md index a440ce97f..8c8941fef 100644 --- a/docs/reference/language/parameters-variables-constraints.md +++ b/docs/reference/language/parameters-variables-constraints.md @@ -13,6 +13,20 @@ lists the keys of those five sections, in the order of the also takes `description:`, free text that the [typeset](../typeset.md#descriptions) legend lists beside the entry. +## `dims` is the domain + +The `dims:` of an entry is its domain: the dimensions and the +[multi-dims](relations.md#multi-dims) it is over. A multi-dim stands for its +members at the rows it lists. Two items join on the member names they share, +so `dims: [connection, period]` with `connection: { over: [unit, port] }` is +over `unit`, `port` and `period` at the pairs that `connection` lists. An +[alias](dimensions.md#alias) keeps two copies of one dimension apart. + +Every rule that compares `dims:` compares those members: the two sides of a +constraint, a `given:` entry against the entry it reads, and an +[`adds_to:`](named.md#adds_to) term against its sum. So a constraint over +`[connection, period]` whose expression is over `[unit, port, period]` loads. + ## `given` `given:` lists what this file uses and another file declares. It takes five keys: data under `parameters:`, named conditions under @@ -486,8 +500,8 @@ The typeset legend lists the condition beside the dimensions, after ### A multi-dim in `dims` -A [multi-dim](relations.md) named in `dims:` stands for its dimensions, at the -combinations it lists. The parameter has coordinates only there, and a tool +A [multi-dim](relations.md) named in `dims:` stands for its members, at the +combinations it lists ([`dims` is the domain](#dims-is-the-domain)). The parameter has coordinates only there, and a tool refuses a row of the data anywhere else. A [`where:`](#where) drops that row instead. In this example, `efficiency` and `p_nom_max` have values only at the pairs that `connection` lists: @@ -783,7 +797,14 @@ Constraint 'power_balance': the expression carries dims ['generator'] that are n ``` mathspec lists the extra dimension and the `dims`, and suggests two -rewrites. +rewrites. A multi-dim in `dims:` counts as its members, so +`dims: [connection, period]` equals an expression over `[unit, port, period]` +([`dims` is the domain](#dims-is-the-domain)). + +The bare name of a multi-dim may stand in the `where:` of a constraint, as a +test of whether a row is in it. A constraint's rows follow its operands, so +the name is a test like any other there. In the `where:` of a parameter or a +variable it is refused, and the rewrite names it in `dims:`. At least one side of the comparison must have a variable, so at load mathspec refuses a comparison between numbers and parameters alone: diff --git a/docs/reference/language/relations.md b/docs/reference/language/relations.md index 3fcdca2f0..0f344e1dd 100644 --- a/docs/reference/language/relations.md +++ b/docs/reference/language/relations.md @@ -35,7 +35,10 @@ $`\sum_{(g, b) \in \mathrm{connection}}`$. So its name may stand in the members at the combinations it lists ([a multi-dim in `dims`](parameters-variables-constraints.md#a-multi-dim-in-dims)). Two multi-dims in one `dims:` join on the members they share, at the rows that -both list, so TIMES writes `act: { dims: [prc_ts, rtp_vintyr] }`. +both list, so TIMES writes `act: { dims: [prc_ts, rtp_vintyr] }`. An +[alias](dimensions.md#alias) keeps two copies of one dimension apart. So +`dims:` is the domain of the entry +([`dims` is the domain](parameters-variables-constraints.md#dims-is-the-domain)). | Key | What it sets | Default | | ------------- | ---------------------------------------------------- | ------- | @@ -56,13 +59,16 @@ mathspec prints: Multi-dim 'zp' names member 'period' after dimension 'period', but the member is over 'zone'. Rename the member after what it holds. ``` -A multi-dim with one member is refused, because one member holds labels of one -dimension, and a map or a condition states that. With -`r: { over: [generator] }`, mathspec prints: +A multi-dim over one member lists some labels of that dimension, so +`extendable: { over: [unit] }` is a subset of `unit`, and `dims: [extendable, +period]` confines an entry to it. A multi-dim with no member is refused. -```text -Multi-dim 'r' has one member under 'over:', and a multi-dim needs at least two. To give each generator a label, declare a map under maps: generator:. To mark some of them, declare a bool parameter or a condition. -``` +The bare name of a multi-dim is a condition: true where it has a row. It +stands in a `when:` of [`cases:`](named.md#cases), in a named condition and in +the `where:` of a constraint, where the entry covers rows both in and out of +the multi-dim. In the `where:` of a parameter or a variable it is refused, +because `dims:` states those rows +([example](../../examples/candidates.md)). A list that names one dimension twice is refused, and the rewrite is an [alias](dimensions.md#alias). With `link: { over: [bus, bus] }`, mathspec diff --git a/examples/candidates.yaml b/examples/candidates.yaml new file mode 100644 index 000000000..291da6e64 --- /dev/null +++ b/examples/candidates.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: mathspec Contributors +# +# SPDX-License-Identifier: MIT + +description: >- + Units whose candidate ports cost more to run, and the units that may be + extended: a multi-dim as a condition, and a multi-dim over one member. + +dimensions: + unit: { description: a converter } + port: { description: an output of a unit } + period: { dtype: int, ordered: true, description: a step of the horizon } + candidate: { over: [unit, port], description: the ports a unit may add } + extendable: { over: [unit], description: the units whose capacity may grow } + +parameters: + run_cost: { dims: [unit, port, period], description: the cost of a unit of flow through a port that exists } + new_cost: { dims: [unit, port, period], description: the cost of a unit of flow through a candidate port } + capacity: { dims: [unit], description: the most a unit produces in a period } + max_build: { dims: [extendable], description: the most a unit's capacity may grow } + load: { dims: [period], description: demand to be met } + +variables: + flow: { dims: [unit, port, period], bounds: { lower: 0 }, description: what a port delivers } + build: { dims: [extendable, period], bounds: { lower: 0 }, description: the capacity added to a unit } + +expressions: + port_cost: + description: a candidate port costs its own price, every other port the running price + dims: [unit, port, period] + cases: + new: { when: candidate, expression: new_cost * flow } + otherwise: run_cost * flow + +constraints: + unit_limit: + description: the ports of a unit deliver no more than its capacity and what it adds + dims: [extendable, period] + expression: sum(flow, over=port) <= capacity + build + build_limit: + description: a unit adds no more than its limit + dims: [extendable, period] + expression: build <= max_build + balance: + description: the ports meet the load + dims: [period] + expression: sum(flow, over=[unit, port]) >= load + +objective: + sense: minimize + expression: sum(port_cost) + sum(build) diff --git a/mkdocs.yml b/mkdocs.yml index dd6f992f3..2dff8af70 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,6 +75,7 @@ nav: - Unit commitment: examples/commitment.md - Units with ports: examples/connection.md - A sparse set, read four ways: examples/sparse.md + - A multi-dim as a test and a subset: examples/candidates.md - One construct per spec: examples/operators.md - A curve by convex combination: examples/piecewise.md - A curve that is not convex: examples/piecewise_adjacency.md diff --git a/src/mathspec/_lowering.py b/src/mathspec/_lowering.py index 50aaa3419..d53e8686a 100644 --- a/src/mathspec/_lowering.py +++ b/src/mathspec/_lowering.py @@ -178,7 +178,7 @@ def lower(spec: Spec) -> Program: where = resolve_condition_text(cdef.where, ns, context, errors) if (sides := resolve_constraint_text(cdef.expression, ns, context, errors)) is not None: lhs, sense, rhs = sides - mask = _index_mask(cname, cdef.in_, cdef.where, where, ns, context, errors) + mask = _index_mask(cname, cdef.in_, cdef.where, where, ns, context, errors, rows_follow=True) constraints[cname] = Constraint(cdef.dims, lhs, sense, rhs, mask, description=cdef.description) objective = None @@ -454,17 +454,24 @@ def _index_mask( ns: Namespace, context: str, errors: list[str], + *, + rows_follow: bool = False, ) -> Mask | None: """The key of each relation in *in_* and the resolved *where* as one mask, or ``None`` where neither limits anything. A refusal is appended where the relations already decide some of the ``where:``: inside them each bare relation name is true, so reading one there says nothing. A map read as ``subject[map]`` is a test, - and stays. + and stays. A constraint's rows follow its operands, so with *rows_follow* + a bare name of a multi-dim outside ``dims:`` is a test like any other. """ if where is not None and in_ and (written := Mask(where)).within(in_) != written: errors.append(_decided_by_index_set(context, name, text, in_, written.within(in_).root)) - elif where is not None and (problem := _rows_tested(context, text, Mask(where), in_, ns)) is not None: + elif ( + where is not None + and not rows_follow + and (problem := _rows_tested(context, text, Mask(where), in_, ns)) is not None + ): errors.append(problem) rows: list[Condition] = [ RelationDefined(r, tuple(sorted(ns.relations[r].dim(k) for k in ns.relations[r].key))) for r in in_ diff --git a/src/mathspec/_typesetting/walk.py b/src/mathspec/_typesetting/walk.py index 91c55dcb9..9ac182883 100644 --- a/src/mathspec/_typesetting/walk.py +++ b/src/mathspec/_typesetting/walk.py @@ -286,14 +286,15 @@ def _relation_row(self, name: str, at: Mapping[str, str], column: str | None = N """That relation *name* has a row at the key *at* fixes, or with *column*, that the map gives the row a label. A relation is a set of rows, so the row is written out: - ``(p, u) ∈ connection``. A map is a function, so the claim is that it + ``(p, u) ∈ connection``, and a row of one member as ``u ∈ extendable``. A map is a function, so the claim is that it is defined there: ``line(o) is defined``. """ lk = self.program.relations[name] key = self.format.joined([at[k] for k in lk.key], '') if column is not None: return f'{self.format.apply(self.format.upright(column), key)} {self.format.prose(" is defined")}' - return f'{self.format.parenthesise(key)} {self._op("in")} {self.format.upright(name)}' + row = key if len(lk.key) == 1 else self.format.parenthesise(key) + return f'{row} {self._op("in")} {self.format.upright(name)}' def _frame_key(self, name: str, ctx: _Context) -> dict[str, str]: """Table *name*'s key roles at the frame's own indices of their dimensions.""" diff --git a/src/mathspec/_validation.py b/src/mathspec/_validation.py index 5bfcd0e09..a2c348752 100644 --- a/src/mathspec/_validation.py +++ b/src/mathspec/_validation.py @@ -212,12 +212,10 @@ def _frame_dimensions(spec: Spec) -> Iterator[str]: def _relation_targets(spec: Spec) -> Iterator[str]: """A relation has at least two columns, over declared dimensions, each named once.""" for lname, lk in spec.multi_dims.items(): - if len(lk.pairs) < 2: - dim = lk.pairs[0][1] if lk.pairs else '' + if not lk.pairs: yield ( - f"Multi-dim '{lname}' has {'one member' if lk.pairs else 'no member'} under 'over:', and a " - f'multi-dim needs at least two. To give each {dim} a label, declare a map under maps: {dim}:. To ' - f'mark some of them, declare a bool parameter or a condition.' + f"Multi-dim '{lname}' has no member under 'over:', and a multi-dim needs at least one. List the " + f'dimensions whose combinations it lists.' ) if not isinstance(lk.over, Mapping): yield from ( diff --git a/src/mathspec/spec.py b/src/mathspec/spec.py index a249be720..53928e87f 100644 --- a/src/mathspec/spec.py +++ b/src/mathspec/spec.py @@ -590,6 +590,8 @@ class GivenParameterSpec(_GivenEntrySpec): _label: ClassVar[str] = 'a given parameter entry' dims: Dims + in_: Annotated[Dims, SkipJsonSchema()] = () + """The multi-dims named in ``dims:``, which the load writes from it; ``dims`` holds their members.""" dtype: ParameterDtype = 'float' description: str | None = None @@ -604,6 +606,8 @@ class GivenVariableSpec(_GivenEntrySpec): _label: ClassVar[str] = 'a given variable entry' dims: Dims + in_: Annotated[Dims, SkipJsonSchema()] = () + """The multi-dims named in ``dims:``, which the load writes from it; ``dims`` holds their members.""" domain: VariableDomain = 'continuous' description: str | None = None @@ -619,6 +623,8 @@ class GivenConstraintSpec(_GivenEntrySpec): _label: ClassVar[str] = 'a given constraint entry' dims: Dims + in_: Annotated[Dims, SkipJsonSchema()] = () + """The multi-dims named in ``dims:``, which the load writes from it; ``dims`` holds their members.""" description: str | None = None @@ -640,6 +646,8 @@ class GivenExpressionSpec(_GivenEntrySpec): _label: ClassVar[str] = 'a given expression entry' dims: Dims + in_: Annotated[Dims, SkipJsonSchema()] = () + """The multi-dims named in ``dims:``, which the load writes from it; ``dims`` holds their members.""" description: str | None = None @@ -793,6 +801,8 @@ class ExpressionSpec(_StrictSpec): expression: Expression | None = None dims: Dims | None = None + in_: Annotated[Dims, SkipJsonSchema()] = () + """The multi-dims named in ``dims:``, which the load writes from it; ``dims`` holds their members.""" """The dimensions of the expression, sorted by name. An entry with ``cases:`` requires it, and a plain entry without it takes the dimensions of its body. """ @@ -857,6 +867,8 @@ def _as_written(self) -> str | dict[str, object]: written['description'] = self.description if self.adds_to is not None: written['adds_to'] = self.adds_to + if self.in_: + written['in_'] = list(self.in_) return written @@ -1450,7 +1462,7 @@ def _lower(self) -> Spec: return self -_SET_SECTIONS = ('parameters', 'variables', 'constraints') +_SET_SECTIONS = ('parameters', 'variables', 'constraints', 'expressions') def _members(relation: object) -> tuple[str, ...] | None: @@ -1508,6 +1520,13 @@ def _over_sets(data: dict[str, object]) -> dict[str, object]: for section in _SET_SECTIONS: if isinstance(entries := data.get(section), Mapping): out[section] = {name: _entry_over_sets(entry, sets, section) for name, entry in entries.items()} + if isinstance(given := data.get('given'), Mapping): + out['given'] = { + kind: {name: _entry_over_sets(e, sets, kind) for name, e in entries.items()} + if kind in _SET_SECTIONS and isinstance(entries, Mapping) + else entries + for kind, entries in given.items() + } return out @@ -1533,6 +1552,13 @@ def _sets_written_back(written: dict[str, object], relations: Mapping[str, Multi for section in _SET_SECTIONS: if isinstance(entries := written.get(section), Mapping): out[section] = {name: _entry_written_back(entry, relations) for name, entry in entries.items()} + if isinstance(given := written.get('given'), Mapping): + out['given'] = { + kind: {name: _entry_written_back(e, relations) for name, e in entries.items()} + if kind in _SET_SECTIONS and isinstance(entries, Mapping) + else entries + for kind, entries in given.items() + } return out diff --git a/tests/test_validation.py b/tests/test_validation.py index c0ec8e99c..4c1bfcbd3 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1439,14 +1439,6 @@ class TestRulesDecidedWithoutData: ("unknown key 'dtype' in a multi-dim entry. Valid keys: description, over.",), id='relation-with-a-dtype-of-its-own', ), - pytest.param( - {'dimensions.link': {'over': ['g']}}, - ( - "Multi-dim 'link' has one member under 'over:', and a multi-dim needs at least two", - 'To give each g a label, declare a map under maps: g:.', - ), - id='relation-with-one-column', - ), pytest.param( {'dimensions.link': {'over': ['g', 'z']}}, ("Multi-dim 'link' references undeclared dimension 'z'",), @@ -1464,7 +1456,7 @@ class TestRulesDecidedWithoutData: ), pytest.param( {'dimensions.link': {'over': []}}, - ("Multi-dim 'link' has no member under 'over:', and a multi-dim needs at least two",), + ("Multi-dim 'link' has no member under 'over:', and a multi-dim needs at least one",), id='relation-with-no-columns', ), pytest.param( @@ -2092,14 +2084,6 @@ def test_a_sum_over_a_relation_sums_over_its_dimensions(): "Parameter 'cap': `where: pair` tests the rows of 'pair', which dims: states.", id='a-parameter', ), - pytest.param( - { - 'variables.p': {'dims': ['g', 'h']}, - 'constraints.c': {'dims': ['g', 'h'], 'where': 'pair', 'expression': 'p >= 0'}, - }, - "Constraint 'c': `where: pair` tests the rows of 'pair', which dims: states.", - id='a-constraint', - ), pytest.param( {'variables.p.where': 'pair OR flag'}, "tests the rows of 'pair' under OR or NOT, which dims: cannot state. Declare those rows as a bool " @@ -2133,6 +2117,41 @@ def test_a_where_that_reads_a_map_or_a_value_loads(patch: dict[str, Any]): to_spec(varied(ON_A_RELATION, **patch)) +@pytest.mark.parametrize( + 'patch', + [ + pytest.param( + {'constraints.c': {'dims': ['g', 'h'], 'where': 'pair', 'expression': 'p >= 0'}}, id='a-constraint-where' + ), + pytest.param( + { + 'conditions.paired': 'pair AND flag', + 'constraints.c': {'dims': ['g', 'h'], 'where': 'paired', 'expression': 'p >= 0'}, + }, + id='a-named-condition-in-a-constraint-where', + ), + pytest.param( + { + 'parameters.cost': {'dims': ['g', 'h']}, + 'expressions.price': { + 'dims': ['g', 'h'], + 'cases': {'paired': {'when': 'pair', 'expression': 'cost'}}, + 'otherwise': '2 * cost', + }, + }, + id='a-case-of-a-named-expression', + ), + ], +) +def test_a_multi_dim_tested_where_its_rows_are_no_domain_loads(patch: dict[str, Any]): + """A constraint's rows follow its operands, and a case covers both branches, so neither can move the name into `dims:`. + + Was: a constraint's `where: pair` was refused with `dims:` as the rewrite, + where a relation of the same file loads. + """ + to_spec(varied({**ON_A_RELATION, 'variables': {'p': {'dims': ['g', 'h']}}}, **patch)) + + class TestAssumptions: """What an ``assumptions:`` entry may state, and what the load refuses. @@ -3250,6 +3269,64 @@ def test_a_multi_dim_is_declared_under_dimensions_alone(patch: dict[str, Any], m assert message in _refusal(**patch) +def test_a_multi_dim_over_one_member_is_a_subset_of_it(): + """`extendable: {over: [g]}` lists some labels of `g`; it was refused as needing two members.""" + spec = _spec(**{'dimensions.extendable': {'over': ['g']}, 'variables.build': {'dims': ['extendable', 'h']}}) + build = spec.program.variables['build'] + assert (build.dims, build.in_) == (('g', 'h'), ('extendable',)), 'the subset stands for g, at the rows it lists' + assert to_spec(spec.to_yaml()) == spec, 'the subset writes back as the file wrote it' + + +def test_a_row_of_a_multi_dim_over_one_member_prints_without_a_tuple(): + """A constraint over a one-member multi-dim printed its row as `(g) ∈ extendable`, a tuple of one.""" + spec = _spec( + **{ + 'dimensions.extendable': {'over': ['g']}, + 'constraints.c': {'dims': ['extendable'], 'expression': 'p <= c'}, + } + ) + latex = typeset(spec, 'latex') + assert r'\left( g \right) \in' not in latex, 'one index is no tuple' + assert r'g \in \mathrm{extendable}' in latex, latex + + +@pytest.mark.parametrize( + 'patch', + [ + pytest.param( + {'constraints.c': {'dims': ['pair'], 'expression': 'q <= c'}}, id='a-constraint-over-its-operands-rows' + ), + pytest.param({'expressions.e': {'dims': ['pair'], 'expression': 'q * c'}}, id='a-named-expression'), + pytest.param( + {'given.parameters.gp': {'dims': ['pair']}, 'constraints.c': {'dims': ['pair'], 'expression': 'q <= gp'}}, + id='a-given-parameter', + ), + pytest.param( + { + 'given.expressions.inj': {'dims': ['pair']}, + 'constraints.c': {'dims': ['g', 'h'], 'expression': 'q <= inj'}, + }, + id='a-given-expression', + ), + pytest.param( + { + 'given.expressions.inj': {'dims': ['pair']}, + 'expressions.term': {'dims': ['g', 'h'], 'expression': 'q', 'adds_to': 'inj'}, + }, + id='an-adds-to-term', + ), + ], +) +def test_dims_compares_the_members_of_a_multi_dim(patch: dict[str, Any]): + """`dims:` is the entry's domain: a multi-dim in it is its members, so every rule that compares `dims:` sees them. + + Was: a named expression and a `given:` entry refused a multi-dim in `dims:` + as not a dimension. + """ + spec = _spec(**{'dimensions.pair': {'over': ['g', 'h']}}, **patch) + assert to_spec(spec.to_yaml()) == spec, 'the multi-dim writes back into dims: as the file wrote it' + + def test_a_multi_dim_has_the_combinations_its_rows_list(): """A combination with no row is not a label of a multi-dim, so it takes no `missing:` and reads `absent`.""" spec = to_spec(varied(SMALL_MODEL, **{'dimensions.pair': {'over': ['g', 'h']}})) diff --git a/tools/gallery.py b/tools/gallery.py index 2434ad2b8..5d18022a7 100644 --- a/tools/gallery.py +++ b/tools/gallery.py @@ -50,6 +50,7 @@ 'commitment.md': ROOT / 'examples' / 'commitment.yaml', 'connection.md': ROOT / 'examples' / 'connection.yaml', 'sparse.md': ROOT / 'examples' / 'sparse.yaml', + 'candidates.md': ROOT / 'examples' / 'candidates.yaml', 'piecewise.md': ROOT / 'examples' / 'piecewise.yaml', 'piecewise_adjacency.md': ROOT / 'examples' / 'piecewise_adjacency.yaml', 'sos.md': ROOT / 'examples' / 'sos.yaml', From 8ddae5dc39275693ae4f980ca814cc35779ed4e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 22:15:18 +0000 Subject: [PATCH 9/9] feat(language): a dimension's members are read by name Four refusals of #9 become the spelling each one stood for: - an operand over a member meets the dimension: read at each tuple, as at(x, by=dimension[member]) reads it; - sum(x, over=member) on an operand over the dimension: the sum through the dimension, grouped by the members over= leaves; - two dimensions in one dims: that share a member: the entry over the members, where: the dimensions; - a step along a member: the entry it steps in, and the entry it steps, over the members, where: the dimension. Every spec #9 loads builds the same program. A parameter over the members carries the tuples as its domain, where a tool refuses a stray row; that is the one change no #9 spelling states. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B3nA494URRSeWy6wmELGP --- CHANGELOG.md | 1 + docs/about/limits.md | 6 +- docs/examples/connection.md | 5 +- docs/examples/sparse.md | 16 +- docs/examples/times/capacity.md | 10 +- docs/examples/times/trade.md | 12 +- docs/reference/language/dimensions.md | 108 +++++----- examples/connection.yaml | 2 +- examples/sparse.yaml | 6 +- examples/times/capacity.yaml | 3 +- examples/times/trade.yaml | 6 +- src/mathspec/_condition_resolver.py | 5 +- src/mathspec/_expression_resolver.py | 99 +++++++++- src/mathspec/_flat.py | 167 ++++++++++++++++ src/mathspec/_resolution.py | 16 +- src/mathspec/_validation.py | 1 - src/mathspec/program.py | 4 + src/mathspec/spec.py | 10 +- tests/fixtures/strict/connection.yaml | 55 ++++++ tests/fixtures/strict/sparse.yaml | 51 +++++ tests/fixtures/strict/times/capacity.yaml | 62 ++++++ tests/fixtures/strict/times/trade.yaml | 50 +++++ tests/test_dimensions.py | 10 +- tests/test_tuple_dimensions.py | 214 -------------------- tests/test_tuple_sugar.py | 228 ++++++++++++++++++++++ tests/test_validation.py | 22 --- 26 files changed, 832 insertions(+), 337 deletions(-) create mode 100644 src/mathspec/_flat.py create mode 100644 tests/fixtures/strict/connection.yaml create mode 100644 tests/fixtures/strict/sparse.yaml create mode 100644 tests/fixtures/strict/times/capacity.yaml create mode 100644 tests/fixtures/strict/times/trade.yaml create mode 100644 tests/test_tuple_sugar.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 631f7c3f8..a8fb319e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ it releases that version ([RELEASING.md](https://github.com/energy-models/mathsp ## Upcoming version +- feat(language): a dimension's members are read by name ([#8](https://github.com/fluxopt/mathspec/pull/8)) - feat(language)!: a dimension lists its members, strictly ([#9](https://github.com/fluxopt/mathspec/pull/9)) - Breaks a file: `relations:` is gone. A set of combinations of labels is a dimension with `over:`, read through its members with `at()` and tested by its bare name in a `where:`. A dimension twice in a relation is an alias member. - feat(language): a dimension lists its members ([#1017](https://github.com/energy-models/mathspec/pull/1017)) diff --git a/docs/about/limits.md b/docs/about/limits.md index f01d4490b..9f09db7e3 100644 --- a/docs/about/limits.md +++ b/docs/about/limits.md @@ -116,10 +116,10 @@ same thing. | A vocabulary for tracked metrics: `impacts:`, `effects:`, a `costs` dimension | A tracked metric is already a [named expression](../reference/language/named.md#expressions). | An `impact` dimension and one named expression. A constraint can cap the expression, the objective can weight it, and a tool reads it back after the solver runs. | | Helpers for one domain, such as `reduce_carrier_dim` | A macro already states a helper. | A [component library](../examples/library/index.md) of macros. Its macros use only the operators that exist. | | A Python application programming interface (API) for building specs | The file is already the API, and the file is what you review and diff. | YAML, or a `dict` with the same keys. A tool can combine dicts before it calls `to_spec`. | -| Access to a member of a dimension that lists its members, by a dot or by aligning `flow <= capacity` | [Each member is already a mapping](../reference/language/dimensions.md#over) from each tuple to its label. | A lookup or a sum through that mapping: `at(capacity, by=connection[unit])`. | +| Access to a member of a dimension that lists its members, by a dot, as in `connection.unit` | [Each member is already a mapping](../reference/language/dimensions.md#over) from each tuple to its label. | A lookup or a sum through that mapping: `at(capacity, by=connection[unit])`. | | A `relations:` section for a set of combinations of labels | A [dimension with members](../reference/language/dimensions.md#over) already lists the combinations, and its mappings already give each one its labels. | `dimensions: { connection: { over: [generator, bus] } }`, tested by its bare name in a `where:` on an entry over every member. | -| Two dimensions in one `dims:` that share a member, or a dimension beside its own member, read as their cross product | A tool may store each dimension as the columns of its members, so the shared member would index the entry twice. | One of the two in `dims:`, and a `where:` that reads a relation through the shared member, or an alias of the member: `site: { over: [from_unit, bus] }`. | -| An order along a dimension that lists its members, for `shift`, `sum_back`, `position`, `piecewise:` or `sos:` of `type: 2` | A tuple of labels has no order, and each member can already declare one. | An entry over the members, with `where:` the bare name of the dimension. The construct then steps along an ordered member. | +| Two dimensions in one `dims:` that share a member, read as their cross product | A tool may store each dimension as the columns of its members, so the shared member would index the entry twice. | The entry over the members, at the rows both list: mathspec [reads `dims: [connection, active]` so](../reference/language/dimensions.md#members-by-name). An alias of the member keeps the two apart. | +| An order along a dimension that lists its members, for `shift`, `sum_back`, `position`, `piecewise:` or `sos:` of `type: 2` | A tuple of labels has no order, and each member can already declare one. | A step along an ordered member, which [reads the entry over the members](../reference/language/dimensions.md#members-by-name). | | A `join` operator apart from `sum` and `at`, such as `sum(join(p, generator[bus]), over=generator)` | [`sum(by=)` and `at(by=)` already state a join](relations-as-linear-maps.md#why-the-language-has-no-join-operator). | A sum or a lookup with `by=`, as in the first row. | ### Data preparation diff --git a/docs/examples/connection.md b/docs/examples/connection.md index 4742dd613..313d71363 100644 --- a/docs/examples/connection.md +++ b/docs/examples/connection.md @@ -18,7 +18,8 @@ the bus, so two ports of one unit may attach to the same bus. [`where:`](../reference/language/parameters-variables-constraints.md#where) keeps only the output ports, and `missing: 1` applies only there. At an input port, `injection` takes its case, and elsewhere `efficiency` times the flow of -the unit, read with `at`. The sum in `balance` adds each connection to the bus +the unit, read at each connection +[by member name](../reference/language/dimensions.md#members-by-name). The sum in `balance` adds each connection to the bus it attaches to. @@ -64,7 +65,7 @@ expressions: dims: [connection, snapshot] cases: draws: { when: "connection[role] == 'input'", expression: "-at(flow, by=connection[unit])" } - otherwise: efficiency * at(flow, by=connection[unit]) + otherwise: efficiency * flow constraints: balance: diff --git a/docs/examples/sparse.md b/docs/examples/sparse.md index be4c2ec3a..1457c6ec5 100644 --- a/docs/examples/sparse.md +++ b/docs/examples/sparse.md @@ -19,10 +19,12 @@ the sets in one way: `connection` is a dimension whose labels are the pairs of a unit and a port. `efficiency` and `flow` are over it, so `missing: 1` fills only the ports it -lists, and a data row for a port it does not list is refused. `active` is a -dimension over `unit` and `period`. `output` is over both members, and -`where: active` keeps the pairs that `active` lists, so `ramp_up` can step -along the ordered member `period`. +lists, and a data row for a port it does not list is refused. `port_limit` +reads `capacity` at the unit of each connection, and `split` sums over `port`, +both [by member name](../reference/language/dimensions.md#members-by-name). +`active` is a dimension over `unit` and `period`, and `output` is over it. +`ramp_up` steps `output` along the ordered member `period`, so mathspec reads +`output` over `unit` and `period`, confined to the pairs that `active` lists. [#1019](https://github.com/energy-models/mathspec/issues/1019) compares how three designs for sparse sets write this spec. @@ -51,18 +53,18 @@ parameters: load: { dims: [bus, period], description: demand to be met } variables: - output: { dims: [unit, period], where: active, bounds: { lower: 0 }, description: what a unit produces } + output: { dims: [active], bounds: { lower: 0 }, description: what a unit produces } flow: { dims: [connection, period], bounds: { lower: 0 }, description: what a port takes of it } constraints: port_limit: description: a port takes no more than the unit's capacity dims: [connection, period] - expression: flow <= at(capacity, by=connection[unit]) + expression: flow <= capacity split: description: the ports of a unit take what it produces dims: [unit, period] - expression: sum(flow, over=connection, by=connection[unit]) == output + expression: sum(flow, over=port) == output ramp_up: description: output rises by at most the ramp from one active period to the next dims: [unit, period] diff --git a/docs/examples/times/capacity.md b/docs/examples/times/capacity.md index 1c84d2ef3..a97a615b1 100644 --- a/docs/examples/times/capacity.md +++ b/docs/examples/times/capacity.md @@ -18,9 +18,10 @@ without storage, with the upper bound only. The GAMS conditions map to the spec as follows: - `RTP_VINTYR` and `PRC_TS` are dimensions with - [members](../../reference/language/dimensions.md#over). `act` is over every - member of both, and `where: "prc_ts AND rtp_vintyr"` keeps the coordinates - that both list. + [members](../../reference/language/dimensions.md#over). `act` is over both, + which share `region` and `process`, so mathspec + [reads it over their members](../../reference/language/dimensions.md#members-by-name), + at the coordinates that both list. - `TS_MAP(R,S,TS)` is the dimension `ts_map`. Its upper timeslice is the alias `upper_timeslice`, so the two timeslices are two members. `eq_capact_up` reads the activity at each tuple of `ts_map`, sums it to each @@ -69,8 +70,7 @@ variables: cap: { dims: [region, period, process], bounds: { lower: 0 }, description: installed capacity (VAR_CAP) } act: description: activity of a vintage of a process in a timeslice (VAR_ACT) - dims: [region, vintage, period, process, timeslice] - where: "prc_ts AND rtp_vintyr" + dims: [prc_ts, rtp_vintyr] bounds: { lower: 0 } constraints: diff --git a/docs/examples/times/trade.md b/docs/examples/times/trade.md index 992c4d724..07d283972 100644 --- a/docs/examples/times/trade.md +++ b/docs/examples/times/trade.md @@ -19,8 +19,10 @@ as TIMES' `TOP_IRE` does, each a tuple of its five members, and `ire_flo`, exist is refused. The data of `ire_flo` has the five columns that the TIMES data already has, and no label is invented. -`balance` adds the imports of each region and commodity through the members -`region` and `commodity` of each link. It sums the exports to each +`balance` adds the imports of each region and commodity: it sums over the +other three members of each link, which +[groups by the two it leaves](../../reference/language/dimensions.md#members-by-name). +It sums the exports to each `from_region` and `from_commodity`, and reads those back onto `region` and `commodity` through the mapping of each alias. @@ -66,9 +68,9 @@ constraints: dims: [commodity, region] expression: >- supply - + sum(ire_flo * export, over=top_ire, by=top_ire[commodity, region]) - - sum(sum(sum(export, over=top_ire, by=top_ire[from_commodity, from_region]), - over=from_region, by=from_region[region]), over=from_commodity, by=from_commodity[commodity]) + + sum(ire_flo * export, over=[from_commodity, from_region, process]) + - sum(sum(sum(export, over=[commodity, process, region]), over=from_region, by=from_region[region]), + over=from_commodity, by=from_commodity[commodity]) >= demand objective: diff --git a/docs/reference/language/dimensions.md b/docs/reference/language/dimensions.md index 587122071..e23a4f056 100644 --- a/docs/reference/language/dimensions.md +++ b/docs/reference/language/dimensions.md @@ -242,54 +242,50 @@ mathspec refuses a declared mapping with the name of a member. With mappings: 'connection': mapping 'unit' has the name of a member of 'connection', which already gives each tuple its label of 'unit'. Read the member as connection[unit], or rename the mapping. ``` -### One dimension in every operator - -A dimension with members is one dimension wherever a `dims:` list or an -operator names it. mathspec does not align an entry over a member with an -entry over the dimension. With `flow <= capacity` in `limit` above, mathspec -prints: - -```text -Constraint 'limit': an operand over ['unit'] meets one over 'connection'. 'unit' is a member of 'connection', which is one dimension here. Read the operand at each tuple: at(capacity, by=connection[unit]), or sum flow to the member: sum(flow, over=connection, by=connection[unit]). -``` - -mathspec suggests the lookup through the mapping of the member, or the sum -of `flow` to the member. The same refusal applies wherever the two operands -meet: in a product, in a sum, and in the objective, as in -`sum(flow * capacity)`, in an [assumption](assumptions.md), and in the -`where:` of a `piecewise:` entry. - -### Each member once in `dims:` - -A tool may store a dimension with members as the columns of its members. So a -`dims:` list reaches each member once. mathspec refuses two dimensions in one -`dims:` that share a member, and a dimension beside one of its own members. -With `site: { over: [unit, bus] }` and `dims: [connection, site, snapshot]`, -mathspec prints: - -```text -Variable 'flow': 'connection' and 'site' in dims: share the member 'unit', so the entry would be indexed by 'unit' twice. Keep 'connection' in dims:, list ['bus'] beside it, and confine the entry with where: "at(site, by=connection[unit])". Or declare an alias of 'unit', such as from_unit: {alias: unit}, and list it in 'site' in place of 'unit', so the members differ. -``` - -With `dims: [connection, unit, snapshot]`, mathspec prints: - -```text -Variable 'flow': 'connection' and 'unit' in dims: share the member 'unit', so the entry would be indexed by 'unit' twice. Delete 'unit' from dims: and read it as connection[unit], or declare an alias of 'unit', such as from_unit: {alias: unit}, and list it in place of 'unit'. -``` - -The refusal applies to every `dims:` list: a parameter, a variable, a -constraint, a named expression, a given entry and a `piecewise:` entry. A -different unit at each connection is an alias of `unit`, which is a member of -its own. - -A sum over a member is refused the same way. With `sum(flow, over=unit)`, -mathspec prints: - -```text -Constraint 'balance': sum(over=unit) but the expression has only the dims ['connection', 'snapshot']. 'unit' is a member of 'connection', which is one dimension here. Write sum(..., over=connection, by=connection[port]). -``` - -mathspec suggests the sum over the dimension, grouped by the other member. +### Members by name + +A member is a dimension of its own, so an entry may name it where it means the +dimension that lists it. mathspec reads the name as the call through the +dimension that it stands for, and builds the same program. With `flow` over +`[connection, snapshot]`, `capacity` over `[unit]` and `active` over +`[unit, period]`: + +| Written | Read as | +| ------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| `flow <= capacity` | `flow <= at(capacity, by=connection[unit])` | +| `sum(flow, over=port)` | `sum(flow, over=connection, by=connection[unit])` | +| `sum(flow, over=[port, unit])` | `sum(flow, over=connection)` | +| `dims: [active, connection, snapshot]` | `dims: [period, port, snapshot, unit]`, `where: active AND connection` | +| `output: { dims: [active] }`, and `shift(output, along=period, offset=1)` | `output: { dims: [period, unit], where: active }` | + +The first three rows read one operand through the dimension: + +- An operand over a member meets an operand over the dimension in a product, + a sum, a comparison, the objective, a condition or an + [assumption](assumptions.md), and mathspec reads it at each tuple. +- A `sum` over members of a dimension that its operand is over sums over the + dimension, grouped by the members that `over=` leaves. + +The last two rows read the entry over the members: + +- Two dimensions in one `dims:` that share a member, or a dimension beside one + of its own members, make the entry over the members, each once. The entry is + confined to the tuples that every dimension lists, by a `where:` that names + them. +- A `shift`, `sum_back` or `position` along a member makes the entry it steps + in, and the entry it steps, over the members, with `where:` the dimension. + The dimension has no order, and the member can have one. + +An entry over the members reads an operand over the dimension at the members, +as `sum(flow, over=connection, by=connection[port, unit])` does. An entry over +the dimension reads an entry over the members at each tuple, as +`at(output, by=active[period, unit])` does. + +A parameter over two dimensions that share a member differs in one way from +its spelling with `where:`. A tool refuses a data row outside the tuples that +both list, where it drops a row outside a `where:`. The program holds those +tuples as the +[`domain`](../program.md#mathspec.program.Parameter.domain) of the parameter. A `dims:` list is the domain of an entry, so a tool refuses a data row at a tuple that the dimension does not list. A @@ -297,14 +293,14 @@ tuple that the dimension does not list. A At load, mathspec also refuses each of these, and prints the rewrite: -| Refused | What you write instead | -| ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | -| a member that has members of its own | the members themselves, in one list | -| a dimension twice, as in `over: [region, region]` | an [alias](#alias) of the dimension as one of the members | -| `ordered: true`, `dtype:` or `one_of:` beside `over:` | the key on a member | -| `shift`, `sum_back`, `position`, `piecewise:` or `sos:` of `type: 2` along the dimension | an entry over the members, with `where:` the bare name of the dimension | -| a comparison of the dimension with a label, as in `connection == 'a'` | a comparison of a member, as in `connection[unit] == 'u1'` | -| two dimensions in one `dims:` that share a member, or a dimension beside its own member | [one of the two, and a `where:`, or an alias](#each-member-once-in-dims) | +| Refused | What you write instead | +| ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | +| a member that has members of its own | the members themselves, in one list | +| a dimension twice, as in `over: [region, region]` | an [alias](#alias) of the dimension as one of the members | +| `ordered: true`, `dtype:` or `one_of:` beside `over:` | the key on a member | +| `shift`, `sum_back`, `position`, `piecewise:` or `sos:` of `type: 2` along the dimension | a step along a member, which [reads the entry over the members](#members-by-name) | +| a comparison of the dimension with a label, as in `connection == 'a'` | a comparison of a member, as in `connection[unit] == 'u1'` | +| two dimensions in the `dims:` of a `piecewise:` entry that share a member | the members themselves in `dims:`, and a `where:` that names the dimensions | ### The bare name in a condition diff --git a/examples/connection.yaml b/examples/connection.yaml index db4b7ff1a..e1c8f8d7c 100644 --- a/examples/connection.yaml +++ b/examples/connection.yaml @@ -43,7 +43,7 @@ expressions: dims: [connection, snapshot] cases: draws: { when: "connection[role] == 'input'", expression: "-at(flow, by=connection[unit])" } - otherwise: efficiency * at(flow, by=connection[unit]) + otherwise: efficiency * flow constraints: balance: diff --git a/examples/sparse.yaml b/examples/sparse.yaml index 152d5774d..1d3d6c6af 100644 --- a/examples/sparse.yaml +++ b/examples/sparse.yaml @@ -25,18 +25,18 @@ parameters: load: { dims: [bus, period], description: demand to be met } variables: - output: { dims: [unit, period], where: active, bounds: { lower: 0 }, description: what a unit produces } + output: { dims: [active], bounds: { lower: 0 }, description: what a unit produces } flow: { dims: [connection, period], bounds: { lower: 0 }, description: what a port takes of it } constraints: port_limit: description: a port takes no more than the unit's capacity dims: [connection, period] - expression: flow <= at(capacity, by=connection[unit]) + expression: flow <= capacity split: description: the ports of a unit take what it produces dims: [unit, period] - expression: sum(flow, over=connection, by=connection[unit]) == output + expression: sum(flow, over=port) == output ramp_up: description: output rises by at most the ramp from one active period to the next dims: [unit, period] diff --git a/examples/times/capacity.yaml b/examples/times/capacity.yaml index f98e69414..2a71aa913 100644 --- a/examples/times/capacity.yaml +++ b/examples/times/capacity.yaml @@ -37,8 +37,7 @@ variables: cap: { dims: [region, period, process], bounds: { lower: 0 }, description: installed capacity (VAR_CAP) } act: description: activity of a vintage of a process in a timeslice (VAR_ACT) - dims: [region, vintage, period, process, timeslice] - where: "prc_ts AND rtp_vintyr" + dims: [prc_ts, rtp_vintyr] bounds: { lower: 0 } constraints: diff --git a/examples/times/trade.yaml b/examples/times/trade.yaml index c21968756..e9d87d97d 100644 --- a/examples/times/trade.yaml +++ b/examples/times/trade.yaml @@ -40,9 +40,9 @@ constraints: dims: [commodity, region] expression: >- supply - + sum(ire_flo * export, over=top_ire, by=top_ire[commodity, region]) - - sum(sum(sum(export, over=top_ire, by=top_ire[from_commodity, from_region]), - over=from_region, by=from_region[region]), over=from_commodity, by=from_commodity[commodity]) + + sum(ire_flo * export, over=[from_commodity, from_region, process]) + - sum(sum(sum(export, over=[commodity, process, region]), over=from_region, by=from_region[region]), + over=from_commodity, by=from_commodity[commodity]) >= demand objective: diff --git a/src/mathspec/_condition_resolver.py b/src/mathspec/_condition_resolver.py index e39ca2b93..37aa449aa 100644 --- a/src/mathspec/_condition_resolver.py +++ b/src/mathspec/_condition_resolver.py @@ -413,9 +413,10 @@ def _expression_comparison(self, node: UnresolvedComparisonNode) -> ExpressionCo self.errors.extend(zero_divisor_refusals(side, context, ns.spec.parameters, {})) if len(self.errors) > found: return node - left, right = sides - if self._expressions.met(left, right): + if (pair := self._expressions.aligned(*sides)) is None: return node + left, right = pair + dims = set(sorted_dims(left.dims, right.dims)) if all(_is_number(side) for side in sides): self.errors.append( f"{context}: '{node.left} {node.op} {node.right}' compares two numbers. Name the parameter one " diff --git a/src/mathspec/_expression_resolver.py b/src/mathspec/_expression_resolver.py index 42b638436..5764b9df2 100644 --- a/src/mathspec/_expression_resolver.py +++ b/src/mathspec/_expression_resolver.py @@ -13,7 +13,7 @@ from __future__ import annotations from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, assert_never, cast +from typing import TYPE_CHECKING, Literal, assert_never, cast from mathspec._dimensions import check_meeting, check_node, joined, operand_over, sorted_dims from mathspec._expression_parser import ( @@ -158,8 +158,9 @@ def arith(self, node: ArithmeticNode) -> Expression | None: def _binary(self, node: BinaryOperatorNode) -> Expression | None: """A subtraction is an addition of the negation, so a program has one additive node.""" left, right = self.arith(node.left), self.arith(node.right) - if left is None or right is None or self.met(left, right): + if left is None or right is None or (pair := self.aligned(left, right)) is None: return None + left, right = pair dims = sorted_dims(left.dims, right.dims) match node.op: case '+': @@ -175,6 +176,77 @@ def _binary(self, node: BinaryOperatorNode) -> Expression | None: case _: assert_never(node.op) + def aligned(self, left: Expression, right: Expression) -> tuple[Expression, Expression] | None: + """*left* and *right*, an operand over members of a dimension the other carries read at each of its tuples. + + A member operand meets the dimension by name: ``flow <= capacity`` reads + ``capacity`` at the unit of each connection, as + ``at(capacity, by=connection[unit])`` does, and builds the same node. + Where the entry is over the members and not the dimension, the operand + over the dimension is read at the members instead, as + [`framed`][mathspec._expression_resolver.ExpressionResolver.framed] reads it. + + Returns: + The two operands, or ``None`` with the refusal appended where a + read fails or the two still meet a member against its dimension. + """ + while (read := self._read_at_tuples(left, right)) is not None: + if read is False: + return None + left, right = read + return None if self.met(left, right) else (left, right) + + def _read_at_tuples( + self, left: Expression, right: Expression + ) -> tuple[Expression, Expression] | Literal[False] | None: + """The pair with one operand read through the first dimension the other carries members of, or ``None`` where none is.""" + spec = self.ns.spec + for one, other, flipped in ((left, right, False), (right, left, True)): + for name in sorted(set(one.dims) - set(other.dims)): + members = {dim for _, dim in spec.dimensions[name].members} + if not (stray := tuple(sorted((members & set(other.dims)) - set(one.dims)))): + continue + if name not in self.frame and members <= self.frame: + read = self._sum(one, (name,), ColumnsNode(name, tuple(sorted(members)))) + if read is None: + return False + return (other, read) if flipped else (read, other) + join = self.lookup(ColumnsNode(name, stray), frozenset(other.dims)) + if join is None or (read := self._joined(other, join)) is None: + return False + return (read, one) if flipped else (one, read) + return None + + def framed(self, node: Expression) -> Expression | None: + """*node* read at the entry's own dimensions, where it meets one of them by its members. + + Two readings meet the frame by name. A *node* over members of a + dimension the frame lists is read at each of its tuples, as + ``at(node, by=dimension[members])`` reads it. A *node* over a dimension + whose members the frame lists is read at those members, as + ``sum(node, over=dimension, by=dimension[members])`` reads it. The frame + lists each tuple once, so each group holds one term. + + Returns: + The node, or ``None`` with the refusal appended where a read fails. + """ + spec, frame = self.ns.spec, self.frame + for name in sorted(frame - set(node.dims)): + members = {dim for _, dim in spec.dimensions[name].members} + if stray := tuple(sorted((members & set(node.dims)) - frame)): + join = self.lookup(ColumnsNode(name, stray), frozenset(node.dims)) + if join is None or (read := self._joined(node, join)) is None: + return None + node = read + for name in sorted(set(node.dims) - frame): + members = {dim for _, dim in spec.dimensions[name].members} + if members and members <= frame: + summed = self._sum(node, (name,), ColumnsNode(name, tuple(sorted(members)))) + if summed is None: + return None + node = summed + return node + def met(self, left: Expression, right: Expression) -> bool: """Whether *left* and *right* meet a member against the dimension that lists it, the refusal appended.""" try: @@ -293,6 +365,8 @@ def _sum(self, operand: Expression, over: tuple[str, ...] | None, columns: Colum call, so no axis is ever left open. A name in *over* that is no column of the relation is a dim, summed away after the group-by. """ + if columns is None and over is not None and (meant := self._over_members(operand, over)) is not None: + over, columns = meant if columns is None: if over is None: return self._bare_sum(operand) @@ -307,6 +381,27 @@ def _sum(self, operand: Expression, over: tuple[str, ...] | None, columns: Colum grouped = self._summed(through, join.axes) return self._summed(grouped, tuple(Axis(d) for d in plain)) if plain and grouped is not None else grouped + def _over_members( + self, operand: Expression, over: tuple[str, ...] + ) -> tuple[tuple[str, ...], ColumnsNode | None] | None: + """``over=`` naming members of a dimension *operand* carries, as the sum through that dimension it means. + + ``sum(flow, over=port)`` on ``flow`` over ``connection`` sums over + ``connection`` grouped by its other members, as + ``sum(flow, over=connection, by=connection[unit])`` does. ``None`` + where *over* names no such member, or members of more than one + dimension, which the plain sum then refuses. + """ + if not (stray := {d for d in over if d not in operand.dims}): + return None + for name in sorted(operand.dims): + members = {dim for _, dim in self.ns.spec.dimensions[name].members} + if members and stray <= members: + rest = tuple(sorted(members - set(over))) + summed = tuple(sorted({*(d for d in over if d not in members), name})) + return summed, ColumnsNode(name, rest) if rest else None + return None + def _summed(self, operand: Expression, over: tuple[Axis, ...]) -> Sum | None: """A [`Sum`][mathspec.program.Sum] over *over*, which removes the dims its plain axes name; a join's axes are no dims.""" closed = {axis.dimension for axis in over if axis.column is None} diff --git a/src/mathspec/_flat.py b/src/mathspec/_flat.py new file mode 100644 index 000000000..cdd15dfab --- /dev/null +++ b/src/mathspec/_flat.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: mathspec Contributors +# +# SPDX-License-Identifier: MIT + +"""The spec an entry over dimensions with members lowers as, where it needs their members by name. + +Two cases read an entry over its members rather than over its tuples: two of +its dimensions share a member, as in ``dims: [connection, active]``, or a +step runs along a member, as ``shift(output, along=period)`` does on +``output`` over ``active``. Each such entry lowers as the same entry over the +members, with ``where:`` the dimensions it listed, which is the spelling a +dimension without an order and without a shared member must write. Every +other entry keeps its dimensions, and meets these by name. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, cast + +from mathspec._expression_parser import FunctionCallNode, NameNode, nodes, parse_expression +from mathspec.errors import LanguageError +from mathspec.program import And, Mask, RelationDefined + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Mapping + + from mathspec.program import Condition + from mathspec.spec import ConstraintSpec, ExpressionSpec, ParameterSpec, Spec, VariableSpec + + type Framed = ParameterSpec | VariableSpec | ConstraintSpec | ExpressionSpec + +_SECTIONS = ('parameters', 'variables', 'constraints', 'expressions') +_STEP = re.compile(r'\balong\s*=\s*([A-Za-z_]\w*)|\bposition\(\s*([A-Za-z_]\w*)') + +type Entry = tuple[str, str] + + +def flat_spec(spec: Spec) -> tuple[Spec, dict[str, Mask]]: + """*spec* with each entry that needs its members by name declared over them, and the domain of each such parameter. + + Returns *spec* itself, and no domain, where no entry needs it. A + parameter's domain is the tuples its ``dims:`` listed: a tool refuses its + data row outside them, where the ``where:`` of the flat spelling drops it. + """ + from mathspec.spec import Spec + + tuples = {name: frozenset(dim for _, dim in d.members) for name, d in spec.dimensions.items() if d.members} + entries: dict[Entry, Framed] = { + (section, name): entry + for section in _SECTIONS + for name, entry in cast('Mapping[str, Framed]', getattr(spec, section)).items() + if set(entry.dims or ()) & set(tuples) + } + flat = _flattened(entries, tuples, spec) + if not flat: + return spec, {} + written = spec.to_dict() + for (section, name), over in flat.items(): + declared = cast('dict[str, dict[str, object]]', written[section]) + declared[name] = _over_members(section, declared[name], entries[section, name], over, tuples) + domains = { + name: _domain(entries[section, name].dims or (), tuples) for section, name in flat if section == 'parameters' + } + return Spec.model_validate(written), domains + + +def _flattened( + entries: Mapping[Entry, Framed], tuples: Mapping[str, frozenset[str]], spec: Spec +) -> dict[Entry, frozenset[str]]: + """Each entry to read over its members, with the dimensions it lists that it reads so. + + An entry whose dimensions share a member, an entry that steps along a + member of one of its dimensions, and an entry that a step reads so. + """ + stepped = _stepped(spec, tuples, entries) + flat = { + key + for key, entry in entries.items() + if _shares(entry.dims or (), tuples) or _steps(entry, tuples) or key[1] in stepped + } + return {key: frozenset(d for d in entries[key].dims or () if d in tuples) for key in flat} + + +def _shares(dims: Iterable[str], tuples: Mapping[str, frozenset[str]]) -> bool: + """Whether two of *dims* reach one member: two dimensions with members, or one beside a member of its own.""" + reached = [member for d in dims for member in tuples.get(d, {d})] + return len(reached) != len(set(reached)) + + +def _members(entry: Framed, tuples: Mapping[str, frozenset[str]]) -> set[str]: + """The members of the dimensions *entry* is over, less those it is over as dimensions of their own.""" + dims = set(entry.dims or ()) + return {m for d in dims if d in tuples for m in tuples[d]} - dims + + +def _steps(entry: Framed, tuples: Mapping[str, frozenset[str]]) -> bool: + """Whether the entry steps along a member of one of its dimensions, which a tuple has no order to step along.""" + members = _members(entry, tuples) + return any(set(found) & members for found in _STEP.findall(' '.join(_texts(entry)))) + + +def _stepped(spec: Spec, tuples: Mapping[str, frozenset[str]], entries: Mapping[Entry, Framed]) -> set[str]: + """The entries a ``shift`` or ``sum_back`` steps along a member of one of their dimensions, anywhere in the file.""" + members_of = {name: _members(entry, tuples) for (_, name), entry in entries.items()} + texts = [text for entry in _every_entry(spec) for text in _texts(entry)] + if spec.objective is not None: + texts.append(spec.objective.expression) + stepped: set[str] = set() + for text in texts: + try: + tree = parse_expression(text) + except LanguageError: + continue + for call in nodes(tree): + if not (isinstance(call, FunctionCallNode) and call.name in ('shift', 'sum_back')): + continue + if not isinstance(along := call.kwargs.get('along'), NameNode): + continue + read = {n.name for n in nodes(*call.args) if isinstance(n, NameNode)} + stepped |= {name for name in read & set(members_of) if along.name in members_of[name]} + return stepped + + +def _every_entry(spec: Spec) -> Iterator[Framed]: + """Every entry of the four sections that declare ``dims:``.""" + for section in _SECTIONS: + yield from cast('Mapping[str, Framed]', getattr(spec, section)).values() + + +def _texts(entry: Framed) -> Iterator[str]: + """Every expression and condition *entry* writes.""" + for key in ('where', 'expression', 'otherwise'): + if isinstance(text := getattr(entry, key, None), str): + yield text + if (bounds := getattr(entry, 'bounds', None)) is not None: + yield from (side for side in (bounds.lower, bounds.upper) if isinstance(side, str)) + for case in getattr(entry, 'cases', {}).values(): + yield case.when + if isinstance(case.expression, str): + yield case.expression + + +def _over_members( + section: str, written: dict[str, object], entry: Framed, over: frozenset[str], tuples: Mapping[str, frozenset[str]] +) -> dict[str, object]: + """The written *entry* over the members of the dimensions *over* names, confined to them by ``where:``. + + A named expression takes no ``where:``, so it is over the members alone, + and reads absent wherever the entries in its body are. + """ + flat = dict(written) + flat['dims'] = sorted({m for d in entry.dims or () for m in (tuples[d] if d in over else {d})}) + if section != 'expressions': + listed = ' AND '.join(sorted(over)) + own = getattr(entry, 'where', None) + flat['where'] = listed if own is None else f'({listed}) AND ({own})' + return flat + + +def _domain(dims: Iterable[str], tuples: Mapping[str, frozenset[str]]) -> Mask: + """The tuples a parameter's ``dims:`` listed, as the condition that holds at its domain.""" + listed: list[Condition] = [RelationDefined(d, tuple(sorted(tuples[d]))) for d in sorted(dims) if d in tuples] + root = listed[0] + for atom in listed[1:]: + root = And(root, atom) + return Mask(root) diff --git a/src/mathspec/_resolution.py b/src/mathspec/_resolution.py index dfdd576b5..d643e18f0 100644 --- a/src/mathspec/_resolution.py +++ b/src/mathspec/_resolution.py @@ -345,9 +345,9 @@ def resolve_expression( ) -> Expression | None: """Build the program tree *node* stands for, checking every name and operator call shape on the way. - *frame* is the ``dims:`` of the entry *node* belongs to: where it declares - a dimension with members and a member, an operand over the member meets - one over the dimension as their cross product. + *frame* is the ``dims:`` of the entry *node* belongs to. A tree that + meets it by the members of a dimension is read at it + ([`ExpressionResolver.framed`][mathspec._expression_resolver.ExpressionResolver.framed]). Returns: The tree, or ``None`` once anything failed — appending to *errors* @@ -359,7 +359,10 @@ def resolve_expression( """ before = len(errors) - resolved = ExpressionResolver(ns, context, errors, formals=formals, frame=frozenset(frame)).build(node) + resolver = ExpressionResolver(ns, context, errors, formals=formals, frame=frozenset(frame)) + resolved = resolver.build(node) + if resolved is not None and frame: + resolved = resolver.framed(resolved) return None if len(errors) > before else resolved @@ -456,7 +459,10 @@ def resolve_constraint_text( found = len(errors) resolver = ExpressionResolver(ns, context, errors, frame=frozenset(frame)) left, right = resolver.build(ast.left), resolver.build(ast.right) - if len(errors) > found or left is None or right is None or resolver.met(left, right): + if len(errors) > found or left is None or right is None or (pair := resolver.aligned(left, right)) is None: + return None + left, right = resolver.framed(pair[0]), resolver.framed(pair[1]) + if left is None or right is None: return None if any(_over_the_ceiling(side, context, errors, ceiling=2) for side in (left, right)): return None diff --git a/src/mathspec/_validation.py b/src/mathspec/_validation.py index 0199eff64..ec9ef5f70 100644 --- a/src/mathspec/_validation.py +++ b/src/mathspec/_validation.py @@ -187,7 +187,6 @@ def _frame_dimensions(spec: Spec) -> Iterator[str]: for d, count in Counter(dims).items() if count > 1 ) - yield from _shared_members(kind, name, dims, spec) def _shared_members(kind: str, name: str, dims: Iterable[str], spec: Spec) -> Iterator[str]: diff --git a/src/mathspec/program.py b/src/mathspec/program.py index d784fa92f..dda99a065 100644 --- a/src/mathspec/program.py +++ b/src/mathspec/program.py @@ -753,6 +753,10 @@ class Parameter: ``dims``. Where the condition is false, the parameter is absent and a tool drops its rows of data there. ``missing`` applies where the condition is true. """ + domain: Mask | None = None + """The tuples of the dimensions with members that the file lists in ``dims:``, where the program reads the parameter + over their members, or ``None``. A tool refuses a row of data outside them, where it drops a row outside ``where``. + """ @dataclass(frozen=True) diff --git a/src/mathspec/spec.py b/src/mathspec/spec.py index c97479d65..ee8881ad1 100644 --- a/src/mathspec/spec.py +++ b/src/mathspec/spec.py @@ -1238,9 +1238,17 @@ def program(self) -> Program: so a consumer building rows reads ``spec.expand(...).program`` and refuses an entry it does not take. """ + from dataclasses import replace + + from mathspec._flat import flat_spec from mathspec._lowering import lower - return lower(self) + flat, domains = flat_spec(self) + if flat is self: + return lower(self) + built = flat.program + parameters = {n: replace(p, domain=domains[n]) if n in domains else p for n, p in built.parameters.items()} + return replace(built, parameters=Frozen(parameters)) @classmethod @override diff --git a/tests/fixtures/strict/connection.yaml b/tests/fixtures/strict/connection.yaml new file mode 100644 index 000000000..db4b7ff1a --- /dev/null +++ b/tests/fixtures/strict/connection.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: mathspec Contributors +# +# SPDX-License-Identifier: MIT + +description: Units that draw power from one bus and deliver it to others, through numbered ports. + +dimensions: + snapshot: { dtype: int, description: dispatch periods } + bus: { description: nodes of the network } + unit: { description: "converters, such as an electrolyser or a heat pump" } + port: { dtype: int, description: "an attachment of a unit, numbered as PyPSA's bus0, bus1, ..." } + role: { description: input or output } + connection: + description: the bus each port of a unit attaches to, and whether the port draws or delivers + over: [unit, port] + +mappings: + connection: { bus: { dim: bus }, role: { dim: role } } + +parameters: + capacity: { dims: [unit], description: the most a unit can draw at its input } + efficiency: + description: share of the drawn power that an output port delivers + dims: [connection, snapshot] + where: "connection[role] == 'output'" + missing: 1 + load: { dims: [bus, snapshot], description: demand to be met } + cost: { dims: [bus], description: price of power bought at a bus } + +variables: + flow: + description: power a unit draws at its input + dims: [unit, snapshot] + bounds: { lower: 0, upper: capacity } + supply: + description: power bought at a bus + dims: [bus, snapshot] + bounds: { lower: 0 } + +expressions: + injection: + description: what a port puts into its bus + dims: [connection, snapshot] + cases: + draws: { when: "connection[role] == 'input'", expression: "-at(flow, by=connection[unit])" } + otherwise: efficiency * at(flow, by=connection[unit]) + +constraints: + balance: + dims: [bus, snapshot] + expression: supply + sum(injection, over=connection, by=connection[bus]) == load + +objective: + sense: minimize + expression: sum(cost * supply) diff --git a/tests/fixtures/strict/sparse.yaml b/tests/fixtures/strict/sparse.yaml new file mode 100644 index 000000000..152d5774d --- /dev/null +++ b/tests/fixtures/strict/sparse.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: mathspec Contributors +# +# SPDX-License-Identifier: MIT + +description: >- + Units that feed buses through the ports they have, and run only in the + periods they are active: two sparse sets, read four ways. + +dimensions: + unit: { description: a converter } + port: { description: an output of a unit } + bus: { description: a node of the network } + period: { dtype: int, ordered: true, description: a step of the horizon } + connection: { over: [unit, port], description: the ports a unit has } + active: { over: [unit, period], description: the periods a unit runs in } + +mappings: + connection: + bus: { dim: bus, description: the bus a port feeds } + +parameters: + efficiency: { dims: [connection, period], missing: 1, description: share of the output a port delivers } + capacity: { dims: [unit], description: the most a unit produces in a period } + ramp: { dims: [unit], description: the most a unit's output rises from one period to the next } + load: { dims: [bus, period], description: demand to be met } + +variables: + output: { dims: [unit, period], where: active, bounds: { lower: 0 }, description: what a unit produces } + flow: { dims: [connection, period], bounds: { lower: 0 }, description: what a port takes of it } + +constraints: + port_limit: + description: a port takes no more than the unit's capacity + dims: [connection, period] + expression: flow <= at(capacity, by=connection[unit]) + split: + description: the ports of a unit take what it produces + dims: [unit, period] + expression: sum(flow, over=connection, by=connection[unit]) == output + ramp_up: + description: output rises by at most the ramp from one active period to the next + dims: [unit, period] + expression: output - shift(output, along=period, offset=1) <= ramp + balance: + description: what the ports deliver to a bus meets its load + dims: [bus, period] + expression: sum(efficiency * flow, over=connection, by=connection[bus]) >= load + +objective: + sense: minimize + expression: sum(output) diff --git a/tests/fixtures/strict/times/capacity.yaml b/tests/fixtures/strict/times/capacity.yaml new file mode 100644 index 000000000..f98e69414 --- /dev/null +++ b/tests/fixtures/strict/times/capacity.yaml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: mathspec Contributors +# +# SPDX-License-Identifier: MIT + +description: >- + Capacity transfer and capacity–activity from TIMES: EQ_CPT and the upper + form of EQ_CAPACT, for a process without vintages and without storage. + +dimensions: + region: { description: a region of the model } + period: { dtype: int, ordered: true, description: a milestone year } + vintage: { dtype: int, description: the period a capacity was built in } + process: { description: a technology } + timeslice: { description: "a slice of the year, at any level of the timeslice tree" } + upper_timeslice: { alias: timeslice, description: a timeslice that another is at or below } + rtp_vintyr: + description: the vintages of a process that exist in a period (RTP_VINTYR) + over: [region, vintage, period, process] + prc_ts: + description: the timeslices a process operates at (PRC_TS) + over: [region, process, timeslice] + ts_map: + description: each timeslice and the timeslices at or below it (TS_MAP) + over: [region, upper_timeslice, timeslice] + +parameters: + coef_cpt: { dims: [region, vintage, period, process], description: the share of a vintage that still exists in a period } + ncap_pasti: { dims: [region, vintage, process], missing: 0, description: capacity built before the first period } + coef_af_up: { dims: [region, period, process, timeslice], missing: absent, description: the upper availability factor } + prc_capact: { dims: [region, process], description: the activity a unit of capacity can produce in a year } + g_yrfr: { dims: [region, timeslice], description: the share of the year a timeslice covers } + obj_icost: { dims: [region, vintage, process], description: "the cost of a unit of new capacity, discounted" } + obj_acost: { dims: [region, period, process], description: "the cost of a unit of activity, discounted" } + +variables: + ncap: { dims: [region, vintage, process], bounds: { lower: 0 }, description: new capacity (VAR_NCAP) } + cap: { dims: [region, period, process], bounds: { lower: 0 }, description: installed capacity (VAR_CAP) } + act: + description: activity of a vintage of a process in a timeslice (VAR_ACT) + dims: [region, vintage, period, process, timeslice] + where: "prc_ts AND rtp_vintyr" + bounds: { lower: 0 } + +constraints: + eq_cpt: + description: the installed capacity of each period, from the new capacity of each vintage (EQ_CPT) + dims: [region, period, process] + expression: cap == sum(coef_cpt * (ncap + ncap_pasti), over=vintage) + eq_capact_up: + description: activity summed over the timeslices at or below each one, up to the available capacity (EQ_CAPACT, UP) + dims: [region, period, process, timeslice] + where: coef_af_up + expression: >- + sum(sum(at(sum(act, over=vintage), by=ts_map[region, timeslice]), over=ts_map, by=ts_map[region, upper_timeslice]), + over=upper_timeslice, by=upper_timeslice[timeslice]) + <= coef_af_up * prc_capact * g_yrfr * cap + +objective: + sense: minimize + expression: >- + sum(obj_icost * ncap) + + sum(obj_acost * sum(act, over=[vintage, timeslice])) diff --git a/tests/fixtures/strict/times/trade.yaml b/tests/fixtures/strict/times/trade.yaml new file mode 100644 index 000000000..c21968756 --- /dev/null +++ b/tests/fixtures/strict/times/trade.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: mathspec Contributors +# +# SPDX-License-Identifier: MIT + +description: >- + Trade between regions from TIMES: IRE_FLO, the efficiency of a flow from one + region and commodity to another through a process, over each region and + commodity twice. + +dimensions: + region: { description: a region of the model } + commodity: { description: an energy carrier or a material } + process: { description: a technology that carries trade } + from_region: { alias: region, description: the region an export leaves } + from_commodity: { alias: commodity, description: the commodity an export leaves as } + top_ire: + description: the processes that carry each link from a region and commodity to another (TOP_IRE) + over: [from_region, from_commodity, region, commodity, process] + +parameters: + ire_flo: + description: the share of an export that arrives as an import (IRE_FLO) + dims: [top_ire] + demand: { dims: [commodity, region], description: what a region needs of a commodity } + supply_cost: { dims: [commodity, region], description: the cost of a unit produced in a region } + trade_cost: + description: the cost of a unit exported along a link + dims: [top_ire] + +variables: + supply: { dims: [commodity, region], bounds: { lower: 0 }, description: what a region produces of a commodity } + export: + description: what leaves along a link through a process (VAR_IRE) + dims: [top_ire] + bounds: { lower: 0 } + +constraints: + balance: + description: production and imports less exports meet the demand of each region and commodity + dims: [commodity, region] + expression: >- + supply + + sum(ire_flo * export, over=top_ire, by=top_ire[commodity, region]) + - sum(sum(sum(export, over=top_ire, by=top_ire[from_commodity, from_region]), + over=from_region, by=from_region[region]), over=from_commodity, by=from_commodity[commodity]) + >= demand + +objective: + sense: minimize + expression: sum(supply_cost * supply) + sum(trade_cost * export) diff --git a/tests/test_dimensions.py b/tests/test_dimensions.py index b2c4ee785..403946f7a 100644 --- a/tests/test_dimensions.py +++ b/tests/test_dimensions.py @@ -673,13 +673,17 @@ def test_a_row_over_more_dims_than_the_entry_it_reads_is_refused(): def _derived(node: Expression, spec: Spec) -> frozenset[str]: - """The dims of *node* by the rules of the language reference, derived from the spec without reading ``dims``.""" + """The dims of *node* by the rules of the language reference, derived from the entries without reading ``dims``. + + An entry over members its file reads by name is over them in the program, so a leaf takes the program entry's dims. + """ if isinstance(node, Constant): return frozenset() if isinstance(node, ParameterReference): - return frozenset({**spec.parameters, **spec.given.parameters}[node.name].dims) + return frozenset({**spec.given.parameters, **spec.program.parameters}[node.name].dims) if isinstance(node, VariableReference): - return frozenset({**spec.variables, **spec.given.variables, **spec.given.expressions}[node.name].dims or ()) + declared = {**spec.given.variables, **spec.given.expressions, **spec.program.variables} + return frozenset(declared[node.name].dims or ()) if isinstance(node, Dual): return frozenset({**spec.constraints, **spec.given.constraints}[node.constraint].dims) if isinstance(node, ExpressionReference): diff --git a/tests/test_tuple_dimensions.py b/tests/test_tuple_dimensions.py index c2afeb29a..d1cf4a1b0 100644 --- a/tests/test_tuple_dimensions.py +++ b/tests/test_tuple_dimensions.py @@ -125,83 +125,6 @@ def test_a_member_is_read_as_a_map_of_the_dimension(expression, dims): assert sorted({*constraint.lhs.dims, *constraint.rhs.dims}) == sorted(dims) -@pytest.mark.parametrize( - 'patch', - [ - pytest.param(_constraint('flow <= capacity', ['connection', 'unit', 'snapshot']), id='a-constraint'), - pytest.param( - {'expressions.e': {'dims': ['connection', 'unit', 'snapshot'], 'expression': 'flow * capacity'}}, - id='a-named-expression', - ), - pytest.param( - _constraint('sum(flow * capacity, over=snapshot) <= 1', ['connection', 'unit']), - id='under-a-sum-that-keeps-both', - ), - pytest.param( - { - 'variables.both': { - 'dims': ['connection', 'unit'], - 'where': 'at(rating, by=connection[unit, port]) > capacity', - } - }, - id='a-condition', - ), - pytest.param( - { - 'constraints.probe': { - 'dims': ['connection', 'unit'], - 'expression': 'at(rating, by=connection[unit, port]) <= capacity', - 'where': 'at(rating, by=connection[unit, port]) > capacity', - } - }, - id='a-constraint-where', - ), - pytest.param( - { - 'parameters.p': { - 'dims': ['connection', 'unit'], - 'where': 'at(rating, by=connection[unit, port]) > capacity', - } - }, - id='a-parameter-where', - ), - pytest.param( - { - 'expressions.e': { - 'dims': ['connection', 'unit', 'snapshot'], - 'cases': { - 'a': { - 'when': 'at(rating, by=connection[unit, port]) > capacity', - 'expression': 'flow * capacity', - } - }, - 'otherwise': 'flow * capacity', - } - }, - id='a-case', - ), - pytest.param( - { - 'parameters.xs': {'dims': ['period']}, - 'parameters.ys': {'dims': ['period']}, - 'variables.w': {'dims': ['connection']}, - 'variables.v': {'dims': ['unit']}, - 'variables.y': {'dims': ['connection', 'unit']}, - 'piecewise.curve': { - 'dims': ['connection', 'unit'], - 'along': 'period', - 'links': {'x': ['w + v', 'xs'], 'y': ['y', 'ys']}, - }, - }, - id='a-piecewise-link', - ), - ], -) -def test_a_frame_over_the_dimension_and_its_member_is_refused(patch): - """Rule 11 of #1011 read these as the cross product; a later reading by member name would read them as a join.""" - assert "'connection' and 'unit' in dims: share the member 'unit'" in _refusal(**patch) - - _SPAN = {'dimensions.span': {'over': ['unit', 'period']}} @@ -275,27 +198,6 @@ def test_a_tuple_dimension_tests_membership_only_where_the_entry_has_every_membe assert 'outside the declared dims' in message -_STEPPED = {'dimensions.span': {'over': ['unit', 'period']}, 'variables.out': {'dims': ['span']}} - - -@pytest.mark.parametrize( - 'patch', - [ - pytest.param(_constraint('out - shift(out, along=period, offset=1) <= 1', ['span']), id='shift'), - pytest.param(_constraint('sum_back(out, along=period, window=2) <= 1', ['span']), id='sum-back'), - pytest.param( - {'constraints.probe': {'dims': ['span'], 'where': 'position(period) > 0', 'expression': 'out <= 1'}}, - id='position', - ), - ], -) -def test_a_step_along_a_member_of_an_entry_over_the_tuple_names_the_flat_entry(patch): - """The entry is over the tuple, which has no order, so the rewrite is the entry over the members, confined by the tuple.""" - message = _refusal(**_STEPPED, **patch) - assert "'period' is a member of 'span'" in message, message - assert "dims: ['period', 'unit'], with where: span" in message, message - - def test_a_condition_compares_a_member_label(): spec = to_spec(varied(MODEL, **{'variables.flow.where': "connection[unit] == 'gas'"})) (atom,) = spec.program.variables['flow'].where.atoms @@ -401,23 +303,6 @@ def test_a_sparse_set_with_an_ordered_member_is_read_at_a_tuple(): ("Sos 'pick': type: 2 along connection reads the order of 'connection'", "over ['port', 'unit']"), id='sos-of-type-2', ), - pytest.param( - {'variables.both': {'dims': ['connection', 'unit']}} - | _constraint('at(both, by=connection[unit]) >= 0', ['connection']), - ( - "Variable 'both': 'connection' and 'unit' in dims: share the member 'unit'", - 'read it as connection[unit]', - ), - id='an-operand-with-the-tuple-and-a-member', - ), - pytest.param( - _constraint('sum(flow, over=unit) >= 0', ['connection', 'snapshot']), - ( - "'unit' is a member of 'connection'", - 'sum(..., over=connection, by=connection[port])', - ), - id='over-names-a-member', - ), pytest.param( _constraint('sum(flow, over=unit, by=connection[port]) >= 0', ['port', 'snapshot']), ("over=unit names a member of 'connection'", 'sum(..., over=connection, by=connection[port])'), @@ -428,110 +313,11 @@ def test_a_sparse_set_with_an_ordered_member_is_read_at_a_tuple(): ("over=unit names a member of 'connection'", 'sum(..., over=connection, by=connection[bus])'), id='over-names-a-member-beside-a-declared-mapping', ), - pytest.param( - _constraint('sum(flow, over=[unit, port]) >= 0', ['snapshot']), - ("'unit' is a member of 'connection'", 'Write sum(..., over=connection).'), - id='over-names-every-member', - ), - pytest.param( - _constraint('sum(shipped, over=region) >= 0', ['trade']), - ("'region' is a member of 'trade'", 'Write sum(..., over=trade, by=trade[from_region]).'), - id='over-names-a-member-beside-its-alias', - ), - pytest.param( - _constraint('flow <= capacity', ['connection', 'snapshot']), - ( - "an operand over ['unit'] meets one over 'connection'", - 'at(capacity, by=connection[unit])', - 'sum(flow, over=connection, by=connection[unit])', - ), - id='a-member-operand-meets-the-tuple', - ), - pytest.param( - _constraint('flow <= capacity', ['unit', 'snapshot']), - ("meets one over 'connection'", 'sum(flow, over=connection, by=connection[unit])'), - id='a-member-operand-meets-the-tuple-in-a-frame-over-the-member', - ), - pytest.param( - {'objective': {'sense': 'minimize', 'expression': 'sum(flow * capacity)'}}, - ( - "The objective: an operand over ['unit'] meets one over 'connection'", - 'at(capacity, by=connection[unit])', - ), - id='a-member-operand-meets-the-tuple-in-a-product', - ), - pytest.param( - _constraint('sum(flow * capacity, over=unit) <= 1', ['connection', 'snapshot']), - ("meets one over 'connection'", 'at(capacity, by=connection[unit])'), - id='a-member-operand-meets-the-tuple-under-a-sum', - ), - pytest.param( - _constraint('sum(flow * capacity, over=unit) <= 1', ['connection', 'unit', 'snapshot']), - ("Constraint 'probe': 'connection' and 'unit' in dims: share the member 'unit'",), - id='a-member-operand-meets-the-tuple-under-a-sum-over-the-member-the-frame-declares', - ), - pytest.param( - _constraint('sum(flow * capacity) <= 1', ['connection', 'snapshot']), - ("meets one over 'connection'", 'at(capacity, by=connection[unit])'), - id='a-member-operand-meets-the-tuple-under-a-sum-over-everything', - ), - pytest.param( - {'assumptions.a': 'at(rating, by=connection[unit, port]) > capacity'}, - ("Assumption 'a': an operand over ['unit'] meets one over 'connection'",), - id='a-member-operand-meets-the-tuple-in-an-assumption', - ), - pytest.param( - { - 'parameters.xs': {'dims': ['period']}, - 'parameters.ys': {'dims': ['period']}, - 'variables.y': {'dims': ['connection']}, - 'piecewise.curve': { - 'dims': ['connection'], - 'along': 'period', - 'links': {'x': ['y', 'xs'], 'y': ['y', 'ys']}, - 'where': 'at(rating, by=connection[unit, port]) > capacity', - }, - }, - ("piecewise 'curve' where: an operand over ['unit'] meets one over 'connection'",), - id='a-member-operand-meets-the-tuple-in-a-piecewise-where-its-assumptions-repeat', - ), - pytest.param( - {'variables.flow.where': 'at(rating, by=connection[unit, port]) > capacity'}, - ("Variable 'flow': an operand over ['unit'] meets one over 'connection'",), - id='a-member-operand-meets-the-tuple-in-a-condition', - ), - pytest.param( - {'expressions.e': {'dims': ['unit'], 'expression': '2 * capacity'}} - | _constraint('flow <= e', ['connection', 'snapshot']), - ('at(e, by=connection[unit])',), - id='a-named-expression-over-a-member-meets-the-tuple', - ), - pytest.param( - {'expressions.e': {'dims': ['connection', 'snapshot'], 'expression': 'capacity'}}, - ("the body carries dims ['unit']", 'at(capacity, by=connection[unit])'), - id='a-named-expression-body-over-a-member', - ), - pytest.param( - { - 'expressions.e': { - 'dims': ['connection', 'snapshot'], - 'cases': {'a': {'when': "connection[unit] == 'u'", 'expression': 'capacity'}}, - 'otherwise': 'flow', - } - }, - ("the value carries dims ['unit']", 'at(capacity, by=connection[unit])'), - id='a-case-value-over-a-member', - ), pytest.param( {'variables.u': {'dims': ['unit']}} | _constraint('sum(u, over=unit, by=connection[port]) >= 0', ['port']), ('sum(at(u, by=connection[unit]), over=connection, by=connection[port])',), id='over-names-a-member-of-an-operand-without-the-dimension', ), - pytest.param( - {'variables.u': {'dims': ['unit']}} | _constraint('u <= 1', ['connection']), - ("carries dims ['unit'] that are not in its dims", 'at(u, by=connection[unit])'), - id='a-constraint-over-a-member-its-dims-lack', - ), pytest.param( {'variables.flow.bounds.upper': 'capacity'}, ( diff --git a/tests/test_tuple_sugar.py b/tests/test_tuple_sugar.py new file mode 100644 index 000000000..61caaba8d --- /dev/null +++ b/tests/test_tuple_sugar.py @@ -0,0 +1,228 @@ +# SPDX-FileCopyrightText: mathspec Contributors +# +# SPDX-License-Identifier: MIT + +"""A dimension's members read by name: each shorter spelling builds the program of the spelling it stands for. + +A spelling that names a member where the tuple is meant loads as the call +through the tuple that it means, so the program cannot tell the two apart. A +parameter over two tuples is the one exception: its data is refused outside +the rows that both list. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from mathspec import to_spec +from mathspec.program import And, Mask, RelationDefined +from tests.fixtures import EXAMPLES, varied + +#: Units on the ports they have and in the periods they run, and the sites of +#: each unit: three tuples, two of which share `unit` with `connection`. +MODEL: dict[str, Any] = { + 'dimensions': { + 'unit': {}, + 'port': {}, + 'bus': {}, + 'snapshot': {'dtype': 'int'}, + 'period': {'dtype': 'int', 'ordered': True}, + 'connection': {'over': ['unit', 'port']}, + 'active': {'over': ['unit', 'period']}, + 'site': {'over': ['unit', 'bus']}, + }, + 'parameters': { + 'capacity': {'dims': ['unit']}, + 'ramp': {'dims': ['unit']}, + 'rating': {'dims': ['bus']}, + }, + 'variables': {'flow': {'dims': ['connection', 'snapshot']}}, +} + + +def _program(**patch: Any): + return to_spec(varied(MODEL, **patch)).program + + +def _same(sugar: dict[str, Any], spelled: dict[str, Any], section: str, name: str) -> None: + built, meant = _program(**sugar), _program(**spelled) + assert getattr(built, section)[name] == getattr(meant, section)[name], 'the sugar builds what it stands for' + + +@pytest.mark.parametrize( + ('sugar', 'spelled'), + [ + pytest.param( + 'flow <= capacity', 'flow <= at(capacity, by=connection[unit])', id='a-constraint-reads-the-member' + ), + pytest.param( + 'flow * capacity <= 1', 'flow * at(capacity, by=connection[unit]) <= 1', id='a-product-reads-the-member' + ), + pytest.param('capacity >= flow', 'at(capacity, by=connection[unit]) >= flow', id='the-member-on-the-left'), + ], +) +def test_an_operand_over_a_member_meets_the_tuple_by_name(sugar, spelled): + """Was: refused, with the at() rewrite. A member operand is read at each tuple, as at(…, by=tuple[member]) reads it.""" + frame = ['connection', 'snapshot'] + _same( + {'constraints.c': {'dims': frame, 'expression': sugar}}, + {'constraints.c': {'dims': frame, 'expression': spelled}}, + 'constraints', + 'c', + ) + + +def test_a_member_meets_the_tuple_in_the_objective(): + built = _program(objective={'expression': 'sum(flow * capacity)'}).objective + meant = _program(objective={'expression': 'sum(flow * at(capacity, by=connection[unit]))'}).objective + assert built == meant, 'the objective reads the member at each tuple too' + + +@pytest.mark.parametrize( + ('dims', 'sugar', 'spelled'), + [ + pytest.param( + ['snapshot', 'unit'], + 'sum(flow, over=port)', + 'sum(flow, over=connection, by=connection[unit])', + id='one-member-summed-away', + ), + pytest.param( + ['snapshot'], 'sum(flow, over=[port, unit])', 'sum(flow, over=connection)', id='every-member-summed-away' + ), + pytest.param( + ['unit'], + 'sum(flow, over=[port, snapshot])', + 'sum(flow, over=[connection, snapshot], by=connection[unit])', + id='a-member-beside-a-plain-dimension', + ), + ], +) +def test_a_sum_over_a_member_sums_the_tuple_grouped_by_the_rest(dims, sugar, spelled): + """Was: refused, with the sum through the tuple as the rewrite.""" + _same( + {'constraints.c': {'dims': dims, 'expression': f'{sugar} <= 1'}}, + {'constraints.c': {'dims': dims, 'expression': f'{spelled} <= 1'}}, + 'constraints', + 'c', + ) + + +@pytest.mark.parametrize( + ('section', 'sugar', 'spelled'), + [ + pytest.param( + 'variables', + {'dims': ['active', 'connection']}, + {'dims': ['period', 'port', 'unit'], 'where': 'active AND connection'}, + id='a-variable', + ), + pytest.param( + 'variables', + {'dims': ['active', 'site'], 'where': 'rating > 0'}, + {'dims': ['bus', 'period', 'unit'], 'where': '(active AND site) AND (rating > 0)'}, + id='a-variable-with-its-own-where', + ), + pytest.param( + 'variables', + {'dims': ['connection', 'unit']}, + {'dims': ['port', 'unit'], 'where': 'connection'}, + id='a-tuple-beside-its-own-member', + ), + pytest.param( + 'parameters', + {'dims': ['active', 'connection']}, + {'dims': ['period', 'port', 'unit'], 'where': 'active AND connection'}, + id='a-parameter', + ), + ], +) +def test_two_tuples_that_share_a_member_are_joined_on_it(section, sugar, spelled): + """Was: refused, as an entry indexed by the shared member twice. The entry is over the members, at the rows both list.""" + built, meant = _program(**{f'{section}.e': sugar}), _program(**{f'{section}.e': spelled}) + entry, flat = getattr(built, section)['e'], getattr(meant, section)['e'] + assert entry.dims == flat.dims, 'the entry is over the members, each once' + assert entry.where == flat.where, 'and confined to the rows both tuples list' + + +def test_a_parameter_over_two_tuples_refuses_a_row_outside_both(): + """The one change no flat spelling states: a tool refuses its data row outside the joined rows, where a `where:` drops it.""" + joined = _program(**{'parameters.e': {'dims': ['active', 'connection']}}).parameters['e'] + flat = _program(**{'parameters.e': {'dims': ['period', 'port', 'unit'], 'where': 'active AND connection'}}) + listed = And(RelationDefined('active', ('period', 'unit')), RelationDefined('connection', ('port', 'unit'))) + assert joined.domain == Mask(listed), 'the joined rows are the domain' + assert flat.parameters['e'].domain is None, 'a where: is no domain: a tool drops a row outside it' + + +#: `output` over the tuples of `active`, and a ramp that steps along its ordered member. +_STEPPED = { + 'variables.output': {'dims': ['active']}, + 'constraints.ramp_up': { + 'dims': ['active'], + 'expression': 'output - shift(output, along=period, offset=1) <= ramp', + }, +} + +#: The same, over the members, as a dimension without an order must spell it. +_SPELLED = { + 'variables.output': {'dims': ['period', 'unit'], 'where': 'active'}, + 'constraints.ramp_up': { + 'dims': ['period', 'unit'], + 'where': 'active', + 'expression': 'output - shift(output, along=period, offset=1) <= ramp', + }, +} + + +@pytest.mark.parametrize(('section', 'name'), [('variables', 'output'), ('constraints', 'ramp_up')]) +def test_a_step_along_a_member_reads_the_entry_over_its_members(section, name): + """Was: refused, as a step along a member of an entry over the tuple, with the flat entry as the rewrite.""" + _same(_STEPPED, _SPELLED, section, name) + + +def test_an_entry_over_the_tuple_reads_a_stepped_one_at_each_tuple(): + """`peak` stays over `active`, and reads `output`, which a step reads over its members, at each tuple.""" + peak = {'constraints.peak': {'dims': ['active'], 'expression': 'output <= ramp'}} + spelled = 'at(output, by=active[period, unit]) <= at(ramp, by=active[unit])' + flat = {'constraints.peak': {'dims': ['active'], 'expression': spelled}} + _same(_STEPPED | peak, _SPELLED | flat, 'constraints', 'peak') + + +def test_an_entry_over_the_members_reads_a_tuple_at_them(): + """An entry over two tuples is over their members, and reads `flow`, over `connection`, at each pair of them.""" + limit = {'parameters.limit': {'dims': ['period', 'unit']}} + sugar = {'dims': ['active', 'connection', 'snapshot'], 'expression': 'flow <= limit'} + read = 'sum(flow, over=connection, by=connection[port, unit])' + spelled = { + 'dims': ['period', 'port', 'snapshot', 'unit'], + 'where': 'active AND connection', + 'expression': f'{read} <= limit', + } + _same(limit | {'constraints.c': sugar}, limit | {'constraints.c': spelled}, 'constraints', 'c') + + +#: Each example this branch shortens, beside the spelling `compare/tuple-strict` writes. +STRICT = Path(__file__).parent / 'fixtures' / 'strict' +SHORTENED = sorted(path.relative_to(STRICT) for path in STRICT.rglob('*.yaml')) + + +@pytest.mark.parametrize('path', SHORTENED, ids=str) +def test_a_shortened_example_builds_the_program_of_its_strict_spelling(path): + """Every #9 spelling keeps its meaning: the shorter file and the strict one build one program.""" + built, meant = to_spec(EXAMPLES / path).program, to_spec(STRICT / path).program + domains = {n: p.domain for n, p in built.parameters.items() if p.domain is not None} + assert not domains or set(domains) <= set(built.parameters), 'only a parameter carries a domain' + stripped = {n: _without_domain(p) for n, p in built.parameters.items()} + assert stripped == dict(meant.parameters), 'the parameters differ in their domain alone' + for section in ('variables', 'constraints', 'expressions'): + assert dict(getattr(built, section)) == dict(getattr(meant, section)), f'{section} are the strict ones' + assert built.objective == meant.objective + + +def _without_domain(parameter): + from dataclasses import replace + + return replace(parameter, domain=None) diff --git a/tests/test_validation.py b/tests/test_validation.py index 1c2005055..10382ab29 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -3310,28 +3310,6 @@ def test_a_divisor_a_shift_fills_with_another_value_loads(divisor): @pytest.mark.parametrize( ('patch', 'fragments'), [ - pytest.param( - {'variables.pair': {'dims': ['connection', 'site']}}, - ( - "Variable 'pair': 'connection' and 'site' in dims: share the member 'unit'", - 'where: "at(site, by=connection[unit])"', - "declare an alias of 'unit'", - ), - id='two-tuple-dimensions-in-a-variable', - ), - pytest.param( - {'constraints.pair': {'dims': ['connection', 'site'], 'expression': 'flow >= 0'}}, - ("Constraint 'pair': 'connection' and 'site' in dims: share the member 'unit'",), - id='two-tuple-dimensions-in-a-constraint', - ), - pytest.param( - {'parameters.rating': {'dims': ['connection', 'unit']}}, - ( - "Parameter 'rating': 'connection' and 'unit' in dims: share the member 'unit'", - 'read it as connection[unit]', - ), - id='a-tuple-dimension-beside-its-own-member', - ), pytest.param( { 'dimensions.period': {'dtype': 'int', 'ordered': True},