Derive the catalog of a stable release line

parser/aggregates.py

@@ -30,6 +30,7 @@
 deserialize function over `bytea` and the state."""
 import re
 
+from parser.families import built_files
 from parser.compositions import _type
 from parser.sqlfn import _split_top_commas, _strip_sql_comments, sql_signature, sql_statements
 
@@ -67,7 +68,7 @@ def _create_agg_stmts(text):
 def _agg_statements(sql_src):
     """Every CREATE AGGREGATE under `sql_src`, as #_create_agg_stmts yields it."""
     out = []
-    for sf in sorted(sql_src.rglob("*.sql")):
+    for sf in sorted(built_files(sql_src, "*.sql")):
         out.extend(_create_agg_stmts(_strip_sql_comments(sf.read_text(errors="ignore"))))
     return out
 

parser/altsqlfn.py

@@ -49,6 +49,7 @@
 import re
 from pathlib import Path
 
+from parser.families import built_files
 from parser.sqlfn import _CSQLFN_END, _meos_to_mdb, _wrapper_sql_sigs
 from parser.typescope import SQL_ALIASES, TypeFacts
 
@@ -77,7 +78,7 @@ def wrapper_names(mdb_src: str | Path) -> dict[str, tuple[list[str], list[str]]]
     and no ``@sqlaggfn``: on a wrapper stating ``@sqlaggfn`` the tag names the aggregate
     (#aggregate_alt_names)."""
     out = {}
-    for cf in sorted(Path(mdb_src).rglob("*.c")):
+    for cf in sorted(built_files(mdb_src, "*.c")):
         for m in _WRAPPER_BLOCK.finditer(cf.read_text(errors="ignore")):
             alts = _tag_names(m.group(1), "altsqlfn")
             if alts and not _tag_names(m.group(1), "sqlaggfn"):
@@ -91,7 +92,7 @@ def aggregate_alt_names(mdb_src: str | Path) -> tuple[dict[str, str], list[str]]
     pairs several ``@sqlfn`` names. A wrapper whose two lists differ in length, or an aggregate
     two wrappers name differently, is an error."""
     out, errors = {}, []
-    for cf in sorted(Path(mdb_src).rglob("*.c")):
+    for cf in sorted(built_files(mdb_src, "*.c")):
         for m in _WRAPPER_BLOCK.finditer(cf.read_text(errors="ignore")):
             alts = _tag_names(m.group(1), "altsqlfn")
             aggs = _tag_names(m.group(1), "sqlaggfn")

parser/boundargs.py

@@ -66,6 +66,8 @@
 import re
 from pathlib import Path
 
+from parser.families import built_files
+
 # `Datum <Name>(PG_FUNCTION_ARGS) {` opens a PG wrapper.
 _WRAP = re.compile(r"Datum\s+(?P<name>\w+)\s*\(\s*PG_FUNCTION_ARGS\s*\)\s*\{")
 # Any local the wrapper ASSIGNS (`var = ...`, excluding `==`): every value the
@@ -150,7 +152,7 @@ def _call_args(body: str, fn: str) -> list[str] | None:
 def extract_wrappers(mdb_src: str | Path) -> dict[str, str]:
     """Return ``{wrapper_name: body_text}`` for every PG wrapper under ``mdb_src``."""
     out: dict[str, str] = {}
-    for cf in Path(mdb_src).rglob("*.c"):
+    for cf in built_files(mdb_src, "*.c"):
         text = cf.read_text(errors="ignore")
         for m in _WRAP.finditer(text):
             out[m.group("name")] = _body(text, m.end() - 1)
@@ -161,7 +163,7 @@ def extract_helpers(mdb_src: str | Path) -> dict[str, tuple[str, list[str]]]:
     """``{helper_name: (body_text, [param names after the call info])}`` for every shared
     wrapper helper under ``mdb_src``."""
     out: dict[str, tuple[str, list[str]]] = {}
-    for cf in Path(mdb_src).rglob("*.c"):
+    for cf in built_files(mdb_src, "*.c"):
         text = cf.read_text(errors="ignore")
         for m in _HELPER.finditer(text):
             rest = (m.group("rest") or "").strip()

parser/compositions.py

@@ -43,6 +43,7 @@
 import re
 from itertools import permutations
 
+from parser.families import built_files
 from parser.sqlfn import _c_base, _fits, _sql_ctypes, sql_signature, sql_statements
 from parser.typescope import SQL_ALIASES
 
@@ -67,6 +68,11 @@ class _Misfit(Exception):
     """A body outside the form of a composition."""
 
 
+class _Unformed(Exception):
+    """A call reaching a body that is no composition: one composed over functions
+    MobilityDB does not declare, or a program outside the form a composition takes."""
+
+
 class _Parser:
     """The expressions of a composition body: `$k`, a literal, `f(e, ...)`, `e::T` and a
     row column `r.c`, in a `SELECT e, ... [FROM f(...) AS r]`."""
@@ -166,6 +172,13 @@ def __init__(self, idl, stmts, vocab, composites, text_casts):
         # whose signatures the catalog carries, read from the same statements.
         self.declared = {stmt[0].lower() for stmt in stmts} | set(self.by_sig)
         self.done, self.active = {}, set()
+        # The signatures whose bodies are no composition, directly (stable-1.3's `rotate`,
+        # a program over `cos` and `-sin`) or through a call of such a body (`rotateZ`).
+        self.unformed = set()
+        # The entry of each plain forward: no composition of its own, but a body calling it
+        # composes through it (stable-1.3's `translate(t, x, y)` through `translate(t, x,
+        # y, 0)`, a forward to `affine` binding nine of its arguments).
+        self.forwards = {}
 
     def foreign(self, node):
         """Whether expression `node` calls a function MobilityDB does not declare (a
@@ -264,6 +277,9 @@ def compose(self, sig, body):
         self.active.add(key)
         try:
             entry = self._compose(sig, body)
+        except _Unformed:
+            self.unformed.add(key)
+            entry = None
         finally:
             self.active.discard(key)
         self.done[key] = entry
@@ -315,6 +331,12 @@ def call(self, node, sig):
             return (self.params(func, csig, ops), func["name"], csig["ret"],
                     csig.get("columns"), None)
         inner = self.composition(name, types)
+        if inner is None:
+            fits = [sig for sig, _ in self.bodies.get(name.lower(), ()) if self._fits(sig, types)]
+            key = (fits[0]["sqlName"].lower(), tuple(fits[0]["args"])) if len(fits) == 1 else None
+            if key in self.unformed:
+                raise _Unformed
+            inner = self.forwards.get(key)
         if inner is None:
             raise _Misfit(f"{name}({', '.join(t or '?' for t in types)}) resolves to "
                           f"neither a C-backed signature nor a composition")
@@ -341,12 +363,15 @@ def call(self, node, sig):
         return merged, inner.get("call"), inner["ret"], inner.get("columns"), restore
 
     def _compose(self, sig, body):
+        # A body that is no composition raises #_Unformed, which #compose records, so that
+        # a body calling it is no composition either.
         try:
             items, source, alias = _Parser(body).statement()
         except _Misfit:
-            return None                                         # a program, not a form
+            raise _Unformed                                     # a program, not a form
         if any(self.foreign(n) for n in items + ([source] if source else [])):
-            return None                                         # composed over PostGIS
+            raise _Unformed                                     # composed over PostGIS
+        key = (sig["sqlName"].lower(), tuple(sig["args"]))
         restore = None
         if source is not None:
             # SELECT r.a, [r.b, ...] at($k, time(r.c)) FROM f(...) AS r
@@ -382,9 +407,13 @@ def _compose(self, sig, body):
                     try:
                         if self.function(top[1], [self.operand(a, sig)["type"]
                                                   for a in top[2]]) is not None:
+                            ops, fn, _, _, restore = self.call(top, sig)
+                            self.forwards[key] = self._entry(sig, ops, fn, restore)
                             return None
                     except _Misfit:
-                        return None
+                        # a forward stating no call of a C function (`affine`, whose
+                        # wrapper packs twelve floats into one AFFINE argument)
+                        raise _Unformed
                 ops, fn, cret, _, restore = self.call(top, sig)
             elif top[0] in ("arg", "cast"):
                 ops, fn = [self.operand(top, sig)], None
@@ -395,12 +424,19 @@ def _compose(self, sig, body):
                 return None
         # A call reaching a composition takes its casts, so it converts as well.
         if not (any(o.get("casts") for o in ops) or restore):
+            self.forwards[key] = self._entry(sig, ops, fn, restore)
             return None                                         # a plain forward
         used = {o["arg"] for o in ops if "arg" in o} | ({restore["of"]} if restore else set())
         if used != set(range(len(sig["args"]))):
             raise _Misfit(f"arguments {sorted(set(range(len(sig['args']))) - used)} unused")
         if restore is None and _type(cret or "") != _type(sig["ret"] or ""):
             raise _Misfit(f"answers {cret} where the function returns {sig['ret']}")
+        return self._entry(sig, ops, fn, restore)
+
+    @staticmethod
+    def _entry(sig, ops, fn, restore):
+        """The composition entry of `sig` calling `fn` over `ops`, as #_compose returns
+        it and keeps it for a plain forward."""
         entry = {k: sig[k] for k in ("sqlName", "args", "required", "argDefaults", "ret")}
         if sig.get("retSet"):
             entry["retSet"] = True
@@ -462,10 +498,9 @@ def _text_casts(sql_src):
     """{(source type, target type): cast function SQL name} of every `CREATE CAST ...
     WITH FUNCTION f(source)` under `sql_src`; a cast whose function takes more than the
     source (a typmod) is left out."""
-    from pathlib import Path
     from parser.sqlfn import _strip_sql_comments
     out = {}
-    for sf in sorted(Path(sql_src).rglob("*.sql")):
+    for sf in sorted(built_files(sql_src, "*.sql")):
         text = _SCHEMA.sub("", _strip_sql_comments(sf.read_text(errors="ignore")))
         for src, dst, fn, fargs in _CAST.findall(text):
             if len([a for a in fargs.split(",") if a.strip()]) == 1:

parser/families.py

@@ -38,16 +38,35 @@ class FamiliesUnavailable(RuntimeError):
     """
 
 
+# A family stated as its own option, as a MobilityDB release line before the
+# `ALL` list states each one: `option(CBUFFER` on a line of its own, or
+# `CMAKE_DEPENDENT_OPTION(RGEO` for a family that needs another one.
+_OPTION_RE = re.compile(r"^(?:option|CMAKE_DEPENDENT_OPTION)\(\s*([A-Z][A-Z0-9_]*)\b", re.M)
+
+
 def read_all_families(cmakelists: Path) -> tuple[str, ...]:
-    """The family tokens named by ``cmakelists``'s ``if(ALL) foreach`` list."""
+    """The family tokens of ``cmakelists``: its ``if(ALL) foreach`` list, or, in a
+    tree that has none, each ``option(<FAMILY>)`` whose ``meos/src/<family>``
+    directory holds the family's sources.
+
+    A MobilityDB release line older than the ``ALL`` list (stable-1.3) states
+    every family as an option of its own. Its options also name what is no
+    family -- ``MEOS``, ``GEOS`` -- and the source directory a family lives in,
+    spelled from the token as #subdir_family spells it, tells the two apart,
+    read from the tree rather than listed here."""
     text = Path(cmakelists).read_text(errors="ignore")
     m = _ALL_BLOCK_RE.search(text)
-    if not m:
+    if m:
+        return tuple(sorted(re.findall(r"[A-Z][A-Z0-9_]*", m.group(1))))
+    src = Path(cmakelists).parent / "meos" / "src"
+    stated = [t for t in _OPTION_RE.findall(text) if (src / t.lower()).is_dir()]
+    if not stated:
         raise FamiliesUnavailable(
-            f"{cmakelists} carries no `if(ALL) foreach(_family ...)` list — "
-            "the family source of truth moved; update parser/families.py to "
-            "read it where it now lives")
-    return tuple(sorted(re.findall(r"[A-Z][A-Z0-9_]*", m.group(1))))
+            f"{cmakelists} carries no `if(ALL) foreach(_family ...)` list and no "
+            "`option(<FAMILY>)` with a meos/src/<family> directory — the family "
+            "source of truth moved; update parser/families.py to read it where "
+            "it now lives")
+    return tuple(sorted(set(stated)))
 
 
 _headers_dir: Path | None = None
@@ -117,3 +136,33 @@ def subdir_family() -> dict:
 def header_family() -> dict:
     """Top-level public header name -> family token."""
     return {f"meos_{f.lower()}.h": f for f in all_families()}
+
+
+def unbuilt_families() -> frozenset:
+    """The families the header tree being parsed was built without.
+
+    A libmeos built without a family installs no public header of it: the stable-1.3
+    default build installs ``meos_npoint.h`` but not ``meos_cbuffer.h``. A family is
+    unbuilt when the checkout has its header, by #header_family, and the parsed tree
+    does not, so the source tree itself and a build of every family leave none out, and
+    a family fronted by no header of its own is never left out.
+    """
+    root = find_mobilitydb_root()
+    if root is None or _headers_dir is None:
+        return frozenset()
+    source = root / "meos" / "include"
+    if source.resolve() == _headers_dir.resolve():
+        return frozenset()
+    return frozenset(f for h, f in header_family().items()
+                     if (source / h).exists() and not (_headers_dir / h).exists())
+
+
+def built_files(root, pattern: str):
+    """The files under ``root`` matching ``pattern``, in ``rglob`` order, less those
+    inside the directory of an #unbuilt_families family (``mobilitydb/sql/cbuffer``,
+    ``mobilitydb/src/cbuffer``): the SQL and the wrappers of a family the parsed headers
+    do not carry state signatures no catalog function has."""
+    root = Path(root)
+    skip = {f.lower() for f in unbuilt_families()}
+    return [p for p in root.rglob(pattern)
+            if not skip & set(p.relative_to(root).parts[:-1])]

parser/indexsearch.py

@@ -30,6 +30,7 @@
 import re
 from pathlib import Path
 
+from parser.families import built_files
 from parser.sqlfn import _strip_sql_comments
 
 # `CREATE OPERATOR <symbol> (` opens an operator declaration; its clauses run to the `)` the
@@ -54,7 +55,7 @@ def operator_decls(sql_src: str | Path) -> dict[tuple[str, str, str], list[tuple
     sql_src = Path(sql_src)
     if not sql_src.exists():
         return decls
-    for sf in sorted(sql_src.rglob("*.sql")):
+    for sf in sorted(built_files(sql_src, "*.sql")):
         text = _strip_sql_comments(sf.read_text(errors="ignore"))
         for m in _CREATE_OP.finditer(text):
             end = text.find(";", m.end())

parser/object_model.py

@@ -27,7 +27,8 @@ class is invented; a function with no prefix match is recorded honestly as
 import re
 from pathlib import Path
 
-from parser.typerelations import locate_catalog, temptype_basetypes
+from parser.typerelations import (MEOSTYPE_PARAM, locate_catalog, temptype_basetypes,
+                                  type_names)
 
 
 def find_mobilitydb_src(headers_dir: Path | None = None) -> Path | None:
@@ -373,7 +374,7 @@ class MembershipUnavailable(RuntimeError):
 
 def _predicate_body(cat_src: str, name: str) -> str:
     """The body of the ``name(MeosType ...)`` membership predicate."""
-    m = re.search(r"\n" + re.escape(name) + r"\(MeosType \w+\)\s*", cat_src)
+    m = re.search(r"\n" + re.escape(name) + r"\(" + MEOSTYPE_PARAM + r" \w+\)\s*", cat_src)
     if not m:
         raise MembershipUnavailable(
             f"meos_catalog.c declares no `{name}` predicate — the lattice names "
@@ -428,6 +429,22 @@ def predicate_temptypes(cat_src: str, name: str) -> list:
     return out
 
 
+def release_nodes(nodes: dict, cat_src: str) -> dict:
+    """The nodes of the model the release line of ``cat_src`` has.
+
+    The model names every type MEOS has on master; a release line older than a type
+    (stable-1.3 has no tjsonb or tbigint) does not list it in ``MEOS_TYPE_NAMES``, so the
+    leaf modelling it is not a class of that line. A leaf whose type the line lists but
+    does not relate is a disagreement, which #derive_membership raises; a catalog whose
+    type names do not parse keeps every node, so that the derivation reports it.
+    """
+    named = type_names(cat_src)
+    if not named:
+        return nodes
+    return {k: v for k, v in nodes.items()
+            if v.get("kind") != "leaf" or all(t in named for t in v.get("temptypes") or ())}
+
+
 def derive_membership(nodes: dict, cat_src: str, basetypes: dict) -> None:
     """Fill each node's membership from the catalog, in place.
 
@@ -531,6 +548,7 @@ def attach_object_model(idl: dict, path: Path,
     if catalog is not None:
         cat_src = catalog.read_text(errors="ignore")
         basetypes = temptype_basetypes(cat_src)
+        lattice_nodes = release_nodes(lattice_nodes, cat_src)
         derive_membership(lattice_nodes, cat_src, basetypes)
         derive_membership(trait_nodes, cat_src, basetypes)
         membership = {"status": "derived", "source": str(catalog)}

parser/portable.py

@@ -72,9 +72,12 @@ def attach_position_names(idl: dict) -> dict:
     the name less its stem. Adds ``portableAliases.positionNames``: operator -> class
     -> SQL name.
 
-    A name that does not end with its operator's stem, or an operator that no
-    function carries, raises: either means the @sqlfn/@sqlop tags and the mapping
-    disagree, which a binding would otherwise inherit silently.
+    A release line spells the names one way throughout: master's ``<class><Stem>``
+    (spanLeft), or the ``<class>_<stem>`` of a release before the class names
+    (span_left, temporal_left on stable-1.3). The spelling of the catalog is the one
+    its names take; a name in neither form, or in the other one, raises, as does an
+    operator that no function carries: each means the @sqlfn/@sqlop tags and the
+    mapping disagree, which a binding would otherwise inherit silently.
 
     MUST run AFTER ``attach_sqlfn_map`` (sqlfn/sqlop) AND ``attach_portable_aliases``
     (byPositionOperator).
@@ -84,17 +87,28 @@ def attach_position_names(idl: dict) -> dict:
         return idl
     by_position_operator = pa["byPositionOperator"]
     names = {op: {} for op in by_position_operator}
+    spelled = {}                      # spelling -> the first function taking it
     for f in idl.get("functions", []):
         op = (f.get("sqlop") or "").replace("\\", "")
         sqlfn = f.get("sqlfn") or ""
         if op not in by_position_operator or not sqlfn:
             continue
         pos = by_position_operator[op]
-        suffix = pos[0].upper() + pos[1:]
-        cls = sqlfn[:-len(suffix)] if sqlfn.endswith(suffix) else ""
+        cls = ""
+        for spelling, suffix in (("<class><Stem>", pos[0].upper() + pos[1:]),
+                                 ("<class>_<stem>", "_" + pos)):
+            if sqlfn.endswith(suffix) and len(sqlfn) > len(suffix):
+                cls = sqlfn[:-len(suffix)]
+                break
         if not cls:
             raise ValueError(f"portable-aliases: {f['name']} has @sqlop {op} and "
-                             f"@sqlfn {sqlfn}, which is not <class>{suffix}")
+                             f"@sqlfn {sqlfn}, which is not "
+                             f"<class>{pos[0].upper() + pos[1:]} or <class>_{pos}")
+        spelled.setdefault(spelling, f["name"])
+        if len(spelled) > 1:
+            other = next(s for s in spelled if s != spelling)
+            raise ValueError(f"portable-aliases: {f['name']} spells @sqlfn {sqlfn} as "
+                             f"{spelling}, {spelled[other]} as {other}")
         names[op][cls] = sqlfn
     missing = sorted(op for op, by_class in names.items() if not by_class)
     if missing: