State the memory image of every value type a binding carries

.github/workflows/pytest.yml

@@ -96,7 +96,7 @@ jobs:
       # carries, or a change to them is not exercised until after it merges.
       # Consumers use the action; this repository owns the rules.
       - name: Refuse a skip, and a suite that shrank
-        run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 536
+        run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 546
 
       # The rules earn their place by refusing a log that carries what they
       # name. Both fixtures are written here rather than tracked, and the

GENERATION.md

@@ -45,7 +45,8 @@ different fidelity, and the choice matters:
 - the **installed** headers (`cmake --install` output) are self-contained — `meos.h` is the
   generated `meos_export.h`, with `postgres_ext_defs.in.h` spliced in place of the source
   tree's `#include <postgres.h>` — so libclang resolves every struct field to its real C type
-  and byte offset. FFI bindings need this.
+  and byte offset, and every defined struct to the bytes it is laid out in (`sizeBytes`). FFI
+  bindings need this.
 - the **source** tree headers (`<checkout>/meos/include`) parse without a build, but wrap the
   PostgreSQL types in stubs, so struct layouts are approximate.
 

docs/enrichment.md

@@ -53,7 +53,12 @@ literals are in the catalog.
     "in": "set_from_hexwkb", "in_aux": [],
     "out": "set_as_hexwkb", "out_aux": [{"name": "variant", "kind": "integer", "default": 4}],
     "bytes": { "decoder": "set_from_wkb", "encoder": "set_as_wkb",
-               "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] }
+               "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] },
+    "image": { "copy": "set_copy", "size": "set_mem_size" }
+  },
+  "Span": {
+    "...": "...",
+    "image": { "copy": "span_copy", "sizeBytes": 24 }
   }
 }
 ```
@@ -99,6 +104,17 @@ literals are in the catalog.
   `Quadbin`, `S2CellId`, each a `uint64_t`) is a class keyed by that name, read and
   written by its own functions.
 
+- **`image`** — the memory image of a value, which a binding holding values in
+  process carries instead of a wire form: `copy` is the public `T *f(const T *)` named
+  `*_copy` that duplicates a value from its image, and the image's length is either what
+  the public `int f(const T *)` named `*_mem_size` answers (`size`), or, for a type no
+  such function sizes, the bytes clang lays its structure in (`sizeBytes`, the
+  structure's own `structs[*].sizeBytes`). A class states exactly one of `size` and
+  `sizeBytes`. A structure ending in a flexible array member is variable-length, so a
+  class over one states no image without a `*_mem_size`; a class with no public copy
+  (`H3Index`, `TimestampTz`) states none either. Two copy or size functions for one
+  class stop the catalog.
+
 The class's `encodings`, `in` and `out` are folded onto each `structs[*]` entry as
 `serialization`.
 

parser/codecs.py

@@ -25,10 +25,15 @@
   by name and refuses one the catalog does not fill.
 - A value whose slot carries a ``typedef`` (``H3Index``, ``Quadbin``, ``S2CellId``) is a
   class of its own, keyed by that name.
+- A class whose values MEOS copies states their memory image in ``image``: the copy
+  function and either the function answering an image's length (``size``) or, for a
+  fixed-size structure, the bytes it is laid out in (``sizeBytes``), so a binding holding
+  values in process reads both from the catalog (#_image).
 """
 import re
 
-from parser.enrich import _DECODERS, _ENCODERS, _STRING_PTR_BASES, _aux_specs, _base
+from parser.enrich import (_DECODERS, _ENCODERS, _INT_BASES, _STRING_PTR_BASES, _aux_specs,
+                           _base, _ptr_depth)
 
 _ORDER = ("text", "mfjson", "wkb")
 
@@ -181,6 +186,7 @@ def resolve(cls, enc, cands, prior, sql_type, side, table):
         return (cands[0] if len(cands) == 1 else None), {}
 
     every = {f["name"]: f for f in idl.get("functions", [])}
+    structs = {s["name"]: s for s in idl.get("structs", [])}
     for cls in sorted(set(encs) | {c for c in decoders if c in encs or _is_identity(c, fns)}):
         e = encs.setdefault(cls, {"encodings": [], "decoders": {}, "encoders": {}})
         prior_dec, prior_enc = dict(e.get("decoders") or {}), dict(e.get("encoders") or {})
@@ -230,6 +236,11 @@ def resolve(cls, enc, cands, prior, sql_type, side, table):
         if "bytes" in e:
             v = _variant(cls, fns, macros, errors)
             e["bytes"]["encoderAux"] = [{"name": "variant", "kind": "integer", "default": v}]
+        image = _image(cls, fns, structs, errors)
+        if image:
+            e["image"] = image
+        else:
+            e.pop("image", None)
 
     for struct in idl.get("structs", []):
         te = encs.get(struct["name"])
@@ -253,6 +264,45 @@ def _byte_codec(cls, fns):
     return None
 
 
+def _image(cls, fns, structs, errors):
+    """{copy, size} or {copy, sizeBytes}: the memory image of a value of ``cls``, which a
+    binding holding values in process carries instead of a wire form. ``copy`` is the
+    public ``T *f(const T *)`` named ``_copy`` that duplicates a value from its image;
+    the image's length is what the public ``int f(const T *)`` named ``_mem_size``
+    answers (``size``), or, for a type no such function sizes, the bytes clang lays its
+    structure in (``sizeBytes`` of #extract_struct of parser/extractors.py). A structure
+    ending in a flexible array member is variable-length, so its laid-out size is not
+    its image and the type states none without a ``_mem_size``. Found by shape and name
+    as #_byte_codec finds the byte codec."""
+    def one(names, what):
+        if len(names) > 1:
+            errors.append(f"{cls}: {what} by {', '.join(sorted(names))}")
+            return None
+        return names[0] if names else None
+
+    copy = one([f["name"] for f in fns if f["name"].endswith("_copy")
+                and len(f.get("params") or []) == 1
+                and _ptr_depth(f["returnType"].get("c") or "") == 1
+                and _cls(f["returnType"]) == cls
+                and _ptr_depth(f["params"][0].get("cType") or "") == 1
+                and _cls(f["params"][0]) == cls], "copied")
+    if not copy:
+        return None
+    size = one([f["name"] for f in fns if f["name"].endswith("_mem_size")
+                and len(f.get("params") or []) == 1
+                and _ptr_depth(f["returnType"].get("c") or "") == 0
+                and _base(f["returnType"].get("canonical") or "") in _INT_BASES
+                and _ptr_depth(f["params"][0].get("cType") or "") == 1
+                and _cls(f["params"][0]) == cls], "sized")
+    if size:
+        return {"copy": copy, "size": size}
+    struct = structs.get(cls) or {}
+    if "sizeBytes" in struct and not any(fld["cType"].endswith("[]")
+                                         for fld in struct.get("fields") or ()):
+        return {"copy": copy, "sizeBytes": struct["sizeBytes"]}
+    return None
+
+
 def _is_identity(cls, fns):
     """Whether ``cls`` is a type of its own some slot names (``H3Index``)."""
     return any(p.get("typedef") == cls for f in fns

parser/extractors.py

@@ -259,11 +259,19 @@ def extract_function(node) -> dict:
 
 
 def extract_struct(node) -> dict:
+    """A structure, its fields at the offset clang lays each at and, when the unit
+    defines it, the bytes clang lays the whole record in (`sizeBytes`), the trailing
+    padding included: the image of a value whose type is the structure (a `Span`, an
+    `STBox`) is that many bytes. A structure only declared (`struct NumericData;`) has
+    no size and states none, as #extract_enum states a value's `doc` only where the
+    value carries one."""
+    size = node.type.get_size()
     return {
         "name": node.spelling,
         "file": Path(node.location.file.name).name,
         "family": _family_of(node.location.file.name, node.location.line),
         "vendored": _is_vendored(node.location.file.name),
+        **({"sizeBytes": size} if size >= 0 else {}),
         "fields": [
             {
                 "name": f.spelling,

tests/test_codecs.py

@@ -177,6 +177,62 @@ def test_two_functions_alike_for_one_type_stop_the_catalog(self):
         self.assertEqual(errors, ["Set: intset read by intset_in and intsetx_in"])
 
 
+class ImageTests(unittest.TestCase):
+    """The memory image #_image of parser/codecs.py states for a class over the synthetic
+    catalog of #CodecTests, with the copy and size functions each case adds."""
+
+    SPAN = {"name": "Span", "sizeBytes": 24,
+            "fields": [{"name": "spantype", "cType": "uint8", "offset_bits": 0}]}
+    TEXT = {"name": "text", "sizeBytes": 4,
+            "fields": [{"name": "vl_len_", "cType": "char[4]", "offset_bits": 0},
+                       {"name": "vl_dat", "cType": "char[]", "offset_bits": 32}]}
+
+    def _image(self, extra, structs=(), classes=()):
+        idl = _idl(FUNCTIONS + extra)
+        idl["structs"] += [dict(s) for s in structs]
+        for cls in classes:
+            idl["typeEncodings"][cls] = {}
+        idl, errors = state_type_encodings(idl)
+        return idl["typeEncodings"], errors
+
+    def test_a_variable_length_value_states_its_copy_and_its_size_function(self):
+        te, errors = self._image([
+            fn("set_copy", "Set *", [(SET, "s")]),
+            fn("set_mem_size", "int", [(SET, "s")]),
+            fn("raster_copy", "Raster *", [(RAST, "rast")]),
+            fn("raster_mem_size", "int", [(RAST, "rast")])])
+        self.assertEqual(te["Set"]["image"], {"copy": "set_copy", "size": "set_mem_size"})
+        self.assertEqual(te["Raster"]["image"],
+                         {"copy": "raster_copy", "size": "raster_mem_size"})
+        self.assertEqual(errors, [])
+
+    def test_a_fixed_size_value_states_its_copy_and_its_structure_size(self):
+        te, errors = self._image([fn("span_copy", "Span *", [("const Span *", "s")])],
+                                 [self.SPAN], ["Span"])
+        self.assertEqual(te["Span"]["image"], {"copy": "span_copy", "sizeBytes": 24})
+        self.assertEqual(errors, [])
+
+    def test_a_flexible_array_structure_without_a_size_function_states_no_image(self):
+        te, _ = self._image([fn("text_copy", "text *", [("const text *", "txt")])],
+                            [self.TEXT], ["text"])
+        self.assertNotIn("image", te["text"])
+
+    def test_no_copy_no_image(self):
+        te, _ = self._image([fn("set_mem_size", "int", [(SET, "s")])])
+        self.assertNotIn("image", te["Set"])
+        self.assertNotIn("image", te["H3Index"])
+
+    def test_an_internal_copy_is_no_image(self):
+        te, _ = self._image([fn("set_copy", "Set *", [(SET, "s")], api="internal"),
+                             fn("set_mem_size", "int", [(SET, "s")])])
+        self.assertNotIn("image", te["Set"])
+
+    def test_two_copies_of_one_class_stop_the_catalog(self):
+        _, errors = self._image([fn("set_copy", "Set *", [(SET, "s")]),
+                                 fn("intset_copy", "Set *", [(SET, "s")])])
+        self.assertEqual(errors, ["Set: copied by intset_copy, set_copy"])
+
+
 class CodecContractTests(unittest.TestCase):
     """Over the generated catalog: every trailing input a codec states names a parameter
     of its function, carries the value a binding passes, and is never an out-parameter."""
@@ -222,6 +278,35 @@ def test_the_classes_a_binding_reads(self):
         for cell in ("H3Index", "Quadbin", "S2CellId"):
             self.assertIn(cell, te)
 
+    def test_the_images_a_binding_reads(self):
+        """The image of #ImageTests over the generated catalog: a value sized by a
+        function, and one whose image is its structure, at the size the structure
+        states."""
+        te = self.idl["typeEncodings"]
+        sizes = {s["name"]: s.get("sizeBytes") for s in self.idl["structs"]}
+        self.assertEqual(te["Temporal"]["image"],
+                         {"copy": "temporal_copy", "size": "temporal_mem_size"})
+        self.assertEqual(te["Raster"]["image"],
+                         {"copy": "raster_copy", "size": "raster_mem_size"})
+        for cls, copy in (("Span", "span_copy"), ("STBox", "stbox_copy"),
+                          ("Npoint", "npoint_copy"), ("Nsegment", "nsegment_copy")):
+            self.assertEqual(te[cls]["image"], {"copy": copy, "sizeBytes": sizes[cls]})
+
+    def test_every_image_names_public_functions_of_its_class(self):
+        bad = []
+        for cls, e in self.idl["typeEncodings"].items():
+            image = e.get("image")
+            if not image:
+                continue
+            for key in ("copy", "size"):
+                f = self.fns.get(image.get(key)) if key in image else None
+                if key in image and (f is None or f.get("api") != "public"
+                                     or len(f["params"]) != 1):
+                    bad.append((cls, key, image[key]))
+            if ("size" in image) == ("sizeBytes" in image):
+                bad.append((cls, "size xor sizeBytes", image))
+        self.assertEqual(bad, [])
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_struct_layout.py

@@ -79,6 +79,22 @@ def test_core_structs_have_real_offsets(self):
                     fld["offset_bits"], 0,
                     f"{s['name']}.{fld['name']} has unresolved offset")
 
+    def test_a_fixed_size_value_states_the_bytes_clang_lays_it_in(self):
+        """The size beside the offsets #test_span_layout_resolved reads: the image of
+        these values is their structure, the trailing padding included (a TBox is two
+        24-byte spans and an int16, padded to 56)."""
+        sizes = {n: self.by_name[n].get("sizeBytes")
+                 for n in ("Span", "TBox", "STBox", "Npoint", "Nsegment")}
+        self.assertEqual(sizes, {"Span": 24, "TBox": 56, "STBox": 80,
+                                 "Npoint": 16, "Nsegment": 24})
+
+    def test_a_defined_structure_states_its_size_and_an_opaque_one_none(self):
+        """Over every structure, as #test_core_structs_have_real_offsets reads every
+        field: one with fields is defined and sized, one without only declared."""
+        bad = [s["name"] for s in self.by_name.values()
+               if ("sizeBytes" in s) != bool(s["fields"])]
+        self.assertEqual(bad, [])
+
     def test_a_structure_declared_first_is_read_from_its_definition(self):
         # `meos.h` declares MeosArray and SkipList before `meos_internal.h` defines
         # them, so their layout is read where they are defined
@@ -105,6 +121,10 @@ def test_the_definition_wins_and_an_opaque_structure_stays(self):
             idl = parse_all_headers(root)
         structs = {s["name"]: [f["name"] for f in s["fields"]] for s in idl["structs"]}
         self.assertEqual(structs, {"S": ["x", "y"], "O": []})
+        # the definition states the bytes clang lays it in, its padding included; the
+        # opaque structure states no size
+        sizes = {s["name"]: s.get("sizeBytes") for s in idl["structs"]}
+        self.assertEqual(sizes, {"S": 16, "O": None})
 
 
 if __name__ == "__main__":