From fb13d77b673860d91ed8bc7405d0762de731c065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Zim=C3=A1nyi?= Date: Sun, 11 Oct 2026 13:46:46 +0200 Subject: [PATCH] State the memory image of every value type a binding carries Every defined structure states the bytes clang lays it in, its trailing padding included, as sizeBytes beside the offset_bits of its fields: extract_struct of parser/extractors.py reads it from the record type clang already lays out for the offsets, so a Span states 24, a TBox 56, an STBox 80, an Npoint 16 and an Nsegment 24. A structure only declared, such as Raster or Pose, has no size and states none. Each class of typeEncodings that MEOS copies states the memory image of its values in image: copy names the public T *f(const T *) named *_copy, and the image's length is either the public int f(const T *) named *_mem_size, as size, or, for a type no such function sizes, the sizeBytes of its structure. _image of parser/codecs.py finds both by shape and name, as _byte_codec finds the byte codec, and stops the catalog on two copy or two size functions for one class. A structure ending in a flexible array member is variable-length, so a class over one states no image without a *_mem_size. Temporal states {copy: temporal_copy, size: temporal_mem_size}, Raster {copy: raster_copy, size: raster_mem_size} and Span {copy: span_copy, sizeBytes: 24}; the classes MEOS passes by value or copies through no public function, H3Index and TimestampTz among them, state none. The floor of .github/workflows/pytest.yml rises to 546, the suite collecting the two size tests of StructLayoutTests, the six of ImageTests and the two image tests of CodecContractTests. Witness: against MEOS-API master 4baf117 over MobilityDB master 4eef012a9f the Span structure states the offsets of its fields and no size, and typeEncodings states no copy or size of any class; against this branch Span states sizeBytes 24, typeEncodings.Span.image reads {copy: span_copy, sizeBytes: 24} and typeEncodings.Raster.image {copy: raster_copy, size: raster_mem_size}. Measured over MobilityDB master 4eef012a9f with an all-families libmeos, run.py on upstream 4baf117 and on this branch with the same installed headers: the two catalogs differ in added keys alone, every key of every function and of the object model being identical; 71 of the 83 structures state sizeBytes, the 12 others being only declared; 21 of the 34 classes of typeEncodings state an image, 13 sized by a function (Cbuffer, GSERIALIZED, JsonPath, Jsonb, Pcpatch, Pcpoint, Pose, PoseChain, Raquet, Raster, Set, SpanSet, Temporal) and 8 by their structure (Interval 16, Npoint 16, Nsegment 24, STBox 80, Span 24, TBox 56, TPCBox 88, TimeTzADT 16). The suite collects 546 tests, all passing, none skipped. JMEOS main e8c65dc4 generates byte-identical sources from both catalogs: its GeneratedFunctions and object layer, and its Spark, Flink, Kafka, Spark SQL and Flink SQL surfaces. Why: a binding holding MEOS values in process carries each as its memory image, allocated to the size the catalog states and duplicated by the copy it names, and reads both from the catalog rather than from a table of its own. --- .github/workflows/pytest.yml | 2 +- GENERATION.md | 3 +- docs/enrichment.md | 18 +++++++- parser/codecs.py | 52 +++++++++++++++++++++- parser/extractors.py | 8 ++++ tests/test_codecs.py | 85 ++++++++++++++++++++++++++++++++++++ tests/test_struct_layout.py | 20 +++++++++ 7 files changed, 184 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 496ed3a..decb222 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -96,7 +96,7 @@ jobs: # carries, or a change to them is not exercised until after it merges. # Consumers use the action; this repository owns the rules. - name: Refuse a skip, and a suite that shrank - run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 536 + run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 546 # The rules earn their place by refusing a log that carries what they # name. Both fixtures are written here rather than tracked, and the diff --git a/GENERATION.md b/GENERATION.md index 041b941..dba1d7e 100644 --- a/GENERATION.md +++ b/GENERATION.md @@ -45,7 +45,8 @@ different fidelity, and the choice matters: - the **installed** headers (`cmake --install` output) are self-contained — `meos.h` is the generated `meos_export.h`, with `postgres_ext_defs.in.h` spliced in place of the source tree's `#include ` — so libclang resolves every struct field to its real C type - and byte offset. FFI bindings need this. + and byte offset, and every defined struct to the bytes it is laid out in (`sizeBytes`). FFI + bindings need this. - the **source** tree headers (`/meos/include`) parse without a build, but wrap the PostgreSQL types in stubs, so struct layouts are approximate. diff --git a/docs/enrichment.md b/docs/enrichment.md index fb3b545..042aedc 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -53,7 +53,12 @@ literals are in the catalog. "in": "set_from_hexwkb", "in_aux": [], "out": "set_as_hexwkb", "out_aux": [{"name": "variant", "kind": "integer", "default": 4}], "bytes": { "decoder": "set_from_wkb", "encoder": "set_as_wkb", - "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] } + "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] }, + "image": { "copy": "set_copy", "size": "set_mem_size" } + }, + "Span": { + "...": "...", + "image": { "copy": "span_copy", "sizeBytes": 24 } } } ``` @@ -99,6 +104,17 @@ literals are in the catalog. `Quadbin`, `S2CellId`, each a `uint64_t`) is a class keyed by that name, read and written by its own functions. +- **`image`** — the memory image of a value, which a binding holding values in + process carries instead of a wire form: `copy` is the public `T *f(const T *)` named + `*_copy` that duplicates a value from its image, and the image's length is either what + the public `int f(const T *)` named `*_mem_size` answers (`size`), or, for a type no + such function sizes, the bytes clang lays its structure in (`sizeBytes`, the + structure's own `structs[*].sizeBytes`). A class states exactly one of `size` and + `sizeBytes`. A structure ending in a flexible array member is variable-length, so a + class over one states no image without a `*_mem_size`; a class with no public copy + (`H3Index`, `TimestampTz`) states none either. Two copy or size functions for one + class stop the catalog. + The class's `encodings`, `in` and `out` are folded onto each `structs[*]` entry as `serialization`. diff --git a/parser/codecs.py b/parser/codecs.py index b25a15a..d03738c 100644 --- a/parser/codecs.py +++ b/parser/codecs.py @@ -25,10 +25,15 @@ by name and refuses one the catalog does not fill. - A value whose slot carries a ``typedef`` (``H3Index``, ``Quadbin``, ``S2CellId``) is a class of its own, keyed by that name. +- A class whose values MEOS copies states their memory image in ``image``: the copy + function and either the function answering an image's length (``size``) or, for a + fixed-size structure, the bytes it is laid out in (``sizeBytes``), so a binding holding + values in process reads both from the catalog (#_image). """ import re -from parser.enrich import _DECODERS, _ENCODERS, _STRING_PTR_BASES, _aux_specs, _base +from parser.enrich import (_DECODERS, _ENCODERS, _INT_BASES, _STRING_PTR_BASES, _aux_specs, + _base, _ptr_depth) _ORDER = ("text", "mfjson", "wkb") @@ -181,6 +186,7 @@ def resolve(cls, enc, cands, prior, sql_type, side, table): return (cands[0] if len(cands) == 1 else None), {} every = {f["name"]: f for f in idl.get("functions", [])} + structs = {s["name"]: s for s in idl.get("structs", [])} for cls in sorted(set(encs) | {c for c in decoders if c in encs or _is_identity(c, fns)}): e = encs.setdefault(cls, {"encodings": [], "decoders": {}, "encoders": {}}) prior_dec, prior_enc = dict(e.get("decoders") or {}), dict(e.get("encoders") or {}) @@ -230,6 +236,11 @@ def resolve(cls, enc, cands, prior, sql_type, side, table): if "bytes" in e: v = _variant(cls, fns, macros, errors) e["bytes"]["encoderAux"] = [{"name": "variant", "kind": "integer", "default": v}] + image = _image(cls, fns, structs, errors) + if image: + e["image"] = image + else: + e.pop("image", None) for struct in idl.get("structs", []): te = encs.get(struct["name"]) @@ -253,6 +264,45 @@ def _byte_codec(cls, fns): return None +def _image(cls, fns, structs, errors): + """{copy, size} or {copy, sizeBytes}: the memory image of a value of ``cls``, which a + binding holding values in process carries instead of a wire form. ``copy`` is the + public ``T *f(const T *)`` named ``_copy`` that duplicates a value from its image; + the image's length is what the public ``int f(const T *)`` named ``_mem_size`` + answers (``size``), or, for a type no such function sizes, the bytes clang lays its + structure in (``sizeBytes`` of #extract_struct of parser/extractors.py). A structure + ending in a flexible array member is variable-length, so its laid-out size is not + its image and the type states none without a ``_mem_size``. Found by shape and name + as #_byte_codec finds the byte codec.""" + def one(names, what): + if len(names) > 1: + errors.append(f"{cls}: {what} by {', '.join(sorted(names))}") + return None + return names[0] if names else None + + copy = one([f["name"] for f in fns if f["name"].endswith("_copy") + and len(f.get("params") or []) == 1 + and _ptr_depth(f["returnType"].get("c") or "") == 1 + and _cls(f["returnType"]) == cls + and _ptr_depth(f["params"][0].get("cType") or "") == 1 + and _cls(f["params"][0]) == cls], "copied") + if not copy: + return None + size = one([f["name"] for f in fns if f["name"].endswith("_mem_size") + and len(f.get("params") or []) == 1 + and _ptr_depth(f["returnType"].get("c") or "") == 0 + and _base(f["returnType"].get("canonical") or "") in _INT_BASES + and _ptr_depth(f["params"][0].get("cType") or "") == 1 + and _cls(f["params"][0]) == cls], "sized") + if size: + return {"copy": copy, "size": size} + struct = structs.get(cls) or {} + if "sizeBytes" in struct and not any(fld["cType"].endswith("[]") + for fld in struct.get("fields") or ()): + return {"copy": copy, "sizeBytes": struct["sizeBytes"]} + return None + + def _is_identity(cls, fns): """Whether ``cls`` is a type of its own some slot names (``H3Index``).""" return any(p.get("typedef") == cls for f in fns diff --git a/parser/extractors.py b/parser/extractors.py index 172d30b..7c10294 100644 --- a/parser/extractors.py +++ b/parser/extractors.py @@ -259,11 +259,19 @@ def extract_function(node) -> dict: def extract_struct(node) -> dict: + """A structure, its fields at the offset clang lays each at and, when the unit + defines it, the bytes clang lays the whole record in (`sizeBytes`), the trailing + padding included: the image of a value whose type is the structure (a `Span`, an + `STBox`) is that many bytes. A structure only declared (`struct NumericData;`) has + no size and states none, as #extract_enum states a value's `doc` only where the + value carries one.""" + size = node.type.get_size() return { "name": node.spelling, "file": Path(node.location.file.name).name, "family": _family_of(node.location.file.name, node.location.line), "vendored": _is_vendored(node.location.file.name), + **({"sizeBytes": size} if size >= 0 else {}), "fields": [ { "name": f.spelling, diff --git a/tests/test_codecs.py b/tests/test_codecs.py index 5b8774b..1511c58 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -177,6 +177,62 @@ def test_two_functions_alike_for_one_type_stop_the_catalog(self): self.assertEqual(errors, ["Set: intset read by intset_in and intsetx_in"]) +class ImageTests(unittest.TestCase): + """The memory image #_image of parser/codecs.py states for a class over the synthetic + catalog of #CodecTests, with the copy and size functions each case adds.""" + + SPAN = {"name": "Span", "sizeBytes": 24, + "fields": [{"name": "spantype", "cType": "uint8", "offset_bits": 0}]} + TEXT = {"name": "text", "sizeBytes": 4, + "fields": [{"name": "vl_len_", "cType": "char[4]", "offset_bits": 0}, + {"name": "vl_dat", "cType": "char[]", "offset_bits": 32}]} + + def _image(self, extra, structs=(), classes=()): + idl = _idl(FUNCTIONS + extra) + idl["structs"] += [dict(s) for s in structs] + for cls in classes: + idl["typeEncodings"][cls] = {} + idl, errors = state_type_encodings(idl) + return idl["typeEncodings"], errors + + def test_a_variable_length_value_states_its_copy_and_its_size_function(self): + te, errors = self._image([ + fn("set_copy", "Set *", [(SET, "s")]), + fn("set_mem_size", "int", [(SET, "s")]), + fn("raster_copy", "Raster *", [(RAST, "rast")]), + fn("raster_mem_size", "int", [(RAST, "rast")])]) + self.assertEqual(te["Set"]["image"], {"copy": "set_copy", "size": "set_mem_size"}) + self.assertEqual(te["Raster"]["image"], + {"copy": "raster_copy", "size": "raster_mem_size"}) + self.assertEqual(errors, []) + + def test_a_fixed_size_value_states_its_copy_and_its_structure_size(self): + te, errors = self._image([fn("span_copy", "Span *", [("const Span *", "s")])], + [self.SPAN], ["Span"]) + self.assertEqual(te["Span"]["image"], {"copy": "span_copy", "sizeBytes": 24}) + self.assertEqual(errors, []) + + def test_a_flexible_array_structure_without_a_size_function_states_no_image(self): + te, _ = self._image([fn("text_copy", "text *", [("const text *", "txt")])], + [self.TEXT], ["text"]) + self.assertNotIn("image", te["text"]) + + def test_no_copy_no_image(self): + te, _ = self._image([fn("set_mem_size", "int", [(SET, "s")])]) + self.assertNotIn("image", te["Set"]) + self.assertNotIn("image", te["H3Index"]) + + def test_an_internal_copy_is_no_image(self): + te, _ = self._image([fn("set_copy", "Set *", [(SET, "s")], api="internal"), + fn("set_mem_size", "int", [(SET, "s")])]) + self.assertNotIn("image", te["Set"]) + + def test_two_copies_of_one_class_stop_the_catalog(self): + _, errors = self._image([fn("set_copy", "Set *", [(SET, "s")]), + fn("intset_copy", "Set *", [(SET, "s")])]) + self.assertEqual(errors, ["Set: copied by intset_copy, set_copy"]) + + class CodecContractTests(unittest.TestCase): """Over the generated catalog: every trailing input a codec states names a parameter of its function, carries the value a binding passes, and is never an out-parameter.""" @@ -222,6 +278,35 @@ def test_the_classes_a_binding_reads(self): for cell in ("H3Index", "Quadbin", "S2CellId"): self.assertIn(cell, te) + def test_the_images_a_binding_reads(self): + """The image of #ImageTests over the generated catalog: a value sized by a + function, and one whose image is its structure, at the size the structure + states.""" + te = self.idl["typeEncodings"] + sizes = {s["name"]: s.get("sizeBytes") for s in self.idl["structs"]} + self.assertEqual(te["Temporal"]["image"], + {"copy": "temporal_copy", "size": "temporal_mem_size"}) + self.assertEqual(te["Raster"]["image"], + {"copy": "raster_copy", "size": "raster_mem_size"}) + for cls, copy in (("Span", "span_copy"), ("STBox", "stbox_copy"), + ("Npoint", "npoint_copy"), ("Nsegment", "nsegment_copy")): + self.assertEqual(te[cls]["image"], {"copy": copy, "sizeBytes": sizes[cls]}) + + def test_every_image_names_public_functions_of_its_class(self): + bad = [] + for cls, e in self.idl["typeEncodings"].items(): + image = e.get("image") + if not image: + continue + for key in ("copy", "size"): + f = self.fns.get(image.get(key)) if key in image else None + if key in image and (f is None or f.get("api") != "public" + or len(f["params"]) != 1): + bad.append((cls, key, image[key])) + if ("size" in image) == ("sizeBytes" in image): + bad.append((cls, "size xor sizeBytes", image)) + self.assertEqual(bad, []) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_struct_layout.py b/tests/test_struct_layout.py index fdf901a..7fda187 100644 --- a/tests/test_struct_layout.py +++ b/tests/test_struct_layout.py @@ -79,6 +79,22 @@ def test_core_structs_have_real_offsets(self): fld["offset_bits"], 0, f"{s['name']}.{fld['name']} has unresolved offset") + def test_a_fixed_size_value_states_the_bytes_clang_lays_it_in(self): + """The size beside the offsets #test_span_layout_resolved reads: the image of + these values is their structure, the trailing padding included (a TBox is two + 24-byte spans and an int16, padded to 56).""" + sizes = {n: self.by_name[n].get("sizeBytes") + for n in ("Span", "TBox", "STBox", "Npoint", "Nsegment")} + self.assertEqual(sizes, {"Span": 24, "TBox": 56, "STBox": 80, + "Npoint": 16, "Nsegment": 24}) + + def test_a_defined_structure_states_its_size_and_an_opaque_one_none(self): + """Over every structure, as #test_core_structs_have_real_offsets reads every + field: one with fields is defined and sized, one without only declared.""" + bad = [s["name"] for s in self.by_name.values() + if ("sizeBytes" in s) != bool(s["fields"])] + self.assertEqual(bad, []) + def test_a_structure_declared_first_is_read_from_its_definition(self): # `meos.h` declares MeosArray and SkipList before `meos_internal.h` defines # them, so their layout is read where they are defined @@ -105,6 +121,10 @@ def test_the_definition_wins_and_an_opaque_structure_stays(self): idl = parse_all_headers(root) structs = {s["name"]: [f["name"] for f in s["fields"]] for s in idl["structs"]} self.assertEqual(structs, {"S": ["x", "y"], "O": []}) + # the definition states the bytes clang lays it in, its padding included; the + # opaque structure states no size + sizes = {s["name"]: s.get("sizeBytes") for s in idl["structs"]} + self.assertEqual(sizes, {"S": 16, "O": None}) if __name__ == "__main__":