diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 496ed3a..decb222 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -96,7 +96,7 @@ jobs: # carries, or a change to them is not exercised until after it merges. # Consumers use the action; this repository owns the rules. - name: Refuse a skip, and a suite that shrank - run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 536 + run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 546 # The rules earn their place by refusing a log that carries what they # name. Both fixtures are written here rather than tracked, and the diff --git a/GENERATION.md b/GENERATION.md index 041b941..dba1d7e 100644 --- a/GENERATION.md +++ b/GENERATION.md @@ -45,7 +45,8 @@ different fidelity, and the choice matters: - the **installed** headers (`cmake --install` output) are self-contained — `meos.h` is the generated `meos_export.h`, with `postgres_ext_defs.in.h` spliced in place of the source tree's `#include ` — so libclang resolves every struct field to its real C type - and byte offset. FFI bindings need this. + and byte offset, and every defined struct to the bytes it is laid out in (`sizeBytes`). FFI + bindings need this. - the **source** tree headers (`/meos/include`) parse without a build, but wrap the PostgreSQL types in stubs, so struct layouts are approximate. diff --git a/docs/enrichment.md b/docs/enrichment.md index fb3b545..042aedc 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -53,7 +53,12 @@ literals are in the catalog. "in": "set_from_hexwkb", "in_aux": [], "out": "set_as_hexwkb", "out_aux": [{"name": "variant", "kind": "integer", "default": 4}], "bytes": { "decoder": "set_from_wkb", "encoder": "set_as_wkb", - "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] } + "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] }, + "image": { "copy": "set_copy", "size": "set_mem_size" } + }, + "Span": { + "...": "...", + "image": { "copy": "span_copy", "sizeBytes": 24 } } } ``` @@ -99,6 +104,17 @@ literals are in the catalog. `Quadbin`, `S2CellId`, each a `uint64_t`) is a class keyed by that name, read and written by its own functions. +- **`image`** — the memory image of a value, which a binding holding values in + process carries instead of a wire form: `copy` is the public `T *f(const T *)` named + `*_copy` that duplicates a value from its image, and the image's length is either what + the public `int f(const T *)` named `*_mem_size` answers (`size`), or, for a type no + such function sizes, the bytes clang lays its structure in (`sizeBytes`, the + structure's own `structs[*].sizeBytes`). A class states exactly one of `size` and + `sizeBytes`. A structure ending in a flexible array member is variable-length, so a + class over one states no image without a `*_mem_size`; a class with no public copy + (`H3Index`, `TimestampTz`) states none either. Two copy or size functions for one + class stop the catalog. + The class's `encodings`, `in` and `out` are folded onto each `structs[*]` entry as `serialization`. diff --git a/parser/codecs.py b/parser/codecs.py index b25a15a..d03738c 100644 --- a/parser/codecs.py +++ b/parser/codecs.py @@ -25,10 +25,15 @@ by name and refuses one the catalog does not fill. - A value whose slot carries a ``typedef`` (``H3Index``, ``Quadbin``, ``S2CellId``) is a class of its own, keyed by that name. +- A class whose values MEOS copies states their memory image in ``image``: the copy + function and either the function answering an image's length (``size``) or, for a + fixed-size structure, the bytes it is laid out in (``sizeBytes``), so a binding holding + values in process reads both from the catalog (#_image). """ import re -from parser.enrich import _DECODERS, _ENCODERS, _STRING_PTR_BASES, _aux_specs, _base +from parser.enrich import (_DECODERS, _ENCODERS, _INT_BASES, _STRING_PTR_BASES, _aux_specs, + _base, _ptr_depth) _ORDER = ("text", "mfjson", "wkb") @@ -181,6 +186,7 @@ def resolve(cls, enc, cands, prior, sql_type, side, table): return (cands[0] if len(cands) == 1 else None), {} every = {f["name"]: f for f in idl.get("functions", [])} + structs = {s["name"]: s for s in idl.get("structs", [])} for cls in sorted(set(encs) | {c for c in decoders if c in encs or _is_identity(c, fns)}): e = encs.setdefault(cls, {"encodings": [], "decoders": {}, "encoders": {}}) prior_dec, prior_enc = dict(e.get("decoders") or {}), dict(e.get("encoders") or {}) @@ -230,6 +236,11 @@ def resolve(cls, enc, cands, prior, sql_type, side, table): if "bytes" in e: v = _variant(cls, fns, macros, errors) e["bytes"]["encoderAux"] = [{"name": "variant", "kind": "integer", "default": v}] + image = _image(cls, fns, structs, errors) + if image: + e["image"] = image + else: + e.pop("image", None) for struct in idl.get("structs", []): te = encs.get(struct["name"]) @@ -253,6 +264,45 @@ def _byte_codec(cls, fns): return None +def _image(cls, fns, structs, errors): + """{copy, size} or {copy, sizeBytes}: the memory image of a value of ``cls``, which a + binding holding values in process carries instead of a wire form. ``copy`` is the + public ``T *f(const T *)`` named ``_copy`` that duplicates a value from its image; + the image's length is what the public ``int f(const T *)`` named ``_mem_size`` + answers (``size``), or, for a type no such function sizes, the bytes clang lays its + structure in (``sizeBytes`` of #extract_struct of parser/extractors.py). A structure + ending in a flexible array member is variable-length, so its laid-out size is not + its image and the type states none without a ``_mem_size``. Found by shape and name + as #_byte_codec finds the byte codec.""" + def one(names, what): + if len(names) > 1: + errors.append(f"{cls}: {what} by {', '.join(sorted(names))}") + return None + return names[0] if names else None + + copy = one([f["name"] for f in fns if f["name"].endswith("_copy") + and len(f.get("params") or []) == 1 + and _ptr_depth(f["returnType"].get("c") or "") == 1 + and _cls(f["returnType"]) == cls + and _ptr_depth(f["params"][0].get("cType") or "") == 1 + and _cls(f["params"][0]) == cls], "copied") + if not copy: + return None + size = one([f["name"] for f in fns if f["name"].endswith("_mem_size") + and len(f.get("params") or []) == 1 + and _ptr_depth(f["returnType"].get("c") or "") == 0 + and _base(f["returnType"].get("canonical") or "") in _INT_BASES + and _ptr_depth(f["params"][0].get("cType") or "") == 1 + and _cls(f["params"][0]) == cls], "sized") + if size: + return {"copy": copy, "size": size} + struct = structs.get(cls) or {} + if "sizeBytes" in struct and not any(fld["cType"].endswith("[]") + for fld in struct.get("fields") or ()): + return {"copy": copy, "sizeBytes": struct["sizeBytes"]} + return None + + def _is_identity(cls, fns): """Whether ``cls`` is a type of its own some slot names (``H3Index``).""" return any(p.get("typedef") == cls for f in fns diff --git a/parser/extractors.py b/parser/extractors.py index 172d30b..7c10294 100644 --- a/parser/extractors.py +++ b/parser/extractors.py @@ -259,11 +259,19 @@ def extract_function(node) -> dict: def extract_struct(node) -> dict: + """A structure, its fields at the offset clang lays each at and, when the unit + defines it, the bytes clang lays the whole record in (`sizeBytes`), the trailing + padding included: the image of a value whose type is the structure (a `Span`, an + `STBox`) is that many bytes. A structure only declared (`struct NumericData;`) has + no size and states none, as #extract_enum states a value's `doc` only where the + value carries one.""" + size = node.type.get_size() return { "name": node.spelling, "file": Path(node.location.file.name).name, "family": _family_of(node.location.file.name, node.location.line), "vendored": _is_vendored(node.location.file.name), + **({"sizeBytes": size} if size >= 0 else {}), "fields": [ { "name": f.spelling, diff --git a/tests/test_codecs.py b/tests/test_codecs.py index 5b8774b..1511c58 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -177,6 +177,62 @@ def test_two_functions_alike_for_one_type_stop_the_catalog(self): self.assertEqual(errors, ["Set: intset read by intset_in and intsetx_in"]) +class ImageTests(unittest.TestCase): + """The memory image #_image of parser/codecs.py states for a class over the synthetic + catalog of #CodecTests, with the copy and size functions each case adds.""" + + SPAN = {"name": "Span", "sizeBytes": 24, + "fields": [{"name": "spantype", "cType": "uint8", "offset_bits": 0}]} + TEXT = {"name": "text", "sizeBytes": 4, + "fields": [{"name": "vl_len_", "cType": "char[4]", "offset_bits": 0}, + {"name": "vl_dat", "cType": "char[]", "offset_bits": 32}]} + + def _image(self, extra, structs=(), classes=()): + idl = _idl(FUNCTIONS + extra) + idl["structs"] += [dict(s) for s in structs] + for cls in classes: + idl["typeEncodings"][cls] = {} + idl, errors = state_type_encodings(idl) + return idl["typeEncodings"], errors + + def test_a_variable_length_value_states_its_copy_and_its_size_function(self): + te, errors = self._image([ + fn("set_copy", "Set *", [(SET, "s")]), + fn("set_mem_size", "int", [(SET, "s")]), + fn("raster_copy", "Raster *", [(RAST, "rast")]), + fn("raster_mem_size", "int", [(RAST, "rast")])]) + self.assertEqual(te["Set"]["image"], {"copy": "set_copy", "size": "set_mem_size"}) + self.assertEqual(te["Raster"]["image"], + {"copy": "raster_copy", "size": "raster_mem_size"}) + self.assertEqual(errors, []) + + def test_a_fixed_size_value_states_its_copy_and_its_structure_size(self): + te, errors = self._image([fn("span_copy", "Span *", [("const Span *", "s")])], + [self.SPAN], ["Span"]) + self.assertEqual(te["Span"]["image"], {"copy": "span_copy", "sizeBytes": 24}) + self.assertEqual(errors, []) + + def test_a_flexible_array_structure_without_a_size_function_states_no_image(self): + te, _ = self._image([fn("text_copy", "text *", [("const text *", "txt")])], + [self.TEXT], ["text"]) + self.assertNotIn("image", te["text"]) + + def test_no_copy_no_image(self): + te, _ = self._image([fn("set_mem_size", "int", [(SET, "s")])]) + self.assertNotIn("image", te["Set"]) + self.assertNotIn("image", te["H3Index"]) + + def test_an_internal_copy_is_no_image(self): + te, _ = self._image([fn("set_copy", "Set *", [(SET, "s")], api="internal"), + fn("set_mem_size", "int", [(SET, "s")])]) + self.assertNotIn("image", te["Set"]) + + def test_two_copies_of_one_class_stop_the_catalog(self): + _, errors = self._image([fn("set_copy", "Set *", [(SET, "s")]), + fn("intset_copy", "Set *", [(SET, "s")])]) + self.assertEqual(errors, ["Set: copied by intset_copy, set_copy"]) + + class CodecContractTests(unittest.TestCase): """Over the generated catalog: every trailing input a codec states names a parameter of its function, carries the value a binding passes, and is never an out-parameter.""" @@ -222,6 +278,35 @@ def test_the_classes_a_binding_reads(self): for cell in ("H3Index", "Quadbin", "S2CellId"): self.assertIn(cell, te) + def test_the_images_a_binding_reads(self): + """The image of #ImageTests over the generated catalog: a value sized by a + function, and one whose image is its structure, at the size the structure + states.""" + te = self.idl["typeEncodings"] + sizes = {s["name"]: s.get("sizeBytes") for s in self.idl["structs"]} + self.assertEqual(te["Temporal"]["image"], + {"copy": "temporal_copy", "size": "temporal_mem_size"}) + self.assertEqual(te["Raster"]["image"], + {"copy": "raster_copy", "size": "raster_mem_size"}) + for cls, copy in (("Span", "span_copy"), ("STBox", "stbox_copy"), + ("Npoint", "npoint_copy"), ("Nsegment", "nsegment_copy")): + self.assertEqual(te[cls]["image"], {"copy": copy, "sizeBytes": sizes[cls]}) + + def test_every_image_names_public_functions_of_its_class(self): + bad = [] + for cls, e in self.idl["typeEncodings"].items(): + image = e.get("image") + if not image: + continue + for key in ("copy", "size"): + f = self.fns.get(image.get(key)) if key in image else None + if key in image and (f is None or f.get("api") != "public" + or len(f["params"]) != 1): + bad.append((cls, key, image[key])) + if ("size" in image) == ("sizeBytes" in image): + bad.append((cls, "size xor sizeBytes", image)) + self.assertEqual(bad, []) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_struct_layout.py b/tests/test_struct_layout.py index fdf901a..7fda187 100644 --- a/tests/test_struct_layout.py +++ b/tests/test_struct_layout.py @@ -79,6 +79,22 @@ def test_core_structs_have_real_offsets(self): fld["offset_bits"], 0, f"{s['name']}.{fld['name']} has unresolved offset") + def test_a_fixed_size_value_states_the_bytes_clang_lays_it_in(self): + """The size beside the offsets #test_span_layout_resolved reads: the image of + these values is their structure, the trailing padding included (a TBox is two + 24-byte spans and an int16, padded to 56).""" + sizes = {n: self.by_name[n].get("sizeBytes") + for n in ("Span", "TBox", "STBox", "Npoint", "Nsegment")} + self.assertEqual(sizes, {"Span": 24, "TBox": 56, "STBox": 80, + "Npoint": 16, "Nsegment": 24}) + + def test_a_defined_structure_states_its_size_and_an_opaque_one_none(self): + """Over every structure, as #test_core_structs_have_real_offsets reads every + field: one with fields is defined and sized, one without only declared.""" + bad = [s["name"] for s in self.by_name.values() + if ("sizeBytes" in s) != bool(s["fields"])] + self.assertEqual(bad, []) + def test_a_structure_declared_first_is_read_from_its_definition(self): # `meos.h` declares MeosArray and SkipList before `meos_internal.h` defines # them, so their layout is read where they are defined @@ -105,6 +121,10 @@ def test_the_definition_wins_and_an_opaque_structure_stays(self): idl = parse_all_headers(root) structs = {s["name"]: [f["name"] for f in s["fields"]] for s in idl["structs"]} self.assertEqual(structs, {"S": ["x", "y"], "O": []}) + # the definition states the bytes clang lays it in, its padding included; the + # opaque structure states no size + sizes = {s["name"]: s.get("sizeBytes") for s in idl["structs"]} + self.assertEqual(sizes, {"S": 16, "O": None}) if __name__ == "__main__":