State an array return as long as an array the function reads

.github/workflows/pytest.yml

@@ -96,7 +96,7 @@ jobs:
       # carries, or a change to them is not exercised until after it merges.
       # Consumers use the action; this repository owns the rules.
       - name: Refuse a skip, and a suite that shrank
-        run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 469
+        run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 534
 
       # The rules earn their place by refusing a log that carries what they
       # name. Both fixtures are written here rather than tracked, and the

docs/enrichment.md

@@ -181,6 +181,13 @@ Otherwise `exposable` is `false` and `reason` lists the blockers
 > (element = the serialized `Elem`, `count_outparam` names the byref
 > length). Recovers `temporal_instants`/`segments`/`sequences`,
 > `tgeo_values`, `geo_pointarr`, … whose element type is encodable.
+> An array as long as an input array, `Elem **f(Elem **arr, int count, ..)`
+> (`shape.arrayReturn.lengthFrom` names the by-value `count`), becomes the
+> same `kind: "array"` result with `count_param` naming that length: the
+> response holds one element per element of the request's array
+> (`temparr_round`, `cbufferarr_round`, `posearr_round`). One counted by an
+> argument the request carries as one value is refused naming it
+> (`stboxarr_round`: `array-or-out-param:boxarr`).
 
 This is the precise, machine-checkable boundary of what a generator can emit
 today. Functions still blocked only by `array-or-out-param` (multi- or

parser/enrich.py

@@ -434,6 +434,41 @@ def _array_return(fn: dict, type_encodings: dict):
     return cnt[0]["name"], rb
 
 
+def _counted_array_return(fn: dict, type_encodings: dict, params: list, count_i):
+    """The array a function returns as long as an array argument, as
+    ``shape.arrayReturn`` states it when its ``lengthFrom`` is a by-value parameter
+    (``temparr_round(Temporal **temp, int count, int maxdd)`` returns ``count``
+    values). Returns ``(length_param, elem_base, refusal)``.
+
+    The response holds one encoded element per element of the request's array, so the
+    return is served as #_array_return serves one, an array of encodable element
+    pointers, when its length is the length #_array_param gives the request's array
+    (``params[count_i]``). An array whose length is that of an argument the request
+    does not carry as an array cannot be read back at its length, and is refused
+    naming that argument (``stboxarr_round`` reads ``boxarr`` as one box); an array of
+    elements that are not encodable pointers is refused naming the return. A function
+    with no such ``arrayReturn`` is ``(None, None, None)``, and #assess reads its
+    return as any other."""
+    ar = (fn.get("shape") or {}).get("arrayReturn") or {}
+    length = (ar.get("lengthFrom") or {}).get("name")
+    by_value = {p["name"] for p in fn.get("params", [])
+                if _ptr_depth(p["canonical"]) == 0}
+    if length not in by_value:
+        return None, None, None
+    ret = fn["returnType"]["canonical"]
+    if count_i is None or params[count_i]["name"] != length:
+        source = next((a["param"]
+                       for a in (fn.get("shape") or {}).get("inputArrays") or ()
+                       if a["lengthFrom"].get("name") == length), length)
+        return length, None, f"array-or-out-param:{source}"
+    te = type_encodings.get(_base(ret))
+    if _ptr_depth(ret) != 2:
+        return length, None, f"unsupported-return:{ret}"
+    if not (te and te.get("out")):
+        return length, None, f"no-encoder:{_base(ret)}"
+    return length, _base(ret), None
+
+
 def assess(fn: dict, type_encodings: dict, enums: set) -> tuple:
     """Return ``(network, wire)`` for one function.
 
@@ -452,6 +487,9 @@ def assess(fn: dict, type_encodings: dict, enums: set) -> tuple:
     ret_count_i = (next((i for i, p in enumerate(eff)
                          if p["name"] == ret_count_name), None)
                    if ret_elem is not None else None)
+    ret_length, counted_elem, counted_refusal = (
+        _counted_array_return(fn, type_encodings, eff, count_i)
+        if ret_elem is None and out_p is None else (None, None, None))
 
     for idx, p in enumerate(eff):
         if idx == count_i or idx == ret_count_i:
@@ -529,6 +567,23 @@ def assess(fn: dict, type_encodings: dict, enums: set) -> tuple:
             "count_outparam": ret_count_name,
         }
         scalar = "handled"
+    elif counted_elem is not None:
+        te = type_encodings[counted_elem]
+        wire_result = {
+            "kind": "array",
+            "element": {
+                "kind": "serialized",
+                "cType": " ".join(ret.replace("*", " ").split()) + " *",
+                "encode": te["out"], "encode_aux": te.get("out_aux", []),
+                "encodings": te["encodings"],
+            },
+            "count_param": ret_length,
+        }
+        scalar = "handled"
+    elif counted_refusal is not None:
+        reasons.append(counted_refusal)
+        wire_result = {"kind": "unsupported"}
+        scalar = "handled"
     else:
         scalar = _scalar_wire(ret, enums)
     if scalar == "handled":

parser/shapeinfer.py

@@ -7,6 +7,8 @@
     TYPE **f(..., TYPE **extra, int *count)   -> primary array return PLUS one
                                                   or more parallel out-arrays
     f(..., TYPE **values, int count, ...)     -> reads an array of ``count``
+    TYPE **f(..., TYPE **values, int count)   -> reads and returns an array of
+                                                  ``count``
 
 The output length is always passed *by pointer* (``int *count``); an *input*
 array instead carries its length *by value* (``int count``).  That pointer/value
@@ -212,6 +214,48 @@ def attach_struct_input_arrays(idl: dict, docs: dict) -> tuple[dict, int]:
     return idl, n
 
 
+def attach_counted_array_returns(idl: dict) -> tuple[dict, int]:
+    """(idl, number of returns stated) once ``shape.arrayReturn`` also states each array
+    a function returns as long as an array it reads.
+
+    ``temparr_round(Temporal **temp, int count, int maxdd)`` answers one rounded value
+    per value it reads, so its ``Temporal **`` holds ``count`` elements, and no ``int
+    *count`` out-parameter says so, as #infer_shapes requires. The types say it: the
+    return is the element of a stated input array with one more pointer level, which is
+    how an array of that element is spelled. ``stboxarr_round(const STBox *boxarr, int
+    count, int maxdd)`` reads ``count`` boxes and returns ``count`` boxes as ``STBox *``.
+    A return of the element type itself is one element, not an array:
+    ``temporal_merge_array(Temporal **temparr, int count)`` returns one ``Temporal *``.
+
+    The array return takes its length from that input array's ``lengthFrom`` and reads
+    its ``element`` from the return, as #infer_shapes states both. It runs after
+    #attach_struct_input_arrays, which states the arrays of structs it compares with."""
+    n = 0
+    for func in idl["functions"]:
+        shape = func.get("shape") or {}
+        if "arrayReturn" in shape or _out_count_param(func):
+            continue
+        rtype = func.get("returnType", {})
+        ret = rtype.get("c", "")
+        if not ret.rstrip().endswith("*"):
+            continue
+        elem = _bare(_strip_one_ptr(ret))
+        lengths = {a["lengthFrom"]["name"]: a["lengthFrom"]
+                   for a in shape.get("inputArrays") or ()
+                   if _bare(a["element"]["c"]) == elem}
+        if len(lengths) != 1:
+            continue
+        shape["arrayReturn"] = {
+            "lengthFrom": dict(next(iter(lengths.values()))),
+            "element": {
+                "c": _strip_one_ptr(ret),
+                "canonical": _strip_one_ptr(rtype.get("canonical", ret)),
+            },
+        }
+        n += 1
+    return idl, n
+
+
 def _is_index_pair_return(func: dict, count: str) -> bool:
     """Whether the ``int *`` return is a FLATTENED array of index PAIRS.
 

run.py

@@ -11,7 +11,8 @@
 from parser.typerecover import (recover_collapsed_types, normalize_canonical,
                                  postgres_scalar_names)
 from parser.header_types import reconcile
-from parser.shapeinfer import infer_shapes, attach_struct_input_arrays
+from parser.shapeinfer import (infer_shapes, attach_struct_input_arrays,
+                               attach_counted_array_returns)
 from parser.nullable import merge_nullable, extract_param_docs
 from parser.nullresult import attach_null_result
 from parser.outparam import extract_param_names, merge_outparams
@@ -164,6 +165,8 @@ def main():
     idl, nsa = attach_struct_input_arrays(idl, extract_param_docs(_doxy_root))
     print(f"      input arrays of structs from Doxygen `Array of`: {nsa}",
           file=sys.stderr)
+    idl, ncr = attach_counted_array_returns(idl)
+    print(f"      array returns as long as an input array: {ncr}", file=sys.stderr)
     idl, no, out_drift = merge_outparams(idl, _doxy_root)
     print(f"      out params from Doxygen `@param[out]`: {no}", file=sys.stderr)
     if out_drift:

server/app.py

@@ -74,7 +74,8 @@ def build_routes(catalog: dict) -> dict:
                        "out_ctype": r.get("out_ctype"),
                        "presence_return": r.get("presence_return", False),
                        "element": r.get("element"),
-                       "count_outparam": r.get("count_outparam")},
+                       "count_outparam": r.get("count_outparam"),
+                       "count_param": r.get("count_param")},
         }
     return routes
 
@@ -168,9 +169,17 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict):
         return {"result": bool(val) if res.get("json") == "boolean" else val}
 
     if res["kind"] == "array":
-        # Elem **f(.., int *count): MEOS returns a fresh array + byref count.
         el = res["element"]
-        ptrs = engine.invoke_array(route["name"], args)
+        if res.get("count_param"):
+            # Elem **f(Elem **arr, int count, ..): one element per element of
+            # the request's array of that count.
+            n = next(len(body[p["name"]]) for p in route["params"]
+                     if p["kind"] == "array"
+                     and p["count_param"] == res["count_param"])
+            ptrs = engine.invoke_counted_array(route["name"], args, n)
+        else:
+            # Elem **f(.., int *count): MEOS returns a fresh array + byref count.
+            ptrs = engine.invoke_array(route["name"], args)
         return {"result": [engine.encode(el["encode"], p,
                                          _aux_args(el.get("encode_aux", [])))
                            for p in ptrs]}

server/engine.py

@@ -58,6 +58,11 @@ def invoke_array(self, fn_name: str, args: list):
         """``Elem **f(.., int *count)`` — return a list of element handles."""
         raise NotImplementedError
 
+    def invoke_counted_array(self, fn_name: str, args: list, count: int):
+        """``Elem **f(Elem **arr, int count, ..)`` — return the ``count`` element
+        handles of an array as long as an array argument."""
+        raise NotImplementedError
+
     def close(self) -> None:
         pass
 
@@ -87,6 +92,9 @@ def invoke_outparam(self, fn_name, args, out_ctype, presence):
     def invoke_array(self, fn_name, args):
         return []
 
+    def invoke_counted_array(self, fn_name, args, count):
+        return []
+
 
 class CtypesEngine(Engine):
     """Calls a built libmeos via ctypes, driven entirely by the wire model.
@@ -250,6 +258,15 @@ def invoke_array(self, fn_name, args):
         arr = ct.cast(ret, ct.POINTER(ct.c_void_p))
         return [arr[i] for i in range(count)]
 
+    def invoke_counted_array(self, fn_name, args, count):
+        # Elem **f(Elem **arr, int count, ...): MEOS allocates an array as long
+        # as the argument array, so its length is the one the caller passed.
+        ret = self.invoke(fn_name, args, "ptr")
+        if not ret or count <= 0:
+            return []
+        arr = self._ct.cast(ret, self._ct.POINTER(self._ct.c_void_p))
+        return [arr[i] for i in range(count)]
+
 
 def from_env() -> Engine:
     """``CtypesEngine`` if ``MEOS_LIBRARY_PATH`` is set, else ``StubEngine``."""

tests/test_engine_integration.py

@@ -142,6 +142,18 @@ def test_array_return_round_trip(self):
                 for p in ptrs]
         self.assertTrue(all("@" in o for o in outs))
 
+    def test_counted_array_return_round_trip(self):
+        # Temporal **temparr_round(Temporal **temp, int count, int maxdd): MEOS
+        # allocates an array as long as the argument array.
+        h1 = self.eng.decode("tfloat_in", "1.23456@2000-01-01")
+        h2 = self.eng.decode("tfloat_in", "7.65432@2000-01-03")
+        ptrs = self.eng.invoke_counted_array(
+            "temparr_round", [("ptrarray", [h1, h2]), ("int", 2), ("int", 2)], 2)
+        self.assertEqual(len(ptrs), 2)
+        outs = [self.eng.encode("tfloat_out", p, [("int", 15)]) for p in ptrs]
+        self.assertIn("1.23@", outs[0])
+        self.assertIn("7.65@", outs[1])
+
     def test_bad_input_raises_not_exits(self):
         with self.assertRaises(MeosError):
             self.eng.decode("tbool_in", "not a temporal value at all")

tests/test_enrich.py

@@ -94,6 +94,25 @@ def fn(name, ret, *params):
     # array; the count out-param is implicit, result is a JSON array.
     fn("temporal_components", "struct Temporal **",
        (T, "temp"), ("int *", "count")),
+    # Array return as long as an array argument: Elem **f(Elem **arr, int count, ..)
+    # returns `count` elements, as `shape.arrayReturn` states; the response is a JSON
+    # array as long as the request's.
+    dict(fn("temparr_round", "struct Temporal **",
+            ("struct Temporal **", "temp"), ("int", "count"), ("int", "maxdd")),
+         shape={"arrayReturn": {"lengthFrom": {"kind": "param", "name": "count"},
+                                "element": {"c": "struct Temporal *",
+                                            "canonical": "struct Temporal *"}}}),
+    # The same return counted by an array of structs the request carries as one value:
+    # its length is not the request's, so it is refused naming that argument.
+    dict(fn("boxarr_round", "struct Box *",
+            ("const struct Box *", "boxarr"), ("int", "count"), ("int", "maxdd")),
+         shape={"inputArrays": [{"param": "boxarr",
+                                 "lengthFrom": {"kind": "param", "name": "count"},
+                                 "element": {"c": "struct Box",
+                                             "canonical": "struct Box"}}],
+                "arrayReturn": {"lengthFrom": {"kind": "param", "name": "count"},
+                                "element": {"c": "struct Box",
+                                            "canonical": "struct Box"}}}),
 ]
 
 STRUCTS = [{"name": n, "fields": []} for n in
@@ -233,6 +252,23 @@ def test_array_return_not_exposable(self):
         self.assertFalse(self.n("temporal_timestamps")["exposable"])
         self.assertIn("unsupported-return:int *", r)
 
+    def test_array_return_as_long_as_an_array_argument(self):
+        f = self.fns["temparr_round"]
+        self.assertTrue(f["network"]["exposable"])
+        self.assertEqual([p["name"] for p in f["wire"]["params"]], ["temp", "maxdd"])
+        self.assertEqual(f["wire"]["params"][0]["count_param"], "count")
+        r = f["wire"]["result"]
+        self.assertEqual(r["kind"], "array")
+        self.assertEqual(r["count_param"], "count")
+        self.assertNotIn("count_outparam", r)
+        self.assertEqual(r["element"]["encode"], "temporal_out")
+
+    def test_array_return_counted_by_an_argument_read_as_one_value(self):
+        f = self.fns["boxarr_round"]
+        self.assertFalse(f["network"]["exposable"])
+        self.assertIn("array-or-out-param:boxarr", f["network"]["reason"])
+        self.assertEqual(f["wire"]["result"], {"kind": "unsupported"})
+
     def test_lifecycle_and_index_not_exposable(self):
         self.assertIn("lifecycle", self.n("meos_initialize")["reason"])
         self.assertIn("index", self.n("rtree_insert")["reason"])
@@ -493,8 +529,9 @@ def test_enrichment_summary(self):
         self.assertEqual(e["internalFunctions"], 1)        # internal_op
         self.assertEqual(e["publicFunctions"], len(FUNCTIONS) - 1)
         # 13 + setspan_value_n + boxset_value_n + temporal_merge_array
-        # + temporal_components (array return); internal_op excluded.
-        self.assertEqual(e["exposableFunctions"], 17)
+        # + temporal_components (array return) + temparr_round (array return as
+        # long as an array argument); internal_op excluded.
+        self.assertEqual(e["exposableFunctions"], 18)
 
 
 if __name__ == "__main__":

tests/test_server.py

@@ -97,6 +97,21 @@ def serialized(name, decode="temporal_in"):
                                          "encode": "tsequence_out",
                                          "encode_aux": [],
                                          "encodings": ["text"]}}}},
+        {"name": "temparr_round", "category": "transformation",
+         "network": {"exposable": True},
+         "wire": {"params": [
+             {"name": "temp", "kind": "array", "count_param": "count",
+              "element": {"kind": "serialized",
+                          "cType": "struct Temporal *",
+                          "decode": "temporal_in", "decode_aux": [],
+                          "encodings": ["text"]}},
+             {"name": "maxdd", "kind": "json", "json": "integer"}],
+                  "result": {"kind": "array", "count_param": "count",
+                             "element": {"kind": "serialized",
+                                         "cType": "struct Temporal *",
+                                         "encode": "temporal_out",
+                                         "encode_aux": [],
+                                         "encodings": ["text"]}}}},
     ],
     "enums": [{"name": "interpType",
                "values": [{"name": "STEP", "value": 0},
@@ -138,6 +153,10 @@ def invoke_array(self, fn, args):
             raise MeosError("boom", 7)
         return [101, 102]
 
+    def invoke_counted_array(self, fn, args, count):
+        self.calls.append(("invoke_counted_array", fn, args, count))
+        return list(range(201, 201 + count))
+
     present = True
 
 
@@ -167,13 +186,13 @@ def test_routes_exclude_non_exposable(self):
         routes = build_routes(CATALOG)
         self.assertIn("/temporal_eq", routes)
         self.assertNotIn("/tsequence_make", routes)
-        self.assertEqual(len(routes), 8)
+        self.assertEqual(len(routes), 9)
 
     def test_health(self):
         st, body = self.req("GET", "/healthz")
         self.assertEqual(st, 200)
         self.assertEqual(body["engine"], "fake")
-        self.assertEqual(body["operations"], 8)
+        self.assertEqual(body["operations"], 9)
 
     def test_unknown_route(self):
         self.assertEqual(self.req("POST", "/nope", {})[0], 404)
@@ -272,6 +291,21 @@ def test_array_return(self):
         encs = [c for c in self.engine.calls if c[0] == "encode"]
         self.assertEqual([c[1] for c in encs], ["tsequence_out"] * 2)
 
+    def test_array_return_as_long_as_an_array_argument(self):
+        st, body = self.req("POST", "/temparr_round",
+                            {"temp": ["a", "b", "c"], "maxdd": 2})
+        self.assertEqual(st, 200)
+        # one encoded element per element of the request's array
+        self.assertEqual(body, {"result": ["ENC(temporal_out)"] * 3})
+        ic = next(c for c in self.engine.calls
+                  if c[0] == "invoke_counted_array")
+        self.assertEqual(ic[1], "temparr_round")
+        self.assertEqual([t for t, _ in ic[2]], ["ptrarray", "int", "int"])
+        self.assertEqual(ic[2][1], ("int", 3))          # the implicit count
+        self.assertEqual(ic[3], 3)                      # the length read back
+        encs = [c for c in self.engine.calls if c[0] == "encode"]
+        self.assertEqual([c[2] for c in encs], [201, 202, 203])
+
     # --- validation ---
     def test_missing_field(self):
         st, body = self.req("POST", "/temporal_eq", {"temp1": "a"})