diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index a4facd8..7a3cc40 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -96,7 +96,7 @@ jobs: # carries, or a change to them is not exercised until after it merges. # Consumers use the action; this repository owns the rules. - name: Refuse a skip, and a suite that shrank - run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 469 + run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 534 # The rules earn their place by refusing a log that carries what they # name. Both fixtures are written here rather than tracked, and the diff --git a/docs/enrichment.md b/docs/enrichment.md index d60db36..fb3b545 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -181,6 +181,13 @@ Otherwise `exposable` is `false` and `reason` lists the blockers > (element = the serialized `Elem`, `count_outparam` names the byref > length). Recovers `temporal_instants`/`segments`/`sequences`, > `tgeo_values`, `geo_pointarr`, … whose element type is encodable. +> An array as long as an input array, `Elem **f(Elem **arr, int count, ..)` +> (`shape.arrayReturn.lengthFrom` names the by-value `count`), becomes the +> same `kind: "array"` result with `count_param` naming that length: the +> response holds one element per element of the request's array +> (`temparr_round`, `cbufferarr_round`, `posearr_round`). One counted by an +> argument the request carries as one value is refused naming it +> (`stboxarr_round`: `array-or-out-param:boxarr`). This is the precise, machine-checkable boundary of what a generator can emit today. Functions still blocked only by `array-or-out-param` (multi- or diff --git a/parser/enrich.py b/parser/enrich.py index ff031c7..c840393 100644 --- a/parser/enrich.py +++ b/parser/enrich.py @@ -434,6 +434,41 @@ def _array_return(fn: dict, type_encodings: dict): return cnt[0]["name"], rb +def _counted_array_return(fn: dict, type_encodings: dict, params: list, count_i): + """The array a function returns as long as an array argument, as + ``shape.arrayReturn`` states it when its ``lengthFrom`` is a by-value parameter + (``temparr_round(Temporal **temp, int count, int maxdd)`` returns ``count`` + values). Returns ``(length_param, elem_base, refusal)``. + + The response holds one encoded element per element of the request's array, so the + return is served as #_array_return serves one, an array of encodable element + pointers, when its length is the length #_array_param gives the request's array + (``params[count_i]``). An array whose length is that of an argument the request + does not carry as an array cannot be read back at its length, and is refused + naming that argument (``stboxarr_round`` reads ``boxarr`` as one box); an array of + elements that are not encodable pointers is refused naming the return. A function + with no such ``arrayReturn`` is ``(None, None, None)``, and #assess reads its + return as any other.""" + ar = (fn.get("shape") or {}).get("arrayReturn") or {} + length = (ar.get("lengthFrom") or {}).get("name") + by_value = {p["name"] for p in fn.get("params", []) + if _ptr_depth(p["canonical"]) == 0} + if length not in by_value: + return None, None, None + ret = fn["returnType"]["canonical"] + if count_i is None or params[count_i]["name"] != length: + source = next((a["param"] + for a in (fn.get("shape") or {}).get("inputArrays") or () + if a["lengthFrom"].get("name") == length), length) + return length, None, f"array-or-out-param:{source}" + te = type_encodings.get(_base(ret)) + if _ptr_depth(ret) != 2: + return length, None, f"unsupported-return:{ret}" + if not (te and te.get("out")): + return length, None, f"no-encoder:{_base(ret)}" + return length, _base(ret), None + + def assess(fn: dict, type_encodings: dict, enums: set) -> tuple: """Return ``(network, wire)`` for one function. @@ -452,6 +487,9 @@ def assess(fn: dict, type_encodings: dict, enums: set) -> tuple: ret_count_i = (next((i for i, p in enumerate(eff) if p["name"] == ret_count_name), None) if ret_elem is not None else None) + ret_length, counted_elem, counted_refusal = ( + _counted_array_return(fn, type_encodings, eff, count_i) + if ret_elem is None and out_p is None else (None, None, None)) for idx, p in enumerate(eff): if idx == count_i or idx == ret_count_i: @@ -529,6 +567,23 @@ def assess(fn: dict, type_encodings: dict, enums: set) -> tuple: "count_outparam": ret_count_name, } scalar = "handled" + elif counted_elem is not None: + te = type_encodings[counted_elem] + wire_result = { + "kind": "array", + "element": { + "kind": "serialized", + "cType": " ".join(ret.replace("*", " ").split()) + " *", + "encode": te["out"], "encode_aux": te.get("out_aux", []), + "encodings": te["encodings"], + }, + "count_param": ret_length, + } + scalar = "handled" + elif counted_refusal is not None: + reasons.append(counted_refusal) + wire_result = {"kind": "unsupported"} + scalar = "handled" else: scalar = _scalar_wire(ret, enums) if scalar == "handled": diff --git a/parser/shapeinfer.py b/parser/shapeinfer.py index ece31f2..7080386 100644 --- a/parser/shapeinfer.py +++ b/parser/shapeinfer.py @@ -7,6 +7,8 @@ TYPE **f(..., TYPE **extra, int *count) -> primary array return PLUS one or more parallel out-arrays f(..., TYPE **values, int count, ...) -> reads an array of ``count`` + TYPE **f(..., TYPE **values, int count) -> reads and returns an array of + ``count`` The output length is always passed *by pointer* (``int *count``); an *input* array instead carries its length *by value* (``int count``). That pointer/value @@ -212,6 +214,48 @@ def attach_struct_input_arrays(idl: dict, docs: dict) -> tuple[dict, int]: return idl, n +def attach_counted_array_returns(idl: dict) -> tuple[dict, int]: + """(idl, number of returns stated) once ``shape.arrayReturn`` also states each array + a function returns as long as an array it reads. + + ``temparr_round(Temporal **temp, int count, int maxdd)`` answers one rounded value + per value it reads, so its ``Temporal **`` holds ``count`` elements, and no ``int + *count`` out-parameter says so, as #infer_shapes requires. The types say it: the + return is the element of a stated input array with one more pointer level, which is + how an array of that element is spelled. ``stboxarr_round(const STBox *boxarr, int + count, int maxdd)`` reads ``count`` boxes and returns ``count`` boxes as ``STBox *``. + A return of the element type itself is one element, not an array: + ``temporal_merge_array(Temporal **temparr, int count)`` returns one ``Temporal *``. + + The array return takes its length from that input array's ``lengthFrom`` and reads + its ``element`` from the return, as #infer_shapes states both. It runs after + #attach_struct_input_arrays, which states the arrays of structs it compares with.""" + n = 0 + for func in idl["functions"]: + shape = func.get("shape") or {} + if "arrayReturn" in shape or _out_count_param(func): + continue + rtype = func.get("returnType", {}) + ret = rtype.get("c", "") + if not ret.rstrip().endswith("*"): + continue + elem = _bare(_strip_one_ptr(ret)) + lengths = {a["lengthFrom"]["name"]: a["lengthFrom"] + for a in shape.get("inputArrays") or () + if _bare(a["element"]["c"]) == elem} + if len(lengths) != 1: + continue + shape["arrayReturn"] = { + "lengthFrom": dict(next(iter(lengths.values()))), + "element": { + "c": _strip_one_ptr(ret), + "canonical": _strip_one_ptr(rtype.get("canonical", ret)), + }, + } + n += 1 + return idl, n + + def _is_index_pair_return(func: dict, count: str) -> bool: """Whether the ``int *`` return is a FLATTENED array of index PAIRS. diff --git a/run.py b/run.py index 62657b2..bf8fd33 100644 --- a/run.py +++ b/run.py @@ -11,7 +11,8 @@ from parser.typerecover import (recover_collapsed_types, normalize_canonical, postgres_scalar_names) from parser.header_types import reconcile -from parser.shapeinfer import infer_shapes, attach_struct_input_arrays +from parser.shapeinfer import (infer_shapes, attach_struct_input_arrays, + attach_counted_array_returns) from parser.nullable import merge_nullable, extract_param_docs from parser.nullresult import attach_null_result from parser.outparam import extract_param_names, merge_outparams @@ -164,6 +165,8 @@ def main(): idl, nsa = attach_struct_input_arrays(idl, extract_param_docs(_doxy_root)) print(f" input arrays of structs from Doxygen `Array of`: {nsa}", file=sys.stderr) + idl, ncr = attach_counted_array_returns(idl) + print(f" array returns as long as an input array: {ncr}", file=sys.stderr) idl, no, out_drift = merge_outparams(idl, _doxy_root) print(f" out params from Doxygen `@param[out]`: {no}", file=sys.stderr) if out_drift: diff --git a/server/app.py b/server/app.py index cc3732e..0350959 100644 --- a/server/app.py +++ b/server/app.py @@ -74,7 +74,8 @@ def build_routes(catalog: dict) -> dict: "out_ctype": r.get("out_ctype"), "presence_return": r.get("presence_return", False), "element": r.get("element"), - "count_outparam": r.get("count_outparam")}, + "count_outparam": r.get("count_outparam"), + "count_param": r.get("count_param")}, } return routes @@ -168,9 +169,17 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict): return {"result": bool(val) if res.get("json") == "boolean" else val} if res["kind"] == "array": - # Elem **f(.., int *count): MEOS returns a fresh array + byref count. el = res["element"] - ptrs = engine.invoke_array(route["name"], args) + if res.get("count_param"): + # Elem **f(Elem **arr, int count, ..): one element per element of + # the request's array of that count. + n = next(len(body[p["name"]]) for p in route["params"] + if p["kind"] == "array" + and p["count_param"] == res["count_param"]) + ptrs = engine.invoke_counted_array(route["name"], args, n) + else: + # Elem **f(.., int *count): MEOS returns a fresh array + byref count. + ptrs = engine.invoke_array(route["name"], args) return {"result": [engine.encode(el["encode"], p, _aux_args(el.get("encode_aux", []))) for p in ptrs]} diff --git a/server/engine.py b/server/engine.py index 1a2cb81..7bd6323 100644 --- a/server/engine.py +++ b/server/engine.py @@ -58,6 +58,11 @@ def invoke_array(self, fn_name: str, args: list): """``Elem **f(.., int *count)`` — return a list of element handles.""" raise NotImplementedError + def invoke_counted_array(self, fn_name: str, args: list, count: int): + """``Elem **f(Elem **arr, int count, ..)`` — return the ``count`` element + handles of an array as long as an array argument.""" + raise NotImplementedError + def close(self) -> None: pass @@ -87,6 +92,9 @@ def invoke_outparam(self, fn_name, args, out_ctype, presence): def invoke_array(self, fn_name, args): return [] + def invoke_counted_array(self, fn_name, args, count): + return [] + class CtypesEngine(Engine): """Calls a built libmeos via ctypes, driven entirely by the wire model. @@ -250,6 +258,15 @@ def invoke_array(self, fn_name, args): arr = ct.cast(ret, ct.POINTER(ct.c_void_p)) return [arr[i] for i in range(count)] + def invoke_counted_array(self, fn_name, args, count): + # Elem **f(Elem **arr, int count, ...): MEOS allocates an array as long + # as the argument array, so its length is the one the caller passed. + ret = self.invoke(fn_name, args, "ptr") + if not ret or count <= 0: + return [] + arr = self._ct.cast(ret, self._ct.POINTER(self._ct.c_void_p)) + return [arr[i] for i in range(count)] + def from_env() -> Engine: """``CtypesEngine`` if ``MEOS_LIBRARY_PATH`` is set, else ``StubEngine``.""" diff --git a/tests/test_engine_integration.py b/tests/test_engine_integration.py index 7a8f2dd..0088def 100644 --- a/tests/test_engine_integration.py +++ b/tests/test_engine_integration.py @@ -142,6 +142,18 @@ def test_array_return_round_trip(self): for p in ptrs] self.assertTrue(all("@" in o for o in outs)) + def test_counted_array_return_round_trip(self): + # Temporal **temparr_round(Temporal **temp, int count, int maxdd): MEOS + # allocates an array as long as the argument array. + h1 = self.eng.decode("tfloat_in", "1.23456@2000-01-01") + h2 = self.eng.decode("tfloat_in", "7.65432@2000-01-03") + ptrs = self.eng.invoke_counted_array( + "temparr_round", [("ptrarray", [h1, h2]), ("int", 2), ("int", 2)], 2) + self.assertEqual(len(ptrs), 2) + outs = [self.eng.encode("tfloat_out", p, [("int", 15)]) for p in ptrs] + self.assertIn("1.23@", outs[0]) + self.assertIn("7.65@", outs[1]) + def test_bad_input_raises_not_exits(self): with self.assertRaises(MeosError): self.eng.decode("tbool_in", "not a temporal value at all") diff --git a/tests/test_enrich.py b/tests/test_enrich.py index 9a72e0a..d149bed 100644 --- a/tests/test_enrich.py +++ b/tests/test_enrich.py @@ -94,6 +94,25 @@ def fn(name, ret, *params): # array; the count out-param is implicit, result is a JSON array. fn("temporal_components", "struct Temporal **", (T, "temp"), ("int *", "count")), + # Array return as long as an array argument: Elem **f(Elem **arr, int count, ..) + # returns `count` elements, as `shape.arrayReturn` states; the response is a JSON + # array as long as the request's. + dict(fn("temparr_round", "struct Temporal **", + ("struct Temporal **", "temp"), ("int", "count"), ("int", "maxdd")), + shape={"arrayReturn": {"lengthFrom": {"kind": "param", "name": "count"}, + "element": {"c": "struct Temporal *", + "canonical": "struct Temporal *"}}}), + # The same return counted by an array of structs the request carries as one value: + # its length is not the request's, so it is refused naming that argument. + dict(fn("boxarr_round", "struct Box *", + ("const struct Box *", "boxarr"), ("int", "count"), ("int", "maxdd")), + shape={"inputArrays": [{"param": "boxarr", + "lengthFrom": {"kind": "param", "name": "count"}, + "element": {"c": "struct Box", + "canonical": "struct Box"}}], + "arrayReturn": {"lengthFrom": {"kind": "param", "name": "count"}, + "element": {"c": "struct Box", + "canonical": "struct Box"}}}), ] STRUCTS = [{"name": n, "fields": []} for n in @@ -233,6 +252,23 @@ def test_array_return_not_exposable(self): self.assertFalse(self.n("temporal_timestamps")["exposable"]) self.assertIn("unsupported-return:int *", r) + def test_array_return_as_long_as_an_array_argument(self): + f = self.fns["temparr_round"] + self.assertTrue(f["network"]["exposable"]) + self.assertEqual([p["name"] for p in f["wire"]["params"]], ["temp", "maxdd"]) + self.assertEqual(f["wire"]["params"][0]["count_param"], "count") + r = f["wire"]["result"] + self.assertEqual(r["kind"], "array") + self.assertEqual(r["count_param"], "count") + self.assertNotIn("count_outparam", r) + self.assertEqual(r["element"]["encode"], "temporal_out") + + def test_array_return_counted_by_an_argument_read_as_one_value(self): + f = self.fns["boxarr_round"] + self.assertFalse(f["network"]["exposable"]) + self.assertIn("array-or-out-param:boxarr", f["network"]["reason"]) + self.assertEqual(f["wire"]["result"], {"kind": "unsupported"}) + def test_lifecycle_and_index_not_exposable(self): self.assertIn("lifecycle", self.n("meos_initialize")["reason"]) self.assertIn("index", self.n("rtree_insert")["reason"]) @@ -493,8 +529,9 @@ def test_enrichment_summary(self): self.assertEqual(e["internalFunctions"], 1) # internal_op self.assertEqual(e["publicFunctions"], len(FUNCTIONS) - 1) # 13 + setspan_value_n + boxset_value_n + temporal_merge_array - # + temporal_components (array return); internal_op excluded. - self.assertEqual(e["exposableFunctions"], 17) + # + temporal_components (array return) + temparr_round (array return as + # long as an array argument); internal_op excluded. + self.assertEqual(e["exposableFunctions"], 18) if __name__ == "__main__": diff --git a/tests/test_server.py b/tests/test_server.py index 5fa9333..35cba11 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -97,6 +97,21 @@ def serialized(name, decode="temporal_in"): "encode": "tsequence_out", "encode_aux": [], "encodings": ["text"]}}}}, + {"name": "temparr_round", "category": "transformation", + "network": {"exposable": True}, + "wire": {"params": [ + {"name": "temp", "kind": "array", "count_param": "count", + "element": {"kind": "serialized", + "cType": "struct Temporal *", + "decode": "temporal_in", "decode_aux": [], + "encodings": ["text"]}}, + {"name": "maxdd", "kind": "json", "json": "integer"}], + "result": {"kind": "array", "count_param": "count", + "element": {"kind": "serialized", + "cType": "struct Temporal *", + "encode": "temporal_out", + "encode_aux": [], + "encodings": ["text"]}}}}, ], "enums": [{"name": "interpType", "values": [{"name": "STEP", "value": 0}, @@ -138,6 +153,10 @@ def invoke_array(self, fn, args): raise MeosError("boom", 7) return [101, 102] + def invoke_counted_array(self, fn, args, count): + self.calls.append(("invoke_counted_array", fn, args, count)) + return list(range(201, 201 + count)) + present = True @@ -167,13 +186,13 @@ def test_routes_exclude_non_exposable(self): routes = build_routes(CATALOG) self.assertIn("/temporal_eq", routes) self.assertNotIn("/tsequence_make", routes) - self.assertEqual(len(routes), 8) + self.assertEqual(len(routes), 9) def test_health(self): st, body = self.req("GET", "/healthz") self.assertEqual(st, 200) self.assertEqual(body["engine"], "fake") - self.assertEqual(body["operations"], 8) + self.assertEqual(body["operations"], 9) def test_unknown_route(self): self.assertEqual(self.req("POST", "/nope", {})[0], 404) @@ -272,6 +291,21 @@ def test_array_return(self): encs = [c for c in self.engine.calls if c[0] == "encode"] self.assertEqual([c[1] for c in encs], ["tsequence_out"] * 2) + def test_array_return_as_long_as_an_array_argument(self): + st, body = self.req("POST", "/temparr_round", + {"temp": ["a", "b", "c"], "maxdd": 2}) + self.assertEqual(st, 200) + # one encoded element per element of the request's array + self.assertEqual(body, {"result": ["ENC(temporal_out)"] * 3}) + ic = next(c for c in self.engine.calls + if c[0] == "invoke_counted_array") + self.assertEqual(ic[1], "temparr_round") + self.assertEqual([t for t, _ in ic[2]], ["ptrarray", "int", "int"]) + self.assertEqual(ic[2][1], ("int", 3)) # the implicit count + self.assertEqual(ic[3], 3) # the length read back + encs = [c for c in self.engine.calls if c[0] == "encode"] + self.assertEqual([c[2] for c in encs], [201, 202, 203]) + # --- validation --- def test_missing_field(self): st, body = self.req("POST", "/temporal_eq", {"temp1": "a"}) diff --git a/tests/test_shapeinfer.py b/tests/test_shapeinfer.py index ccaac84..9f8fd8b 100644 --- a/tests/test_shapeinfer.py +++ b/tests/test_shapeinfer.py @@ -11,7 +11,8 @@ """ import unittest -from parser.shapeinfer import infer_shapes, attach_struct_input_arrays +from parser.shapeinfer import (infer_shapes, attach_struct_input_arrays, + attach_counted_array_returns) def _fn(name, ret, params): @@ -266,5 +267,69 @@ def test_an_array_read_at_an_index_is_no_input_array(self): self.assertEqual(n, 0) +class CountedArrayReturnTests(unittest.TestCase): + """#attach_counted_array_returns states an array return as long as an input array.""" + + def test_an_array_of_the_input_arrays_element_is_as_long_as_it(self): + idl = {"functions": [_fn("temparr_round", "Temporal **", + [("temp", "Temporal **"), ("count", "int"), + ("maxdd", "int")])]} + idl, _ = infer_shapes(idl) + idl, n = attach_counted_array_returns(idl) + self.assertEqual(n, 1) + self.assertEqual(idl["functions"][0]["shape"]["arrayReturn"], { + "lengthFrom": {"kind": "param", "name": "count"}, + "element": {"c": "Temporal *", "canonical": "Temporal *"}}) + + def test_an_array_of_structs_is_as_long_as_the_array_of_structs_read(self): + idl = {"functions": [_fn("stboxarr_round", "STBox *", + [("boxarr", "const STBox *"), ("count", "int"), + ("maxdd", "int")])]} + docs = {"stboxarr_round": {"boxarr": "Array of spatiotemporal boxes", + "count": "Number of elements in the array", + "maxdd": "Maximum number of decimal digits"}} + idl, _ = infer_shapes(idl) + idl, _ = attach_struct_input_arrays(idl, docs) + idl, n = attach_counted_array_returns(idl) + self.assertEqual(n, 1) + self.assertEqual(idl["functions"][0]["shape"]["arrayReturn"], { + "lengthFrom": {"kind": "param", "name": "count"}, + "element": {"c": "STBox", "canonical": "STBox"}}) + + def test_one_element_of_the_input_arrays_type_is_no_array(self): + # Refuting case: the return is the element type itself, one value made of + # the `count` read, so it states no array return. + idl = {"functions": [_fn("temporal_merge_array", "Temporal *", + [("temparr", "Temporal **"), ("count", "int")])]} + idl, _ = infer_shapes(idl) + idl, n = attach_counted_array_returns(idl) + self.assertEqual(n, 0) + self.assertNotIn("arrayReturn", idl["functions"][0]["shape"]) + + def test_one_struct_and_an_integer_is_no_array(self): + # Refuting case: without an input array there is no length to take. + idl = {"functions": [_fn("stbox_round", "STBox *", + [("box", "const STBox *"), ("maxdd", "int")])]} + docs = {"stbox_round": {"box": "Spatiotemporal box", + "maxdd": "Maximum number of decimal digits"}} + idl, _ = infer_shapes(idl) + idl, _ = attach_struct_input_arrays(idl, docs) + idl, n = attach_counted_array_returns(idl) + self.assertEqual(n, 0) + self.assertNotIn("arrayReturn", idl["functions"][0].get("shape") or {}) + + def test_an_output_count_keeps_its_length(self): + # geo_cluster_intersecting reads `ngeoms` geometries and returns `*count` + # clusters: the output count, not the input length, is the length. + idl = {"functions": [_fn("geo_cluster_intersecting", "GSERIALIZED **", + [("geoms", "const GSERIALIZED **"), + ("ngeoms", "int"), ("count", "int *")])]} + idl, _ = infer_shapes(idl) + idl, n = attach_counted_array_returns(idl) + self.assertEqual(n, 0) + self.assertEqual(idl["functions"][0]["shape"]["arrayReturn"]["lengthFrom"], + {"kind": "param", "name": "count"}) + + if __name__ == "__main__": unittest.main()