From 07427207e2f5f7f08aed51400351921123dabab8 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:17:15 -0500 Subject: [PATCH] Fix NumPy serialization, Arrow schemas, and publisher isolation Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- ccflow/exttypes/arrow.py | 2 +- ccflow/exttypes/pydantic_numpy/ndarray.py | 32 +++- ccflow/publishers/composite.py | 7 +- ccflow/tests/exttypes/test_arrow.py | 47 +++++ ccflow/tests/exttypes/test_pydantic_numpy.py | 174 ++++++++++++++++++- ccflow/tests/publishers/test_composite.py | 59 +++++++ 6 files changed, 311 insertions(+), 10 deletions(-) diff --git a/ccflow/exttypes/arrow.py b/ccflow/exttypes/arrow.py index 7f701f65..79118a81 100644 --- a/ccflow/exttypes/arrow.py +++ b/ccflow/exttypes/arrow.py @@ -101,7 +101,7 @@ def _validate(cls, v, schema, strict): extra_cols = [c for c in v.schema.names if c not in schema.names] v_checked = v.drop(extra_cols).cast(schema) for extra in extra_cols: - v_checked = v_checked.append_column(extra, v[extra]) + v_checked = v_checked.append_column(v.schema.field(extra), v[extra]) return v_checked return v diff --git a/ccflow/exttypes/pydantic_numpy/ndarray.py b/ccflow/exttypes/pydantic_numpy/ndarray.py index 868d6bf5..a7822847 100644 --- a/ccflow/exttypes/pydantic_numpy/ndarray.py +++ b/ccflow/exttypes/pydantic_numpy/ndarray.py @@ -48,11 +48,33 @@ def _serialize(cls, v, nxt): @classmethod def __get_pydantic_core_schema__(cls, source_type, handler): - from pydantic_core import core_schema + from pydantic_core import PydanticSerializationUnexpectedValue, core_schema + + subtypes = get_args(source_type) + dtype = subtypes[0] if subtypes and subtypes[0] != Any else None + + serialization_schema = core_schema.any_schema() + definitions = [] + if isinstance(dtype, type) and hasattr(dtype, "__get_pydantic_core_schema__"): + + def _serialize_item(value, nxt): + if not isinstance(value, dtype): + raise PydanticSerializationUnexpectedValue() + return nxt(value) + + item_ref = f"{source_type}:{id(source_type)}:items" + serialization_schema = core_schema.definition_reference_schema(item_ref) + item_schema = core_schema.any_schema( + serialization=core_schema.wrap_serializer_function_ser_schema(_serialize_item, schema=handler.generate_schema(dtype)) + ) + definitions.append( + core_schema.union_schema( + [item_schema, core_schema.list_schema(serialization_schema), core_schema.any_schema()], + ref=item_ref, + ) + ) def _validate(v): - subtypes = get_args(source_type) - dtype = subtypes[0] if subtypes and subtypes[0] != Any else None try: if dtype: return np.asarray(v, dtype=dtype) @@ -61,12 +83,14 @@ def _validate(v): except TypeError: raise ValueError("Unable to convert value to an array.") - return core_schema.no_info_before_validator_function( + schema = core_schema.no_info_before_validator_function( _validate, core_schema.any_schema(), serialization=core_schema.wrap_serializer_function_ser_schema( cls._serialize, info_arg=False, + schema=serialization_schema, return_schema=core_schema.list_schema(), ), ) + return core_schema.definitions_schema(schema, definitions) if definitions else schema diff --git a/ccflow/publishers/composite.py b/ccflow/publishers/composite.py index 4ddd6f8d..c6fecb22 100644 --- a/ccflow/publishers/composite.py +++ b/ccflow/publishers/composite.py @@ -59,8 +59,9 @@ def _get_publishers(self, data): if not publisher: for try_publisher in self.default_publishers: try: - try_publisher.data = value - publishers[field] = try_publisher.model_copy() + candidate = try_publisher.model_copy() + candidate.data = value + publishers[field] = candidate publishers[field].name = full_name publishers[field].name_params = self.name_params break @@ -76,9 +77,9 @@ def _get_publishers(self, data): else: # If value is the wrong type for the configured publisher, it will raise # User should provide the right type of publisher for a given field. + publisher = publisher.model_copy() publisher.data = value if not publisher.name: - publisher = publisher.model_copy() publisher.name = full_name publisher.name_params = self.name_params publishers[field] = publisher diff --git a/ccflow/tests/exttypes/test_arrow.py b/ccflow/tests/exttypes/test_arrow.py index 2e82d7b9..d43b3004 100644 --- a/ccflow/tests/exttypes/test_arrow.py +++ b/ccflow/tests/exttypes/test_arrow.py @@ -153,6 +153,53 @@ def test_record_batches(self): m = Model(strict=[batch]) self.assertEqual(m.strict, pa.Table.from_pydict(data, RAW_SCHEMA)) + def test_weak_preserves_extra_field_metadata(self): + extra_fields = [ + pa.field("d", pa.int64(), metadata={b"unit": b"shares"}), + pa.field("e", pa.string(), metadata={b"source": b"example"}), + ] + schema = pa.schema([*RAW_SCHEMA, *extra_fields]) + data = {**self.data, "e": ["foo", None, "baz"]} + table = pa.Table.from_pydict(data, schema=schema) + + result = Model(weak=table).weak + + self.assertTrue(result.schema.equals(schema, check_metadata=True)) + self.assertEqual(result.to_pydict(), table.to_pydict()) + self.assertTrue(table.schema.equals(schema, check_metadata=True)) + + def test_weak_preserves_extra_field_nullability(self): + extra = pa.field("d", pa.int64(), nullable=False) + schema = pa.schema([*RAW_SCHEMA, extra]) + table = pa.Table.from_pydict(self.data, schema=schema) + + result = Model(weak=table).weak + + self.assertTrue(result.schema.field("d").equals(extra, check_metadata=True)) + self.assertEqual(result.column("d"), table.column("d")) + + def test_weak_preserves_extra_fields_from_batches(self): + extra = pa.field("d", pa.int64(), nullable=False, metadata={b"unit": b"shares"}) + source_schema = pa.schema([pa.field("a", pa.float64()), pa.field("b", pa.int64()), RAW_SCHEMA.field("c"), extra]) + expected_schema = pa.schema([*RAW_SCHEMA, extra]) + inputs = { + "empty": [pa.RecordBatch.from_pydict({key: [] for key in self.data}, schema=source_schema)], + "chunked": [ + pa.RecordBatch.from_pydict({key: values[:1] for key, values in self.data.items()}, schema=source_schema), + pa.RecordBatch.from_pydict({key: values[1:] for key, values in self.data.items()}, schema=source_schema), + ], + } + for name, batches in inputs.items(): + with self.subTest(name=name): + table = pa.Table.from_batches(batches) + result = Model(weak=batches, filter=batches) + + self.assertTrue(result.weak.schema.equals(expected_schema, check_metadata=True)) + self.assertEqual(result.weak.column("d"), table.column("d")) + self.assertEqual(result.weak.column("d").num_chunks, table.column("d").num_chunks) + self.assertTrue(result.filter.schema.equals(RAW_SCHEMA, check_metadata=True)) + self.assertRaises(ValidationError, Model, strict=batches) + def test_bad_type(self): t = pa.Table.from_pydict(self.data) data_bad = {"a": ["oops", 2, 3], "b": [4, 5, 6], "c": ["foo", "bar", "baz"]} diff --git a/ccflow/tests/exttypes/test_pydantic_numpy.py b/ccflow/tests/exttypes/test_pydantic_numpy.py index 9eb72bf8..cd8f66f6 100644 --- a/ccflow/tests/exttypes/test_pydantic_numpy.py +++ b/ccflow/tests/exttypes/test_pydantic_numpy.py @@ -23,11 +23,16 @@ SOFTWARE. """ +import json +from enum import IntEnum + import numpy as np +import pytest from numpy.testing import assert_allclose -from pydantic import BaseModel +from pydantic import BaseModel, Field, TypeAdapter +from pydantic_core import PydanticSerializationError, core_schema -from ccflow import NDArray, float32 +from ccflow import Enum, NDArray, float32 class MySettings(BaseModel): @@ -128,3 +133,168 @@ def test_union_field(): assert 0 in model2.x assert len(model2.x) == 1 np.testing.assert_array_equal(model2.x[0], np.array([0.0, 1.0, 2.0])) + + +class SerializedLabel: + def __init__(self, value): + self.value = value + + @classmethod + def __get_pydantic_core_schema__(cls, source_type, handler): + def serialize(value, info): + prefix = (getattr(info, "context", None) or {}).get("prefix", "") + return prefix + value.value + + return core_schema.is_instance_schema( + cls, + serialization=core_schema.plain_serializer_function_ser_schema(serialize, info_arg=True, when_used="json"), + ) + + +@pytest.mark.parametrize( + "shape,expected", + [((2,), ["BUY", "SELL"]), ((1, 2), [["BUY", "SELL"]]), ((2, 1, 1), [[["BUY"]], [["SELL"]]])], +) +def test_custom_item_serializer(shape, expected): + adapter = TypeAdapter(NDArray[SerializedLabel]) + array = adapter.validate_python(np.array([SerializedLabel("BUY"), SerializedLabel("SELL")], dtype=object).reshape(shape)) + original = array.copy() + assert json.loads(adapter.dump_json(array)) == expected + assert array.dtype == object + np.testing.assert_array_equal(array, original) + + +def test_custom_item_serializer_retains_python_mode(): + value = SerializedLabel("BUY") + adapter = TypeAdapter(NDArray[SerializedLabel]) + array = adapter.validate_python([value]) + assert adapter.dump_python(array)[0] is value + assert adapter.dump_python(array, mode="json") == ["BUY"] + + +def test_custom_item_serializer_respects_index_filters(): + adapter = TypeAdapter(NDArray[SerializedLabel]) + array = adapter.validate_python([SerializedLabel("BUY"), SerializedLabel("SELL")]) + assert json.loads(adapter.dump_json(array, include={1})) == ["SELL"] + assert json.loads(adapter.dump_json(array, exclude={0})) == ["SELL"] + + +@pytest.mark.parametrize("shape,expected", [((0,), []), ((2, 0), [[], []])]) +def test_custom_item_serializer_empty_arrays(shape, expected): + adapter = TypeAdapter(NDArray[SerializedLabel]) + assert json.loads(adapter.dump_json(np.empty(shape, dtype=object))) == expected + + +def test_custom_item_serializer_in_optional_field(): + class ArrayModel(BaseModel): + values: NDArray[SerializedLabel] | None = None + + assert json.loads(ArrayModel().model_dump_json()) == {"values": None} + assert json.loads(ArrayModel(values=[SerializedLabel("BUY")]).model_dump_json()) == {"values": ["BUY"]} + + +def test_custom_item_serializer_uses_context(): + import inspect + + if "context" not in inspect.signature(TypeAdapter.dump_json).parameters: + pytest.skip("serialization context is unavailable in this Pydantic version") + adapter = TypeAdapter(NDArray[SerializedLabel]) + array = adapter.validate_python([SerializedLabel("BUY")]) + assert json.loads(adapter.dump_json(array, context={"prefix": "trade:"})) == ["trade:BUY"] + + +def test_custom_item_serializer_preserves_model_options(): + class Payload(BaseModel): + value: str = Field(alias="label") + optional: str | None = None + + adapter = TypeAdapter(NDArray[Payload]) + array = adapter.validate_python([Payload(label="BUY"), Payload(label="SELL")]) + assert json.loads(adapter.dump_json(array, by_alias=True, exclude_none=True)) == [{"label": "BUY"}, {"label": "SELL"}] + assert adapter.dump_python(array, include={1: {"value"}}) == [{"value": "SELL"}] + + +def test_custom_item_serializer_honors_ccflow_enum_names(): + class Side(Enum): + BUY = Enum.auto() + SELL = Enum.auto() + + adapter = TypeAdapter(NDArray[Side]) + array = adapter.validate_python(np.array([Side.BUY, Side.SELL], dtype=object)) + assert json.loads(adapter.dump_json(array)) == ["BUY", "SELL"] + + +def test_untyped_object_array_does_not_guess_custom_schema(): + with pytest.raises(PydanticSerializationError): + TypeAdapter(NDArray).dump_json(np.array([SerializedLabel("BUY")], dtype=object)) + + +def test_native_numpy_dtype_serialization_unchanged(): + adapter = TypeAdapter(NDArray[np.int64]) + array = adapter.validate_python([[1, 2], [3, 4]]) + assert json.loads(adapter.dump_json(array)) == [[1, 2], [3, 4]] + + +@pytest.mark.parametrize("serializer", [lambda value: value.name, lambda value: {0: "BUY", 1: "SELL"}[value]], ids=["attribute", "mapping"]) +def test_numeric_array_does_not_apply_object_item_serializer(serializer): + class NumericSide(IntEnum): + BUY = 0 + SELL = 1 + + @classmethod + def __get_pydantic_core_schema__(cls, source_type, handler): + return core_schema.is_instance_schema( + cls, + serialization=core_schema.plain_serializer_function_ser_schema(serializer, when_used="json"), + ) + + adapter = TypeAdapter(NDArray[NumericSide]) + array = adapter.validate_python(np.array([NumericSide.BUY, NumericSide.SELL])) + # NumPy has already coerced these values to integers, not enum instances. + assert all(not isinstance(value, NumericSide) for value in array.flat) + assert json.loads(adapter.dump_json(array)) == [0, 1] + + +def test_custom_item_serializer_keeps_list_subclass_items(): + class ListLabel(list): + @classmethod + def __get_pydantic_core_schema__(cls, source_type, handler): + return core_schema.is_instance_schema( + cls, + serialization=core_schema.plain_serializer_function_ser_schema(lambda value: "LABEL", when_used="json"), + ) + + array = np.empty(1, dtype=object) + array[0] = ListLabel([1, 2]) + adapter = TypeAdapter(NDArray[ListLabel]) + assert json.loads(adapter.dump_json(adapter.validate_python(array))) == ["LABEL"] + + +def test_custom_item_serializer_uses_model_config(): + class BinaryLabel: + @classmethod + def __get_pydantic_core_schema__(cls, source_type, handler): + return core_schema.is_instance_schema( + cls, + serialization=core_schema.plain_serializer_function_ser_schema(lambda value: b"\xff", when_used="json"), + ) + + class ArrayModel(BaseModel): + model_config = {"ser_json_bytes": "base64"} + values: NDArray[BinaryLabel] + + assert json.loads(ArrayModel(values=[BinaryLabel()]).model_dump_json()) == {"values": ["_w=="]} + + +def test_custom_item_serializer_uses_fallback(): + import inspect + + if "fallback" not in inspect.signature(TypeAdapter.dump_json).parameters: + pytest.skip("serialization fallback is unavailable in this Pydantic version") + + class Payload(BaseModel): + value: object + + adapter = TypeAdapter(NDArray[Payload]) + array = adapter.validate_python([Payload(value=object())]) + assert json.loads(adapter.dump_json(array, fallback=lambda value: "opaque")) == [{"value": "opaque"}] diff --git a/ccflow/tests/publishers/test_composite.py b/ccflow/tests/publishers/test_composite.py index c534ec19..08da0bdf 100644 --- a/ccflow/tests/publishers/test_composite.py +++ b/ccflow/tests/publishers/test_composite.py @@ -125,6 +125,65 @@ def test_root_publisher(self): } self.assertDictEqual(paths, target) + def test_shared_field_and_default_publisher(self): + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + shared = GenericFilePublisher(name=str(root / "custom_a"), suffix=".txt") + publisher = CompositePublisher( + name=str(root / "batch"), + data={"a": "ALPHA", "b": "BETA"}, + field_publishers={"a": shared}, + default_publishers=[shared], + ) + + paths = publisher() + + self.assertEqual(paths, {"a": root / "custom_a.txt", "b": root / "batch" / "b.txt"}) + self.assertEqual({field: path.read_text() for field, path in paths.items()}, {"a": "ALPHA", "b": "BETA"}) + + def test_publisher_templates_are_not_modified(self): + for named in (False, True): + with self.subTest(named=named), tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + template = GenericFilePublisher( + name=str(root / "{{tag}}") if named else None, + name_params={"tag": "template"}, + data="SEED", + suffix=".txt", + ) + original_name = template.name + publisher = CompositePublisher( + name=str(root / "batch"), + name_params={"tag": "composite"}, + data={"a": "ALPHA", "b": "BETA"}, + field_publishers={"a": template}, + default_publishers=[template], + ) + + paths = publisher() + + self.assertEqual({field: path.read_text() for field, path in paths.items()}, {"a": "ALPHA", "b": "BETA"}) + self.assertEqual(template.data, "SEED") + self.assertEqual(template.name, original_name) + self.assertEqual(template.name_params, {"tag": "template"}) + + def test_default_validation_fallback_preserves_templates(self): + with tempfile.TemporaryDirectory() as tempdir: + rejected = DummyPublisher(data=(42,)) + accepted = GenericFilePublisher(data="SEED", suffix=".txt") + publisher = CompositePublisher( + name=str(Path(tempdir) / "batch"), + data={"a": "ALPHA", "b": "BETA"}, + default_publishers=[rejected, accepted], + ) + + paths = publisher() + + self.assertEqual({field: path.read_text() for field, path in paths.items()}, {"a": "ALPHA", "b": "BETA"}) + self.assertEqual(rejected.data, (42,)) + self.assertEqual(accepted.data, "SEED") + self.assertIsNone(accepted.name) + def test_skip_root_publisher(self): with tempfile.TemporaryDirectory() as tempdir: os.chdir(tempdir)