Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ccflow/exttypes/arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def _validate(cls, v, schema, strict):
extra_cols = [c for c in v.schema.names if c not in schema.names]
v_checked = v.drop(extra_cols).cast(schema)
for extra in extra_cols:
v_checked = v_checked.append_column(extra, v[extra])
v_checked = v_checked.append_column(v.schema.field(extra), v[extra])
return v_checked
return v

Expand Down
32 changes: 28 additions & 4 deletions ccflow/exttypes/pydantic_numpy/ndarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,33 @@ def _serialize(cls, v, nxt):

@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
from pydantic_core import core_schema
from pydantic_core import PydanticSerializationUnexpectedValue, core_schema

subtypes = get_args(source_type)
dtype = subtypes[0] if subtypes and subtypes[0] != Any else None

serialization_schema = core_schema.any_schema()
definitions = []
if isinstance(dtype, type) and hasattr(dtype, "__get_pydantic_core_schema__"):

def _serialize_item(value, nxt):
if not isinstance(value, dtype):
raise PydanticSerializationUnexpectedValue()
return nxt(value)

item_ref = f"{source_type}:{id(source_type)}:items"
serialization_schema = core_schema.definition_reference_schema(item_ref)
item_schema = core_schema.any_schema(
serialization=core_schema.wrap_serializer_function_ser_schema(_serialize_item, schema=handler.generate_schema(dtype))
)
definitions.append(
core_schema.union_schema(
[item_schema, core_schema.list_schema(serialization_schema), core_schema.any_schema()],
ref=item_ref,
)
)

def _validate(v):
subtypes = get_args(source_type)
dtype = subtypes[0] if subtypes and subtypes[0] != Any else None
try:
if dtype:
return np.asarray(v, dtype=dtype)
Expand All @@ -61,12 +83,14 @@ def _validate(v):
except TypeError:
raise ValueError("Unable to convert value to an array.")

return core_schema.no_info_before_validator_function(
schema = core_schema.no_info_before_validator_function(
_validate,
core_schema.any_schema(),
serialization=core_schema.wrap_serializer_function_ser_schema(
cls._serialize,
info_arg=False,
schema=serialization_schema,
return_schema=core_schema.list_schema(),
),
)
return core_schema.definitions_schema(schema, definitions) if definitions else schema
7 changes: 4 additions & 3 deletions ccflow/publishers/composite.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,9 @@ def _get_publishers(self, data):
if not publisher:
for try_publisher in self.default_publishers:
try:
try_publisher.data = value
publishers[field] = try_publisher.model_copy()
candidate = try_publisher.model_copy()
candidate.data = value
publishers[field] = candidate
publishers[field].name = full_name
publishers[field].name_params = self.name_params
break
Expand All @@ -76,9 +77,9 @@ def _get_publishers(self, data):
else:
# If value is the wrong type for the configured publisher, it will raise
# User should provide the right type of publisher for a given field.
publisher = publisher.model_copy()
publisher.data = value
if not publisher.name:
publisher = publisher.model_copy()
publisher.name = full_name
publisher.name_params = self.name_params
publishers[field] = publisher
Expand Down
47 changes: 47 additions & 0 deletions ccflow/tests/exttypes/test_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,53 @@ def test_record_batches(self):
m = Model(strict=[batch])
self.assertEqual(m.strict, pa.Table.from_pydict(data, RAW_SCHEMA))

def test_weak_preserves_extra_field_metadata(self):
extra_fields = [
pa.field("d", pa.int64(), metadata={b"unit": b"shares"}),
pa.field("e", pa.string(), metadata={b"source": b"example"}),
]
schema = pa.schema([*RAW_SCHEMA, *extra_fields])
data = {**self.data, "e": ["foo", None, "baz"]}
table = pa.Table.from_pydict(data, schema=schema)

result = Model(weak=table).weak

self.assertTrue(result.schema.equals(schema, check_metadata=True))
self.assertEqual(result.to_pydict(), table.to_pydict())
self.assertTrue(table.schema.equals(schema, check_metadata=True))

def test_weak_preserves_extra_field_nullability(self):
extra = pa.field("d", pa.int64(), nullable=False)
schema = pa.schema([*RAW_SCHEMA, extra])
table = pa.Table.from_pydict(self.data, schema=schema)

result = Model(weak=table).weak

self.assertTrue(result.schema.field("d").equals(extra, check_metadata=True))
self.assertEqual(result.column("d"), table.column("d"))

def test_weak_preserves_extra_fields_from_batches(self):
extra = pa.field("d", pa.int64(), nullable=False, metadata={b"unit": b"shares"})
source_schema = pa.schema([pa.field("a", pa.float64()), pa.field("b", pa.int64()), RAW_SCHEMA.field("c"), extra])
expected_schema = pa.schema([*RAW_SCHEMA, extra])
inputs = {
"empty": [pa.RecordBatch.from_pydict({key: [] for key in self.data}, schema=source_schema)],
"chunked": [
pa.RecordBatch.from_pydict({key: values[:1] for key, values in self.data.items()}, schema=source_schema),
pa.RecordBatch.from_pydict({key: values[1:] for key, values in self.data.items()}, schema=source_schema),
],
}
for name, batches in inputs.items():
with self.subTest(name=name):
table = pa.Table.from_batches(batches)
result = Model(weak=batches, filter=batches)

self.assertTrue(result.weak.schema.equals(expected_schema, check_metadata=True))
self.assertEqual(result.weak.column("d"), table.column("d"))
self.assertEqual(result.weak.column("d").num_chunks, table.column("d").num_chunks)
self.assertTrue(result.filter.schema.equals(RAW_SCHEMA, check_metadata=True))
self.assertRaises(ValidationError, Model, strict=batches)

def test_bad_type(self):
t = pa.Table.from_pydict(self.data)
data_bad = {"a": ["oops", 2, 3], "b": [4, 5, 6], "c": ["foo", "bar", "baz"]}
Expand Down
174 changes: 172 additions & 2 deletions ccflow/tests/exttypes/test_pydantic_numpy.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,16 @@
SOFTWARE.
"""

import json
from enum import IntEnum

import numpy as np
import pytest
from numpy.testing import assert_allclose
from pydantic import BaseModel
from pydantic import BaseModel, Field, TypeAdapter
from pydantic_core import PydanticSerializationError, core_schema

from ccflow import NDArray, float32
from ccflow import Enum, NDArray, float32


class MySettings(BaseModel):
Expand Down Expand Up @@ -128,3 +133,168 @@ def test_union_field():
assert 0 in model2.x
assert len(model2.x) == 1
np.testing.assert_array_equal(model2.x[0], np.array([0.0, 1.0, 2.0]))


class SerializedLabel:
def __init__(self, value):
self.value = value

@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
def serialize(value, info):
prefix = (getattr(info, "context", None) or {}).get("prefix", "")
return prefix + value.value

return core_schema.is_instance_schema(
cls,
serialization=core_schema.plain_serializer_function_ser_schema(serialize, info_arg=True, when_used="json"),
)


@pytest.mark.parametrize(
"shape,expected",
[((2,), ["BUY", "SELL"]), ((1, 2), [["BUY", "SELL"]]), ((2, 1, 1), [[["BUY"]], [["SELL"]]])],
)
def test_custom_item_serializer(shape, expected):
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python(np.array([SerializedLabel("BUY"), SerializedLabel("SELL")], dtype=object).reshape(shape))
original = array.copy()
assert json.loads(adapter.dump_json(array)) == expected
assert array.dtype == object
np.testing.assert_array_equal(array, original)


def test_custom_item_serializer_retains_python_mode():
value = SerializedLabel("BUY")
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python([value])
assert adapter.dump_python(array)[0] is value
assert adapter.dump_python(array, mode="json") == ["BUY"]


def test_custom_item_serializer_respects_index_filters():
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python([SerializedLabel("BUY"), SerializedLabel("SELL")])
assert json.loads(adapter.dump_json(array, include={1})) == ["SELL"]
assert json.loads(adapter.dump_json(array, exclude={0})) == ["SELL"]


@pytest.mark.parametrize("shape,expected", [((0,), []), ((2, 0), [[], []])])
def test_custom_item_serializer_empty_arrays(shape, expected):
adapter = TypeAdapter(NDArray[SerializedLabel])
assert json.loads(adapter.dump_json(np.empty(shape, dtype=object))) == expected


def test_custom_item_serializer_in_optional_field():
class ArrayModel(BaseModel):
values: NDArray[SerializedLabel] | None = None

assert json.loads(ArrayModel().model_dump_json()) == {"values": None}
assert json.loads(ArrayModel(values=[SerializedLabel("BUY")]).model_dump_json()) == {"values": ["BUY"]}


def test_custom_item_serializer_uses_context():
import inspect

if "context" not in inspect.signature(TypeAdapter.dump_json).parameters:
pytest.skip("serialization context is unavailable in this Pydantic version")
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python([SerializedLabel("BUY")])
assert json.loads(adapter.dump_json(array, context={"prefix": "trade:"})) == ["trade:BUY"]


def test_custom_item_serializer_preserves_model_options():
class Payload(BaseModel):
value: str = Field(alias="label")
optional: str | None = None

adapter = TypeAdapter(NDArray[Payload])
array = adapter.validate_python([Payload(label="BUY"), Payload(label="SELL")])
assert json.loads(adapter.dump_json(array, by_alias=True, exclude_none=True)) == [{"label": "BUY"}, {"label": "SELL"}]
assert adapter.dump_python(array, include={1: {"value"}}) == [{"value": "SELL"}]


def test_custom_item_serializer_honors_ccflow_enum_names():
class Side(Enum):
BUY = Enum.auto()
SELL = Enum.auto()

adapter = TypeAdapter(NDArray[Side])
array = adapter.validate_python(np.array([Side.BUY, Side.SELL], dtype=object))
assert json.loads(adapter.dump_json(array)) == ["BUY", "SELL"]


def test_untyped_object_array_does_not_guess_custom_schema():
with pytest.raises(PydanticSerializationError):
TypeAdapter(NDArray).dump_json(np.array([SerializedLabel("BUY")], dtype=object))


def test_native_numpy_dtype_serialization_unchanged():
adapter = TypeAdapter(NDArray[np.int64])
array = adapter.validate_python([[1, 2], [3, 4]])
assert json.loads(adapter.dump_json(array)) == [[1, 2], [3, 4]]


@pytest.mark.parametrize("serializer", [lambda value: value.name, lambda value: {0: "BUY", 1: "SELL"}[value]], ids=["attribute", "mapping"])
def test_numeric_array_does_not_apply_object_item_serializer(serializer):
class NumericSide(IntEnum):
BUY = 0
SELL = 1

@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
return core_schema.is_instance_schema(
cls,
serialization=core_schema.plain_serializer_function_ser_schema(serializer, when_used="json"),
)

adapter = TypeAdapter(NDArray[NumericSide])
array = adapter.validate_python(np.array([NumericSide.BUY, NumericSide.SELL]))
# NumPy has already coerced these values to integers, not enum instances.
assert all(not isinstance(value, NumericSide) for value in array.flat)
assert json.loads(adapter.dump_json(array)) == [0, 1]


def test_custom_item_serializer_keeps_list_subclass_items():
class ListLabel(list):
@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
return core_schema.is_instance_schema(
cls,
serialization=core_schema.plain_serializer_function_ser_schema(lambda value: "LABEL", when_used="json"),
)

array = np.empty(1, dtype=object)
array[0] = ListLabel([1, 2])
adapter = TypeAdapter(NDArray[ListLabel])
assert json.loads(adapter.dump_json(adapter.validate_python(array))) == ["LABEL"]


def test_custom_item_serializer_uses_model_config():
class BinaryLabel:
@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
return core_schema.is_instance_schema(
cls,
serialization=core_schema.plain_serializer_function_ser_schema(lambda value: b"\xff", when_used="json"),
)

class ArrayModel(BaseModel):
model_config = {"ser_json_bytes": "base64"}
values: NDArray[BinaryLabel]

assert json.loads(ArrayModel(values=[BinaryLabel()]).model_dump_json()) == {"values": ["_w=="]}


def test_custom_item_serializer_uses_fallback():
import inspect

if "fallback" not in inspect.signature(TypeAdapter.dump_json).parameters:
pytest.skip("serialization fallback is unavailable in this Pydantic version")

class Payload(BaseModel):
value: object

adapter = TypeAdapter(NDArray[Payload])
array = adapter.validate_python([Payload(value=object())])
assert json.loads(adapter.dump_json(array, fallback=lambda value: "opaque")) == [{"value": "opaque"}]
59 changes: 59 additions & 0 deletions ccflow/tests/publishers/test_composite.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,65 @@ def test_root_publisher(self):
}
self.assertDictEqual(paths, target)

def test_shared_field_and_default_publisher(self):
with tempfile.TemporaryDirectory() as tempdir:
root = Path(tempdir)
shared = GenericFilePublisher(name=str(root / "custom_a"), suffix=".txt")
publisher = CompositePublisher(
name=str(root / "batch"),
data={"a": "ALPHA", "b": "BETA"},
field_publishers={"a": shared},
default_publishers=[shared],
)

paths = publisher()

self.assertEqual(paths, {"a": root / "custom_a.txt", "b": root / "batch" / "b.txt"})
self.assertEqual({field: path.read_text() for field, path in paths.items()}, {"a": "ALPHA", "b": "BETA"})

def test_publisher_templates_are_not_modified(self):
for named in (False, True):
with self.subTest(named=named), tempfile.TemporaryDirectory() as tempdir:
root = Path(tempdir)
template = GenericFilePublisher(
name=str(root / "{{tag}}") if named else None,
name_params={"tag": "template"},
data="SEED",
suffix=".txt",
)
original_name = template.name
publisher = CompositePublisher(
name=str(root / "batch"),
name_params={"tag": "composite"},
data={"a": "ALPHA", "b": "BETA"},
field_publishers={"a": template},
default_publishers=[template],
)

paths = publisher()

self.assertEqual({field: path.read_text() for field, path in paths.items()}, {"a": "ALPHA", "b": "BETA"})
self.assertEqual(template.data, "SEED")
self.assertEqual(template.name, original_name)
self.assertEqual(template.name_params, {"tag": "template"})

def test_default_validation_fallback_preserves_templates(self):
with tempfile.TemporaryDirectory() as tempdir:
rejected = DummyPublisher(data=(42,))
accepted = GenericFilePublisher(data="SEED", suffix=".txt")
publisher = CompositePublisher(
name=str(Path(tempdir) / "batch"),
data={"a": "ALPHA", "b": "BETA"},
default_publishers=[rejected, accepted],
)

paths = publisher()

self.assertEqual({field: path.read_text() for field, path in paths.items()}, {"a": "ALPHA", "b": "BETA"})
self.assertEqual(rejected.data, (42,))
self.assertEqual(accepted.data, "SEED")
self.assertIsNone(accepted.name)

def test_skip_root_publisher(self):
with tempfile.TemporaryDirectory() as tempdir:
os.chdir(tempdir)
Expand Down
Loading