Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 30 additions & 4 deletions ccflow/exttypes/pydantic_numpy/ndarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"""

import sys
from functools import lru_cache
from typing import Any, Generic, TypeVar, get_args

import numpy as np
Expand All @@ -48,11 +49,36 @@ def _serialize(cls, v, nxt):

@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
from pydantic import TypeAdapter
from pydantic_core import core_schema

subtypes = get_args(source_type)
dtype = subtypes[0] if subtypes and subtypes[0] != Any else None

@lru_cache(maxsize=64)
def _serialization_adapter(ndim):
item_type = dtype
for _ in range(ndim):
item_type = list[item_type]
return TypeAdapter(item_type)

def _serialize(v, nxt, info):
if not isinstance(v, np.ndarray) or v.dtype != object or not hasattr(dtype, "__get_pydantic_core_schema__"):
return cls._serialize(v, nxt)
# NumPy can coerce IntEnum items to numbers even in object arrays.
if not isinstance(dtype, type) or any(not isinstance(item, dtype) for item in v.flat):
return cls._serialize(v, nxt)
options = {
name: getattr(info, name)
for name in ("mode", "include", "exclude", "by_alias", "exclude_unset", "exclude_defaults", "exclude_none", "round_trip")
}
# Newer Pydantic versions expose additional serialization controls.
for name in ("context", "serialize_as_any", "exclude_computed_fields", "polymorphic_serialization"):
if hasattr(info, name):
options[name] = getattr(info, name)
return _serialization_adapter(v.ndim).dump_python(v.tolist(), **options)

def _validate(v):
subtypes = get_args(source_type)
dtype = subtypes[0] if subtypes and subtypes[0] != Any else None
try:
if dtype:
return np.asarray(v, dtype=dtype)
Expand All @@ -65,8 +91,8 @@ def _validate(v):
_validate,
core_schema.any_schema(),
serialization=core_schema.wrap_serializer_function_ser_schema(
cls._serialize,
info_arg=False,
_serialize,
info_arg=True,
return_schema=core_schema.list_schema(),
),
)
128 changes: 126 additions & 2 deletions ccflow/tests/exttypes/test_pydantic_numpy.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,16 @@
SOFTWARE.
"""

import json
from enum import IntEnum

import numpy as np
import pytest
from numpy.testing import assert_allclose
from pydantic import BaseModel
from pydantic import BaseModel, Field, TypeAdapter
from pydantic_core import PydanticSerializationError, core_schema

from ccflow import NDArray, float32
from ccflow import Enum, NDArray, float32


class MySettings(BaseModel):
Expand Down Expand Up @@ -128,3 +133,122 @@ def test_union_field():
assert 0 in model2.x
assert len(model2.x) == 1
np.testing.assert_array_equal(model2.x[0], np.array([0.0, 1.0, 2.0]))


class SerializedLabel:
def __init__(self, value):
self.value = value

@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
def serialize(value, info):
prefix = (getattr(info, "context", None) or {}).get("prefix", "")
return prefix + value.value

return core_schema.is_instance_schema(
cls,
serialization=core_schema.plain_serializer_function_ser_schema(serialize, info_arg=True, when_used="json"),
)


@pytest.mark.parametrize(
"shape,expected",
[((2,), ["BUY", "SELL"]), ((1, 2), [["BUY", "SELL"]]), ((2, 1, 1), [[["BUY"]], [["SELL"]]])],
)
def test_custom_item_serializer(shape, expected):
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python(np.array([SerializedLabel("BUY"), SerializedLabel("SELL")], dtype=object).reshape(shape))
original = array.copy()
assert json.loads(adapter.dump_json(array)) == expected
assert array.dtype == object
np.testing.assert_array_equal(array, original)


def test_custom_item_serializer_retains_python_mode():
value = SerializedLabel("BUY")
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python([value])
assert adapter.dump_python(array)[0] is value
assert adapter.dump_python(array, mode="json") == ["BUY"]


def test_custom_item_serializer_respects_index_filters():
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python([SerializedLabel("BUY"), SerializedLabel("SELL")])
assert json.loads(adapter.dump_json(array, include={1})) == ["SELL"]
assert json.loads(adapter.dump_json(array, exclude={0})) == ["SELL"]


@pytest.mark.parametrize("shape,expected", [((0,), []), ((2, 0), [[], []])])
def test_custom_item_serializer_empty_arrays(shape, expected):
adapter = TypeAdapter(NDArray[SerializedLabel])
assert json.loads(adapter.dump_json(np.empty(shape, dtype=object))) == expected


def test_custom_item_serializer_in_optional_field():
class ArrayModel(BaseModel):
values: NDArray[SerializedLabel] | None = None

assert json.loads(ArrayModel().model_dump_json()) == {"values": None}
assert json.loads(ArrayModel(values=[SerializedLabel("BUY")]).model_dump_json()) == {"values": ["BUY"]}


def test_custom_item_serializer_uses_context():
import inspect

if "context" not in inspect.signature(TypeAdapter.dump_json).parameters:
pytest.skip("serialization context is unavailable in this Pydantic version")
adapter = TypeAdapter(NDArray[SerializedLabel])
array = adapter.validate_python([SerializedLabel("BUY")])
assert json.loads(adapter.dump_json(array, context={"prefix": "trade:"})) == ["trade:BUY"]


def test_custom_item_serializer_preserves_model_options():
class Payload(BaseModel):
value: str = Field(alias="label")
optional: str | None = None

adapter = TypeAdapter(NDArray[Payload])
array = adapter.validate_python([Payload(label="BUY"), Payload(label="SELL")])
assert json.loads(adapter.dump_json(array, by_alias=True, exclude_none=True)) == [{"label": "BUY"}, {"label": "SELL"}]
assert adapter.dump_python(array, include={1: {"value"}}) == [{"value": "SELL"}]


def test_custom_item_serializer_honors_ccflow_enum_names():
class Side(Enum):
BUY = Enum.auto()
SELL = Enum.auto()

adapter = TypeAdapter(NDArray[Side])
array = adapter.validate_python(np.array([Side.BUY, Side.SELL], dtype=object))
assert json.loads(adapter.dump_json(array)) == ["BUY", "SELL"]


def test_untyped_object_array_does_not_guess_custom_schema():
with pytest.raises(PydanticSerializationError):
TypeAdapter(NDArray).dump_json(np.array([SerializedLabel("BUY")], dtype=object))


def test_native_numpy_dtype_serialization_unchanged():
adapter = TypeAdapter(NDArray[np.int64])
array = adapter.validate_python([[1, 2], [3, 4]])
assert json.loads(adapter.dump_json(array)) == [[1, 2], [3, 4]]


def test_numeric_array_does_not_apply_object_item_serializer():
class NumericSide(IntEnum):
BUY = 0
SELL = 1

@classmethod
def __get_pydantic_core_schema__(cls, source_type, handler):
return core_schema.is_instance_schema(
cls,
serialization=core_schema.plain_serializer_function_ser_schema(lambda value: value.name, when_used="json"),
)

adapter = TypeAdapter(NDArray[NumericSide])
array = adapter.validate_python(np.array([NumericSide.BUY, NumericSide.SELL]))
# NumPy has already coerced these values to integers, not enum instances.
assert all(not isinstance(value, NumericSide) for value in array.flat)
assert json.loads(adapter.dump_json(array)) == [0, 1]
Loading