Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions ccflow/tests/utils/test_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,46 @@ def test_add_field_metadata(self):

# Get it back in "normal" form
self.assertDictEqual(get_field_metadata(t2), metadata)

def test_add_field_metadata_preserves_schema_metadata(self):
schema = pa.schema(
[
pa.field("price", pa.float64(), nullable=False, metadata={"description": '"closing price"'}),
pa.field("symbol", pa.string(), metadata={"description": '"ticker"'}),
],
metadata={b"dataset": b"prices", b"binary": b"\x00\xff"},
)
table = pa.Table.from_pydict({"price": [10.0, 20.0], "symbol": ["A", "B"]}, schema=schema)

for metadata in ({"price": {"unit": "USD"}}, {}, {"missing": {"unit": "USD"}}):
with self.subTest(metadata=metadata):
result = add_field_metadata(table, metadata)

self.assertEqual(result.schema.metadata, table.schema.metadata)
self.assertTrue(result.schema.field("symbol").equals(schema.field("symbol"), check_metadata=True))
self.assertFalse(result.schema.field("price").nullable)
self.assertEqual(result.to_pydict(), table.to_pydict())

def test_add_field_metadata_preserves_pandas_index(self):
indexes = [
pd.RangeIndex(10, 12, name="row"),
pd.Index(["trade_a", "trade_b"], name="trade"),
pd.MultiIndex.from_tuples([("A", 1), ("B", 2)], names=["symbol", "trade"]),
]
for index in indexes:
with self.subTest(index=index):
frame = pd.DataFrame({"price": [10.0, 20.0]}, index=index)
table = pa.Table.from_pandas(frame)

result = add_field_metadata(table, {"price": {"unit": "USD"}})

pd.testing.assert_frame_equal(result.to_pandas(), frame)

def test_add_field_metadata_still_replaces_selected_field_metadata(self):
schema = pa.schema([pa.field("price", pa.float64(), metadata={"description": '"closing price"'})])
table = pa.Table.from_pydict({"price": [10.0]}, schema=schema)

result = add_field_metadata(table, {"price": {"unit": "USD"}})

self.assertEqual(result.schema.field("price").metadata, {b"unit": b'"USD"'})
self.assertIsNone(result.schema.metadata)
4 changes: 2 additions & 2 deletions ccflow/utils/arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def convert_large_types(table: pa.Table) -> pa.Table:


def add_field_metadata(table: pa.Table, metadata: dict[str, Any]):
"""Helper function to add column-level meta data to an arrow table for multiple columns at once."""
"""Add column-level metadata to multiple columns while preserving table-level metadata."""
# There does not seem to be a pyarrow function to do this easily
new_schema = []
for field in table.schema:
Expand All @@ -74,7 +74,7 @@ def add_field_metadata(table: pa.Table, metadata: dict[str, Any]):
else:
new_field = field
new_schema.append(new_field)
return table.cast(pa.schema(new_schema))
return table.cast(pa.schema(new_schema, metadata=table.schema.metadata))


def get_field_metadata(table: pa.Table) -> dict[str, Any]:
Expand Down
Loading