Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ccflow/exttypes/arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def _validate(cls, v, schema, strict):
extra_cols = [c for c in v.schema.names if c not in schema.names]
v_checked = v.drop(extra_cols).cast(schema)
for extra in extra_cols:
v_checked = v_checked.append_column(extra, v[extra])
v_checked = v_checked.append_column(v.schema.field(extra), v[extra])
return v_checked
return v

Expand Down
47 changes: 47 additions & 0 deletions ccflow/tests/exttypes/test_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,53 @@ def test_record_batches(self):
m = Model(strict=[batch])
self.assertEqual(m.strict, pa.Table.from_pydict(data, RAW_SCHEMA))

def test_weak_preserves_extra_field_metadata(self):
extra_fields = [
pa.field("d", pa.int64(), metadata={b"unit": b"shares"}),
pa.field("e", pa.string(), metadata={b"source": b"example"}),
]
schema = pa.schema([*RAW_SCHEMA, *extra_fields])
data = {**self.data, "e": ["foo", None, "baz"]}
table = pa.Table.from_pydict(data, schema=schema)

result = Model(weak=table).weak

self.assertTrue(result.schema.equals(schema, check_metadata=True))
self.assertEqual(result.to_pydict(), table.to_pydict())
self.assertTrue(table.schema.equals(schema, check_metadata=True))

def test_weak_preserves_extra_field_nullability(self):
extra = pa.field("d", pa.int64(), nullable=False)
schema = pa.schema([*RAW_SCHEMA, extra])
table = pa.Table.from_pydict(self.data, schema=schema)

result = Model(weak=table).weak

self.assertTrue(result.schema.field("d").equals(extra, check_metadata=True))
self.assertEqual(result.column("d"), table.column("d"))

def test_weak_preserves_extra_fields_from_batches(self):
extra = pa.field("d", pa.int64(), nullable=False, metadata={b"unit": b"shares"})
source_schema = pa.schema([pa.field("a", pa.float64()), pa.field("b", pa.int64()), RAW_SCHEMA.field("c"), extra])
expected_schema = pa.schema([*RAW_SCHEMA, extra])
inputs = {
"empty": [pa.RecordBatch.from_pydict({key: [] for key in self.data}, schema=source_schema)],
"chunked": [
pa.RecordBatch.from_pydict({key: values[:1] for key, values in self.data.items()}, schema=source_schema),
pa.RecordBatch.from_pydict({key: values[1:] for key, values in self.data.items()}, schema=source_schema),
],
}
for name, batches in inputs.items():
with self.subTest(name=name):
table = pa.Table.from_batches(batches)
result = Model(weak=batches, filter=batches)

self.assertTrue(result.weak.schema.equals(expected_schema, check_metadata=True))
self.assertEqual(result.weak.column("d"), table.column("d"))
self.assertEqual(result.weak.column("d").num_chunks, table.column("d").num_chunks)
self.assertTrue(result.filter.schema.equals(RAW_SCHEMA, check_metadata=True))
self.assertRaises(ValidationError, Model, strict=batches)

def test_bad_type(self):
t = pa.Table.from_pydict(self.data)
data_bad = {"a": ["oops", 2, 3], "b": [4, 5, 6], "c": ["foo", "bar", "baz"]}
Expand Down
Loading