From 981820ad682d79f49f6aa9d8c42e6f39031c71af Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sat, 7 Nov 2026 23:08:13 +1100 Subject: [PATCH 01/11] fix(serializers): bound AutoSerializer set revival against hash floods The __set__ branch of _auto_object_hook built set()/frozenset() from an untrusted element list with no bound. Ints, floats, UUIDs and tuples hash without a per-process secret, so a forged entry could choose element hashes that make the build quadratic: by sharing one hash (each insert then calls __eq__ on every earlier element), or by distinct hashes whose probe sequences merge onto one cycle (no __eq__ at all). The reader now counts the elements that are neither str nor bytes and refuses more than 1024, and refuses a set where three non-int, non-float elements share a hash. The hash check sorts the hashes rather than tallying them in a table, which would itself be floodable. Ints and floats skip it: they compare in C and only a few dozen share a hash within msgpack's integer range. Both refusals come before set() builds anything. The writer refuses a set over the count, so it never emits one its reader refuses. Measured per byte at about 1 MB against an honest set of the same size: the costliest admitted forged set (1024 ints whose probe sequences merge) decodes in about 4x the time; same-hash UUID or long-prefix tuple pairs about 1x. Same-hash UUID sets of 1024, previously about 30x, and larger ones, previously quadratic, are refused. The guard adds about 1.3-1.5x to decoding a set. Also widen _keys_tested_by to any word-character dunder literal, so a marker key such as "__new_type__" added to a hook fails the Rust marker-list check. --- docs/error-codes.md | 2 +- docs/serializers/auto.md | 4 + src/cachekit/serializers/auto_serializer.py | 49 ++++++- tests/unit/test_auto_interop_decode.py | 2 +- tests/unit/test_auto_serializer_new_types.py | 128 ++++++++++++++++++- 5 files changed, 181 insertions(+), 4 deletions(-) diff --git a/docs/error-codes.md b/docs/error-codes.md index 6b1aefc5..ec55888f 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -493,7 +493,7 @@ export CACHEKIT_CONNECTION_POOL_SIZE=100 **Exception**: none raised to a `@cache`-decorated caller: the result is returned but not stored in the backend. The exception is `@cache(interop=...)`: there an unsupported return value raises `InteropError` (a `ValueError` subclass, `from cachekit.interop import InteropError`) by design, so a cross-SDK entry is never silently skipped. Calling the serializer directly raises `TypeError`, e.g. `StandardSerializer does not support custom classes (Python-specific types). ...` or `StandardSerializer does not support pandas DataFrames or Series (Python-specific types). ...` -**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. +**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes (`AutoSerializer stores at most 1024 elements that are neither str nor bytes in one set; ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). **When it occurs**: ```python diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index cc9d18f2..035afe4b 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -67,6 +67,10 @@ These markers are Python-specific — other language SDKs (Rust, TypeScript, PHP Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is installed but fails to import, the first DataFrame write or Arrow read on each `AutoSerializer` logs a warning on the `cachekit.serializers.auto_serializer` logger. New DataFrames are then written in the columnar format, and reading an Arrow entry raises `SerializationError`. +## Set Size Bound + +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and no three of its elements share a hash value. Writing a larger set raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). A stored set over either bound, which only a forged entry or another writer can hold, raises `SerializationError`, which a cached read treats as a miss. The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. Two elements can share a hash honestly (`hash((-1,)) == hash((-2,))`), but three in one set only by construction. Ints and floats are exempt from that check: they compare in C, and few share a hash within MessagePack's integer range. The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. Within the bound, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 4x the time of an honest set of the same size. To cache more elements, cache them as a list. + ## When to Use **Use `serializer='auto'` when:** diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 7e57a0fa..bf04aded 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -27,6 +27,7 @@ import importlib import logging +import operator import sys from datetime import date, datetime, time from functools import cached_property @@ -73,6 +74,18 @@ # never travel in an envelope. serialize() always writes the field, so an absent one is rot too. _HEADER_FORMATS = _ENVELOPE_FORMATS | {"numpy", "arrow"} +# Most elements one set or frozenset may hold that are neither str nor bytes. str and bytes hash under a +# per-process secret; an int, float, UUID or tuple hashes without one, so a forged entry can choose its +# elements' hashes and make set() quadratic in their count, in two ways. Sharing one hash makes each insert +# call __eq__ on every earlier element, and that __eq__ can be dear (a UUID's runs in Python; a tuple's +# compares a long shared prefix): _check_untrusted_set refuses three elements on one hash outright, so that +# costs at most one __eq__ per insert. Distinct hashes chosen so their probe sequences merge (CPython's +# perturbation is spent after 13 probes, and every sequence then walks the same cycle) make each insert walk +# one shared chain; that costs no __eq__ at all, so this count is what bounds it. At 1024, the costliest +# forged set found (such a chain of ints) decodes in about 4x the time of an honest one of the same size. +# _auto_default refuses to write a larger set, so the reader never refuses a set the writer emitted. +_MAX_NON_STRING_SET_ELEMENTS = 1024 + # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( "AutoSerializer does not support Pydantic models. Use .model_dump() to convert to dict: result = model.model_dump()" @@ -317,7 +330,16 @@ def _auto_default(obj: Any) -> Any: # NEW: set/frozenset support (type-safe roundtrip) if isinstance(obj, (set, frozenset)): - return {"__set__": True, "value": list(obj), "frozen": isinstance(obj, frozenset)} + value = list(obj) + # Counted as the reader will see them: a str or bytes subclass reads back as str or bytes. + exposed = sum(not isinstance(x, (str, bytes)) for x in value) + if exposed > _MAX_NON_STRING_SET_ELEMENTS: + raise TypeError( + f"AutoSerializer stores at most {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes " + f"in one {type(obj).__name__}; this one holds {exposed}. Its reader refuses more, as a hash-flood bound. " + "Cache the elements as a list instead." + ) + return {"__set__": True, "value": value, "frozen": isinstance(obj, frozenset)} # NumPy array support (nested in dicts/lists via msgpack custom encoder) np_loaded = sys.modules.get("numpy") @@ -340,6 +362,30 @@ def _auto_default(obj: Any) -> Any: raise TypeError(f"Object of type {type(obj).__name__} is not MessagePack serializable") +def _check_untrusted_set(values: list[Any]) -> None: + """Refuse a stored set's elements if set() over them could be quadratic (see _MAX_NON_STRING_SET_ELEMENTS).""" + # Counted by list.count, which runs in C: a Python loop over every element would slow honest sets several-fold. + types = list(map(type, values)) + exposed = len(types) - types.count(str) - types.count(bytes) + if exposed > _MAX_NON_STRING_SET_ELEMENTS: + raise SerializationError( + f"Invalid set format: more than {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes: " + "a hash-flood bound, not a framing error. The entry is likely forged." + ) + # An int or float compares in C, and within msgpack's int range at most a few dozen share a hash, so a shared + # hash among them costs little. Every other element is hashed and sorted, not tallied in a dict or set: a table + # keyed by the hashes is the very build being bounded. + if not exposed or exposed == types.count(int) + types.count(float): + return + hashes = sorted(hash(x) for x in values if type(x) not in (str, bytes, int, float)) + # Two elements of one set can share a hash honestly (hash((-1,)) == hash((-2,))); a third means a forged entry. + if any(map(operator.eq, hashes, hashes[2:])): + raise SerializationError( + "Invalid set format: three elements share one hash: a hash-flood bound, not a framing error. " + "The entry is likely forged." + ) + + def _auto_object_hook(obj: Any) -> Any: """Custom decoder for types encoded by _auto_default(). @@ -401,6 +447,7 @@ def _auto_object_hook(obj: Any) -> Any: value_list = obj["value"] if not isinstance(value_list, list): raise SerializationError(f"Invalid set format: expected list, got {type(value_list).__name__}") + _check_untrusted_set(value_list) if obj.get("frozen"): return frozenset(value_list) diff --git a/tests/unit/test_auto_interop_decode.py b/tests/unit/test_auto_interop_decode.py index 2e6ff5e3..2a5ff3a1 100644 --- a/tests/unit/test_auto_interop_decode.py +++ b/tests/unit/test_auto_interop_decode.py @@ -414,7 +414,7 @@ def test_buffer_inputs_decode_like_bytes() -> None: def _keys_tested_by(hook: Callable[..., Any]) -> set[str]: """The marker keys ``hook`` tests: the dunder string literals in its source.""" - return set(re.findall(r'"(__[a-z]+__)"', inspect.getsource(hook))) + return set(re.findall(r'"(__\w+__)"', inspect.getsource(hook))) def test_each_readers_rust_marker_keys_are_its_hooks() -> None: diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index a9a98e78..62c8a27d 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -12,6 +12,7 @@ from __future__ import annotations from collections.abc import Callable +from datetime import datetime, timedelta from uuid import UUID import msgpack @@ -20,7 +21,7 @@ from hypothesis import strategies as st from cachekit._rust_serializer import ByteStorage -from cachekit.serializers.auto_serializer import AutoSerializer +from cachekit.serializers.auto_serializer import _MAX_NON_STRING_SET_ELEMENTS, AutoSerializer from cachekit.serializers.base import SerializationError from tests.utils.hypothesis_settings import slow_first_draw @@ -206,6 +207,131 @@ def test_frozenset_type_preserved(self): assert deserialized == test_frozenset +def _uuid_marker(i: int) -> dict: + return {"__uuid__": True, "value": str(UUID(int=i))} + + +def _set_doc(values: list, frozen: bool) -> bytes: + return msgpack.packb({"__set__": True, "value": values, "frozen": frozen}, use_bin_type=True) + + +# UUID.__hash__ is hash(self.int), and an int hashes mod 2**61 - 1: these all hash alike. +_SAME_HASH = [12345 + k * (2**61 - 1) for k in range(40)] + + +class _Str(str): + pass + + +class TestAutoSerializerSetHashFloodBound: + """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, and no three + of them share a hash; either refusal comes before set() builds the set, so a forged one never pays for a quadratic + build. The writer refuses a set the reader would refuse by count.""" + + @pytest.fixture + def counts(self, monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: + counts = {"hash": 0, "eq": 0} + uuid_hash, uuid_eq = UUID.__hash__, UUID.__eq__ + + def counting_hash(self: UUID) -> int: + counts["hash"] += 1 + return uuid_hash(self) + + def counting_eq(self: UUID, other: object) -> bool: + counts["eq"] += 1 + return uuid_eq(self, other) + + monkeypatch.setattr(UUID, "__hash__", counting_hash) + monkeypatch.setattr(UUID, "__eq__", counting_eq) + return counts + + @pytest.mark.parametrize("frozen", [False, True]) + @pytest.mark.parametrize( + "element", + [_uuid_marker, lambda i: {"__tuple__": True, "value": [_uuid_marker(i), 0]}], + ids=["uuid", "tuple"], + ) + def test_a_same_hash_set_is_refused_before_any_comparison( + self, counts: dict[str, int], frozen: bool, element: Callable[[int], dict] + ) -> None: + data = _set_doc([element(i) for i in _SAME_HASH], frozen) + with pytest.raises(SerializationError, match="three elements share one hash: a hash-flood bound"): + AutoSerializer(enable_integrity_checking=False).deserialize(data) + # Hashed once each, by the sort; set() would have compared each to every earlier one. + assert counts == {"hash": len(_SAME_HASH), "eq": 0} + + def test_two_elements_may_share_a_hash_but_not_three(self) -> None: + serializer = AutoSerializer() + value = {(-1,), (-2,), UUID(int=_SAME_HASH[0]), UUID(int=_SAME_HASH[1]), *range(100)} + assert hash((-1,)) == hash((-2,)) == hash((-1 - (2**61 - 1),)) + assert serializer.deserialize(*serializer.serialize(value)) == value + for third in (UUID(int=_SAME_HASH[2]), (-1 - (2**61 - 1),)): + with pytest.raises(SerializationError, match="three elements share one hash"): + serializer.deserialize(*serializer.serialize(value | {third})) + + def test_ints_and_floats_on_one_hash_are_not_refused(self) -> None: + """They compare in C, and few share a hash within msgpack's range: every int there with hash 0, and 2.0**k + for the k whose float hash is 1.""" + p = 2**61 - 1 + value = {*(k * p for k in range(-4, 8)), *(2.0 ** (61 * k) for k in range(-16, 17))} + assert {hash(x) for x in value} == {0, 1} + value.add(UUID(int=0)) # so the hash check runs, over the UUID alone + serializer = AutoSerializer() + assert serializer.deserialize(*serializer.serialize(value)) == value + + @pytest.mark.parametrize("frozen", [False, True]) + def test_a_set_over_the_bound_is_refused_before_any_hash(self, counts: dict[str, int], frozen: bool) -> None: + data = _set_doc([_uuid_marker(i) for i in range(_MAX_NON_STRING_SET_ELEMENTS + 1)], frozen) + with pytest.raises(SerializationError, match="more than 1024 elements that are neither str nor bytes"): + AutoSerializer(enable_integrity_checking=False).deserialize(data) + assert counts == {"hash": 0, "eq": 0} + + def test_the_bound_counts_only_elements_that_are_neither_str_nor_bytes(self) -> None: + serializer = AutoSerializer(enable_integrity_checking=False) + strings = [*map(str, range(2000)), *(b"%d" % i for i in range(2000))] + assert serializer.deserialize(_set_doc([*strings, *range(1024)], False)) == {*strings, *range(1024)} + with pytest.raises(SerializationError, match="more than 1024 elements"): + serializer.deserialize(_set_doc([*strings, *range(1025)], False)) + + def test_a_set_at_the_bound_hashes_each_element_twice(self, counts: dict[str, int]) -> None: + value = frozenset(UUID(int=i) for i in range(_MAX_NON_STRING_SET_ELEMENTS)) + serializer = AutoSerializer() + data, metadata = serializer.serialize(value) + counts.update(hash=0, eq=0) + result = serializer.deserialize(data, metadata) + # Once by the sort, once by frozenset(); no element shares a hash, so nothing is compared. + assert counts == {"hash": 2 * len(value), "eq": 0} + assert result == value + + @pytest.mark.parametrize( + "value", + [ + set(range(_MAX_NON_STRING_SET_ELEMENTS)), + frozenset(range(-(2**63), -(2**63) + _MAX_NON_STRING_SET_ELEMENTS)), + {i / 3 for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {UUID(int=i * 2**64) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {(i, str(i)) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {frozenset({i, -i - 10}) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {datetime(2026, 1, 1) + timedelta(seconds=i) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, + {*range(_MAX_NON_STRING_SET_ELEMENTS - 3), True, None, 1.5}, + {*range(_MAX_NON_STRING_SET_ELEMENTS), *map(str, range(5000)), *(b"%d" % i for i in range(5000))}, + {*range(_MAX_NON_STRING_SET_ELEMENTS), *(_Str(i) for i in range(5000))}, + ], + ids=["ints", "int64 floor", "floats", "uuids", "tuples", "frozensets", "datetimes", "mixed", "strs", "str subclass"], + ) + @pytest.mark.parametrize("kind", [set, frozenset]) + def test_every_set_shape_at_the_bound_round_trips(self, value: set, kind: type) -> None: + serializer = AutoSerializer() + result = serializer.deserialize(*serializer.serialize({"v": kind(value)}))["v"] + assert type(result) is kind + assert result == value + + @pytest.mark.parametrize("kind", [set, frozenset]) + def test_the_writer_refuses_a_set_its_reader_would_refuse(self, kind: type) -> None: + with pytest.raises(TypeError, match=rf"at most 1024 .* in one {kind.__name__}; this one holds 1025\."): + AutoSerializer().serialize({"v": [kind(range(_MAX_NON_STRING_SET_ELEMENTS + 1))]}) + + class TestAutoSerializerComplexNesting: """Test nested complex objects with new types.""" From d9e5748fbf62dc793fefd3f767f9062ba8fabf9e Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sat, 7 Nov 2026 23:34:08 +1100 Subject: [PATCH 02/11] fix(serializers): count every element in the set same-hash bound, and apply it on write Review found two flaws in the same-hash check. Exempting ints and floats let a forged set carry about 200 distinct floats on one hash (doubling a float rotates its hash bits mod 2**61 - 1), and a UUID pair could share that hash. And refusing three elements on one hash refused honest sets: hash(-1) == hash(-2), so 2-D offset tuples over -1 and -2 put four tuples on one hash. The writer stored such a set and every read then missed. Now every element's hash is sorted, a set may hold at most 8 on one hash (3-D offset tuples fit), and the writer applies the same bounds, so a refused set raises TypeError at write rather than missing on every read. A set of ints alone skips the sort: at most 13 ints in msgpack's range share a hash, and they compare in C. Per byte at about 1 MB against an honest same-size set, the costliest admitted forged set is still 1024 ints on one merged probe chain, about 4x; groups of 8 same-hash UUIDs or long-prefix tuples about 1.1x; groups of 12 same-hash ints about 1.7x. --- docs/error-codes.md | 2 +- docs/serializers/auto.md | 2 +- src/cachekit/serializers/auto_serializer.py | 77 ++++++++++++-------- tests/unit/test_auto_serializer_new_types.py | 75 +++++++++++++------ 4 files changed, 100 insertions(+), 56 deletions(-) diff --git a/docs/error-codes.md b/docs/error-codes.md index ec55888f..a3303d94 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -493,7 +493,7 @@ export CACHEKIT_CONNECTION_POOL_SIZE=100 **Exception**: none raised to a `@cache`-decorated caller: the result is returned but not stored in the backend. The exception is `@cache(interop=...)`: there an unsupported return value raises `InteropError` (a `ValueError` subclass, `from cachekit.interop import InteropError`) by design, so a cross-SDK entry is never silently skipped. Calling the serializer directly raises `TypeError`, e.g. `StandardSerializer does not support custom classes (Python-specific types). ...` or `StandardSerializer does not support pandas DataFrames or Series (Python-specific types). ...` -**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes (`AutoSerializer stores at most 1024 elements that are neither str nor bytes in one set; ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). +**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value (`AutoSerializer cannot store this set: it holds more than ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). **When it occurs**: ```python diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index 035afe4b..18d0a7a3 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,7 +69,7 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and no three of its elements share a hash value. Writing a larger set raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). A stored set over either bound, which only a forged entry or another writer can hold, raises `SerializationError`, which a cached read treats as a miss. The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. Two elements can share a hash honestly (`hash((-1,)) == hash((-2,))`), but three in one set only by construction. Ints and floats are exempt from that check: they compare in C, and few share a hash within MessagePack's integer range. The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. Within the bound, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 4x the time of an honest set of the same size. To cache more elements, cache them as a list. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value. Writing a set over either bound raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 4x the time of an honest set of the same size. To cache more elements, cache them as a list. ## When to Use diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index bf04aded..38333a52 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -74,17 +74,19 @@ # never travel in an envelope. serialize() always writes the field, so an absent one is rot too. _HEADER_FORMATS = _ENVELOPE_FORMATS | {"numpy", "arrow"} -# Most elements one set or frozenset may hold that are neither str nor bytes. str and bytes hash under a -# per-process secret; an int, float, UUID or tuple hashes without one, so a forged entry can choose its -# elements' hashes and make set() quadratic in their count, in two ways. Sharing one hash makes each insert -# call __eq__ on every earlier element, and that __eq__ can be dear (a UUID's runs in Python; a tuple's -# compares a long shared prefix): _check_untrusted_set refuses three elements on one hash outright, so that -# costs at most one __eq__ per insert. Distinct hashes chosen so their probe sequences merge (CPython's -# perturbation is spent after 13 probes, and every sequence then walks the same cycle) make each insert walk -# one shared chain; that costs no __eq__ at all, so this count is what bounds it. At 1024, the costliest -# forged set found (such a chain of ints) decodes in about 4x the time of an honest one of the same size. -# _auto_default refuses to write a larger set, so the reader never refuses a set the writer emitted. +# Bounds on one set or frozenset, so a forged entry cannot make set() quadratic. str and bytes hash under a +# per-process secret; an int, float, UUID or tuple hashes without one, so a forged entry can choose its elements' +# hashes, in two ways (interop.py's _MAX_NON_STRING_KEYS explains both for dicts; sets probe the same way). +# Sharing one hash makes each insert call __eq__ on every earlier element of that hash, and that __eq__ can be +# dear (a UUID's runs in Python; a tuple's compares a long shared prefix): _set_bound_error refuses more than +# _MAX_SAME_HASH_SET_ELEMENTS on one hash, so an insert makes at most that many __eq__ calls. 8 admits honest +# collisions such as 3-D offset tuples over -1 and -2 (hash(-1) == hash(-2)). Distinct hashes whose probe +# sequences merge make each insert walk one shared chain with no __eq__ at all; the count of elements that are +# neither str nor bytes bounds that. At these bounds, the costliest forged set found (1024 ints on one merged +# chain) decodes in about 4x the time of an honest set of the same size. _auto_default applies the same bounds, +# so a refused set fails loudly at write instead of missing on every read. _MAX_NON_STRING_SET_ELEMENTS = 1024 +_MAX_SAME_HASH_SET_ELEMENTS = 8 # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( @@ -331,13 +333,16 @@ def _auto_default(obj: Any) -> Any: # NEW: set/frozenset support (type-safe roundtrip) if isinstance(obj, (set, frozenset)): value = list(obj) - # Counted as the reader will see them: a str or bytes subclass reads back as str or bytes. - exposed = sum(not isinstance(x, (str, bytes)) for x in value) + # Counted as the reader will see them (a str or bytes subclass reads back as str or bytes), but only when the + # count could matter: below it every element is taken as exposed, which at worst sorts a few str hashes too. + exposed = len(value) if exposed > _MAX_NON_STRING_SET_ELEMENTS: + exposed = sum(not isinstance(x, (str, bytes)) for x in value) + reason = _set_bound_error(value, exposed) + if reason: raise TypeError( - f"AutoSerializer stores at most {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes " - f"in one {type(obj).__name__}; this one holds {exposed}. Its reader refuses more, as a hash-flood bound. " - "Cache the elements as a list instead." + f"AutoSerializer cannot store this {type(obj).__name__}: it holds {reason}, which its reader refuses " + "as a hash-flood bound. Cache the elements as a list instead." ) return {"__set__": True, "value": value, "frozen": isinstance(obj, frozenset)} @@ -362,27 +367,35 @@ def _auto_default(obj: Any) -> Any: raise TypeError(f"Object of type {type(obj).__name__} is not MessagePack serializable") +def _set_bound_error(values: list[Any], exposed: int, *, ints_only: bool = False) -> str | None: + """Why set() over ``values`` could be quadratic, or None (see _MAX_NON_STRING_SET_ELEMENTS). + + ``exposed`` is how many of them are neither str nor bytes; ``ints_only`` says every one is an exact int. + """ + if exposed > _MAX_NON_STRING_SET_ELEMENTS: + return f"more than {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes" + # Fewer exposed elements cannot fill one hash past the bound: a str or bytes hash is secret. Nor can ints alone + # cost much on one hash: at most 13 in msgpack's int range share one (k * (2**61 - 1) for k in -4..8), and they + # compare in C. Sorting an honest int set's hashes would cost more than building it. + if ints_only or exposed <= _MAX_SAME_HASH_SET_ELEMENTS: + return None + # Sorted, not tallied in a dict or set: a table keyed by the hashes is the very build being bounded. + hashes = sorted(map(hash, values)) + if any(map(operator.eq, hashes, hashes[_MAX_SAME_HASH_SET_ELEMENTS:])): + return f"more than {_MAX_SAME_HASH_SET_ELEMENTS} elements that share one hash" + return None + + def _check_untrusted_set(values: list[Any]) -> None: - """Refuse a stored set's elements if set() over them could be quadratic (see _MAX_NON_STRING_SET_ELEMENTS).""" + """Refuse a stored set's elements if set() over them could be quadratic, before set() builds it.""" # Counted by list.count, which runs in C: a Python loop over every element would slow honest sets several-fold. types = list(map(type, values)) - exposed = len(types) - types.count(str) - types.count(bytes) - if exposed > _MAX_NON_STRING_SET_ELEMENTS: - raise SerializationError( - f"Invalid set format: more than {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes: " - "a hash-flood bound, not a framing error. The entry is likely forged." - ) - # An int or float compares in C, and within msgpack's int range at most a few dozen share a hash, so a shared - # hash among them costs little. Every other element is hashed and sorted, not tallied in a dict or set: a table - # keyed by the hashes is the very build being bounded. - if not exposed or exposed == types.count(int) + types.count(float): - return - hashes = sorted(hash(x) for x in values if type(x) not in (str, bytes, int, float)) - # Two elements of one set can share a hash honestly (hash((-1,)) == hash((-2,))); a third means a forged entry. - if any(map(operator.eq, hashes, hashes[2:])): + reason = _set_bound_error( + values, len(types) - types.count(str) - types.count(bytes), ints_only=types.count(int) == len(types) + ) + if reason: raise SerializationError( - "Invalid set format: three elements share one hash: a hash-flood bound, not a framing error. " - "The entry is likely forged." + f"Invalid set format: {reason}: a hash-flood bound, not a framing error. The entry is likely forged." ) diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index 62c8a27d..1727caca 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -11,6 +11,7 @@ from __future__ import annotations +import itertools from collections.abc import Callable from datetime import datetime, timedelta from uuid import UUID @@ -224,9 +225,9 @@ class _Str(str): class TestAutoSerializerSetHashFloodBound: - """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, and no three - of them share a hash; either refusal comes before set() builds the set, so a forged one never pays for a quadratic - build. The writer refuses a set the reader would refuse by count.""" + """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, and at most + _MAX_SAME_HASH_SET_ELEMENTS on one hash; either refusal comes before set() builds the set, so a forged one never + pays for a quadratic build. The writer refuses the same sets, so an honest one fails at write, not every read.""" @pytest.fixture def counts(self, monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: @@ -255,29 +256,42 @@ def test_a_same_hash_set_is_refused_before_any_comparison( self, counts: dict[str, int], frozen: bool, element: Callable[[int], dict] ) -> None: data = _set_doc([element(i) for i in _SAME_HASH], frozen) - with pytest.raises(SerializationError, match="three elements share one hash: a hash-flood bound"): + with pytest.raises(SerializationError, match="more than 8 elements that share one hash: a hash-flood bound"): AutoSerializer(enable_integrity_checking=False).deserialize(data) # Hashed once each, by the sort; set() would have compared each to every earlier one. assert counts == {"hash": len(_SAME_HASH), "eq": 0} - def test_two_elements_may_share_a_hash_but_not_three(self) -> None: - serializer = AutoSerializer() - value = {(-1,), (-2,), UUID(int=_SAME_HASH[0]), UUID(int=_SAME_HASH[1]), *range(100)} - assert hash((-1,)) == hash((-2,)) == hash((-1 - (2**61 - 1),)) - assert serializer.deserialize(*serializer.serialize(value)) == value - for third in (UUID(int=_SAME_HASH[2]), (-1 - (2**61 - 1),)): - with pytest.raises(SerializationError, match="three elements share one hash"): - serializer.deserialize(*serializer.serialize(value | {third})) + def test_eight_elements_may_share_a_hash_but_not_nine(self) -> None: + """hash(-1) == hash(-2), so 3-D offsets over -1 and -2 put 8 honest tuples on one hash; 4-D offsets put 16.""" + serializer = AutoSerializer() + offsets = set(itertools.product(range(-2, 3), repeat=3)) + assert len({hash(t) for t in itertools.product((-1, -2), repeat=3)}) == 1 + assert serializer.deserialize(*serializer.serialize(offsets)) == offsets + with pytest.raises(TypeError, match="it holds more than 8 elements that share one hash"): + serializer.serialize(set(itertools.product(range(-2, 3), repeat=4))) + with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): + serializer.deserialize(_set_doc([_uuid_marker(i) for i in _SAME_HASH[:9]], False), None) + assert serializer.deserialize(_set_doc([_uuid_marker(i) for i in _SAME_HASH[:8]], False), None) == { + UUID(int=i) for i in _SAME_HASH[:8] + } - def test_ints_and_floats_on_one_hash_are_not_refused(self) -> None: - """They compare in C, and few share a hash within msgpack's range: every int there with hash 0, and 2.0**k - for the k whose float hash is 1.""" + def test_ints_floats_and_other_types_on_one_hash_count_together(self) -> None: + """No type is exempt: 202 distinct floats can share one hash, and a UUID's Python __eq__ runs against each. + Only an all-int set skips the check: at most 13 ints in msgpack's range share a hash, and they compare in C.""" p = 2**61 - 1 - value = {*(k * p for k in range(-4, 8)), *(2.0 ** (61 * k) for k in range(-16, 17))} - assert {hash(x) for x in value} == {0, 1} - value.add(UUID(int=0)) # so the hash check runs, over the UUID alone - serializer = AutoSerializer() - assert serializer.deserialize(*serializer.serialize(value)) == value + serializer = AutoSerializer(enable_integrity_checking=False) + all_int = [k * p for k in range(-4, 9)] + assert {hash(k) for k in all_int} == {0} + assert serializer.deserialize(_set_doc(all_int, False)) == set(all_int) + for mixed in ( + [*(2.0 ** (61 * k) for k in range(9))], # hash 1 + [*(k * p for k in range(-4, 5)), 0.5], # hash 0, and a float, so not an all-int set + [*(2.0 ** (61 * k) for k in range(5)), *(_uuid_marker(1 + j * p) for j in range(4))], # hash 1 + ): + with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): + serializer.deserialize(_set_doc([*mixed, *range(10**6, 10**6 + 100)], False)) + with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): + serializer.deserialize(_set_doc([*mixed, *range(10**6, 10**6 + 100)], True)) @pytest.mark.parametrize("frozen", [False, True]) def test_a_set_over_the_bound_is_refused_before_any_hash(self, counts: dict[str, int], frozen: bool) -> None: @@ -316,8 +330,21 @@ def test_a_set_at_the_bound_hashes_each_element_twice(self, counts: dict[str, in {*range(_MAX_NON_STRING_SET_ELEMENTS - 3), True, None, 1.5}, {*range(_MAX_NON_STRING_SET_ELEMENTS), *map(str, range(5000)), *(b"%d" % i for i in range(5000))}, {*range(_MAX_NON_STRING_SET_ELEMENTS), *(_Str(i) for i in range(5000))}, + set(itertools.product(range(-2, 3), repeat=3)), + ], + ids=[ + "ints", + "int64 floor", + "floats", + "uuids", + "tuples", + "frozensets", + "datetimes", + "mixed", + "strs", + "str subclass", + "offset tuples", ], - ids=["ints", "int64 floor", "floats", "uuids", "tuples", "frozensets", "datetimes", "mixed", "strs", "str subclass"], ) @pytest.mark.parametrize("kind", [set, frozenset]) def test_every_set_shape_at_the_bound_round_trips(self, value: set, kind: type) -> None: @@ -328,8 +355,12 @@ def test_every_set_shape_at_the_bound_round_trips(self, value: set, kind: type) @pytest.mark.parametrize("kind", [set, frozenset]) def test_the_writer_refuses_a_set_its_reader_would_refuse(self, kind: type) -> None: - with pytest.raises(TypeError, match=rf"at most 1024 .* in one {kind.__name__}; this one holds 1025\."): + with pytest.raises(TypeError, match=rf"cannot store this {kind.__name__}: it holds more than 1024 elements"): AutoSerializer().serialize({"v": [kind(range(_MAX_NON_STRING_SET_ELEMENTS + 1))]}) + # str and bytes subclasses read back as str and bytes, so the writer does not count them either. + value = kind({*range(_MAX_NON_STRING_SET_ELEMENTS), *(_Str(i) for i in range(5000))}) + serializer = AutoSerializer() + assert serializer.deserialize(*serializer.serialize(value)) == value class TestAutoSerializerComplexNesting: From 6fdeeb286155007e7b09add7524d1b26a4ab8a8a Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sat, 7 Nov 2026 23:34:40 +1100 Subject: [PATCH 03/11] docs(serializers): split the set size bound section into paragraphs --- docs/serializers/auto.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index 18d0a7a3..5c475b7b 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,7 +69,13 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value. Writing a set over either bound raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 4x the time of an honest set of the same size. To cache more elements, cache them as a list. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value. Writing a set over either bound raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. + +The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. + +Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. + +The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 4x the time of an honest set of the same size. To cache more elements, cache them as a list. ## When to Use From a6216b97a2deb44a73eb7fa9202d89dcf9f0f39d Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 00:18:43 +1100 Subject: [PATCH 04/11] fix(serializers): refuse same-hash frozensets in a set, and sort only exposed hashes Review found that capping each hash at 8 elements bounds the number of __eq__ calls but not their cost. Two frozensets compare by looking up each element of one in the other, and each lookup can compare same-hash frozensets one level down, so forged nested frozensets on one hash cost more with every level of nesting, and grew super-linearly with size. Wrapping each frozenset in a tuple did the same. Now no two frozensets in a set may share a hash, nor may two tuples once any tuple in the set holds a tuple or frozenset. Both checks run in C passes over the elements' types. Flat tuples keep the limit of 8, so offset tuples over -1 and -2 still round-trip. Also from review: the same-hash sort now skips str and bytes, whose hashes are secret (a large string set beside nine ints decoded up to 3x slower), the writer takes the reader's all-int shortcut, and the tests cover a real bool and drop a duplicate case. The costliest admitted forged set is still 1024 ints on one merged probe chain, about 4-5x an honest same-size set per byte. --- docs/error-codes.md | 2 +- docs/serializers/auto.md | 6 +- src/cachekit/serializers/auto_serializer.py | 61 +++++++++++++------- tests/unit/test_auto_serializer_new_types.py | 35 ++++++++--- 4 files changed, 71 insertions(+), 33 deletions(-) diff --git a/docs/error-codes.md b/docs/error-codes.md index a3303d94..3e11cce1 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -493,7 +493,7 @@ export CACHEKIT_CONNECTION_POOL_SIZE=100 **Exception**: none raised to a `@cache`-decorated caller: the result is returned but not stored in the backend. The exception is `@cache(interop=...)`: there an unsupported return value raises `InteropError` (a `ValueError` subclass, `from cachekit.interop import InteropError`) by design, so a cross-SDK entry is never silently skipped. Calling the serializer directly raises `TypeError`, e.g. `StandardSerializer does not support custom classes (Python-specific types). ...` or `StandardSerializer does not support pandas DataFrames or Series (Python-specific types). ...` -**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value (`AutoSerializer cannot store this set: it holds more than ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). +**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value, or two frozensets that share one (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). **When it occurs**: ```python diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index 5c475b7b..81ac1db1 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,13 +69,13 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value. Writing a set over either bound raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value. No two of its frozensets may share a hash, and no two of its tuples may once any of its tuples holds a tuple or frozenset. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. -Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. +Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance: two frozensets compare by looking up each element of one in the other, and each lookup can compare frozensets one level down, so the cost of nested frozensets on one hash multiplies with every level. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1,), (-2,), ((0,),)}`, where a tuple holds a tuple. -The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 4x the time of an honest set of the same size. To cache more elements, cache them as a list. +The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 5x the time of an honest set of the same size. To cache more elements, cache them as a list. ## When to Use diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 38333a52..1d890607 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -32,6 +32,7 @@ from datetime import date, datetime, time from functools import cached_property from importlib.util import find_spec +from itertools import chain, compress, repeat from typing import TYPE_CHECKING, Any, ClassVar, Optional, cast from uuid import UUID @@ -77,16 +78,22 @@ # Bounds on one set or frozenset, so a forged entry cannot make set() quadratic. str and bytes hash under a # per-process secret; an int, float, UUID or tuple hashes without one, so a forged entry can choose its elements' # hashes, in two ways (interop.py's _MAX_NON_STRING_KEYS explains both for dicts; sets probe the same way). -# Sharing one hash makes each insert call __eq__ on every earlier element of that hash, and that __eq__ can be -# dear (a UUID's runs in Python; a tuple's compares a long shared prefix): _set_bound_error refuses more than -# _MAX_SAME_HASH_SET_ELEMENTS on one hash, so an insert makes at most that many __eq__ calls. 8 admits honest -# collisions such as 3-D offset tuples over -1 and -2 (hash(-1) == hash(-2)). Distinct hashes whose probe -# sequences merge make each insert walk one shared chain with no __eq__ at all; the count of elements that are -# neither str nor bytes bounds that. At these bounds, the costliest forged set found (1024 ints on one merged -# chain) decodes in about 4x the time of an honest set of the same size. _auto_default applies the same bounds, -# so a refused set fails loudly at write instead of missing on every read. +# Distinct hashes whose probe sequences merge make each insert walk one shared chain with no __eq__ at all; the +# count of elements that are neither str nor bytes bounds that. Sharing one hash makes each insert call __eq__ on +# every earlier element of that hash: _set_bound_error refuses more than _MAX_SAME_HASH_SET_ELEMENTS elements on +# one hash, so an insert makes fewer __eq__ calls than that (an all-int set is exempt: at most 13 ints share a hash +# and they compare in C). 8 admits honest collisions such as 3-D offset tuples over -1 and -2 (hash(-1) == +# hash(-2)). That bounds the number of __eq__ calls but not their cost: two frozensets compare by looking up each +# element of one in the other, and each lookup can compare same-hash frozensets one level down, so the cost +# multiplies with every level of nesting. Two frozensets may therefore never share a hash, nor may two tuples once +# any tuple in the set holds a tuple or frozenset (the path to a nested one); a flat tuple compares in time linear +# in its length. At these bounds, the costliest forged set found (1024 ints on one merged chain) decodes in about 5x +# the time of an honest set of the same size. _auto_default applies the same bounds to what it writes, so a refused +# set fails loudly at write instead of missing on every read. _MAX_NON_STRING_SET_ELEMENTS = 1024 _MAX_SAME_HASH_SET_ELEMENTS = 8 +_STR_TYPES = frozenset({str, bytes}) +_CONTAINERS = (tuple, frozenset) # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( @@ -333,12 +340,13 @@ def _auto_default(obj: Any) -> Any: # NEW: set/frozenset support (type-safe roundtrip) if isinstance(obj, (set, frozenset)): value = list(obj) + types = list(map(type, value)) # Counted as the reader will see them (a str or bytes subclass reads back as str or bytes), but only when the - # count could matter: below it every element is taken as exposed, which at worst sorts a few str hashes too. - exposed = len(value) + # count could matter: below it a str subclass counts as exposed, which can only make the writer stricter. + exposed = len(types) - types.count(str) - types.count(bytes) if exposed > _MAX_NON_STRING_SET_ELEMENTS: exposed = sum(not isinstance(x, (str, bytes)) for x in value) - reason = _set_bound_error(value, exposed) + reason = _set_bound_error(value, types, exposed) if reason: raise TypeError( f"AutoSerializer cannot store this {type(obj).__name__}: it holds {reason}, which its reader refuses " @@ -367,20 +375,31 @@ def _auto_default(obj: Any) -> Any: raise TypeError(f"Object of type {type(obj).__name__} is not MessagePack serializable") -def _set_bound_error(values: list[Any], exposed: int, *, ints_only: bool = False) -> str | None: +def _set_bound_error(values: list[Any], types: list[type], exposed: int) -> str | None: """Why set() over ``values`` could be quadratic, or None (see _MAX_NON_STRING_SET_ELEMENTS). - ``exposed`` is how many of them are neither str nor bytes; ``ints_only`` says every one is an exact int. + ``types`` holds each value's type; ``exposed`` is how many values are neither str nor bytes. """ if exposed > _MAX_NON_STRING_SET_ELEMENTS: return f"more than {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes" - # Fewer exposed elements cannot fill one hash past the bound: a str or bytes hash is secret. Nor can ints alone - # cost much on one hash: at most 13 in msgpack's int range share one (k * (2**61 - 1) for k in -4..8), and they - # compare in C. Sorting an honest int set's hashes would cost more than building it. - if ints_only or exposed <= _MAX_SAME_HASH_SET_ELEMENTS: + # At most 13 ints in msgpack's int range share a hash (k * (2**61 - 1) for k in -4..8), and they compare in C: + # an all-int set needs no hash check, and sorting an honest one's hashes would cost more than building it. + if types.count(int) == len(types): return None - # Sorted, not tallied in a dict or set: a table keyed by the hashes is the very build being bounded. - hashes = sorted(map(hash, values)) + # Sorted, not tallied in a dict or set: a table keyed by the hashes is the very build being bounded. Distinct + # types are few, and a type hashes by identity, so set(types) is safe. + if any(issubclass(t, _CONTAINERS) for t in set(types)): + held = set(map(type, chain.from_iterable(compress(values, map(issubclass, types, repeat(tuple)))))) + # A tuple holding a tuple or frozenset is a path to a nested frozenset, so then every tuple counts too. + nesting = _CONTAINERS if any(issubclass(t, _CONTAINERS) for t in held) else frozenset + nested = sorted(map(hash, compress(values, map(issubclass, types, repeat(nesting))))) + if any(map(operator.eq, nested, nested[1:])): + return "two frozensets, or two tuples beside a nested one, that share one hash" + # Fewer exposed elements cannot fill one hash past the bound: a str or bytes hash is secret, so only the + # others' hashes are sorted. + if exposed <= _MAX_SAME_HASH_SET_ELEMENTS: + return None + hashes = sorted(map(hash, compress(values, map(operator.not_, map(_STR_TYPES.__contains__, types))))) if any(map(operator.eq, hashes, hashes[_MAX_SAME_HASH_SET_ELEMENTS:])): return f"more than {_MAX_SAME_HASH_SET_ELEMENTS} elements that share one hash" return None @@ -390,9 +409,7 @@ def _check_untrusted_set(values: list[Any]) -> None: """Refuse a stored set's elements if set() over them could be quadratic, before set() builds it.""" # Counted by list.count, which runs in C: a Python loop over every element would slow honest sets several-fold. types = list(map(type, values)) - reason = _set_bound_error( - values, len(types) - types.count(str) - types.count(bytes), ints_only=types.count(int) == len(types) - ) + reason = _set_bound_error(values, types, len(types) - types.count(str) - types.count(bytes)) if reason: raise SerializationError( f"Invalid set format: {reason}: a hash-flood bound, not a framing error. The entry is likely forged." diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index 1727caca..eedba734 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -288,10 +288,31 @@ def test_ints_floats_and_other_types_on_one_hash_count_together(self) -> None: [*(k * p for k in range(-4, 5)), 0.5], # hash 0, and a float, so not an all-int set [*(2.0 ** (61 * k) for k in range(5)), *(_uuid_marker(1 + j * p) for j in range(4))], # hash 1 ): - with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): - serializer.deserialize(_set_doc([*mixed, *range(10**6, 10**6 + 100)], False)) - with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): - serializer.deserialize(_set_doc([*mixed, *range(10**6, 10**6 + 100)], True)) + for frozen in (False, True): + with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): + serializer.deserialize(_set_doc(mixed, frozen)) + + @pytest.mark.parametrize("frozen", [False, True]) + @pytest.mark.parametrize("wrap", [lambda m: m, lambda m: {"__tuple__": True, "value": [m]}], ids=["bare", "in a tuple"]) + def test_frozensets_on_one_hash_are_refused_before_any_comparison( + self, counts: dict[str, int], frozen: bool, wrap: Callable[[dict], dict] + ) -> None: + """Two frozensets compare by looking each element of one up in the other, and each lookup can compare + same-hash frozensets one level down, so nested same-hash frozensets cost more with every level. Even two on + one hash are refused, bare or in a tuple, before set() compares them.""" + inner = [{"__set__": True, "value": [_uuid_marker(i)], "frozen": True} for i in _SAME_HASH[:2]] + with pytest.raises(SerializationError, match="two frozensets, or two tuples beside a nested one, that share"): + AutoSerializer(enable_integrity_checking=False).deserialize(_set_doc([wrap(m) for m in inner], frozen)) + assert counts["eq"] == 0 + + def test_the_writer_refuses_frozensets_on_one_hash(self) -> None: + assert hash(frozenset({-1})) == hash(frozenset({-2})) + flat_pair = {(-1,), (-2,)} # flat tuples may share a hash, until one tuple in the set nests + assert AutoSerializer().deserialize(*AutoSerializer().serialize(flat_pair)) == flat_pair + nested = ({frozenset({-1}), frozenset({-2})}, {(frozenset({-1}),), (frozenset({-2}),)}, {((-1,),), ((-2,),)}) + for value in (*nested, flat_pair | {((0,),)}): + with pytest.raises(TypeError, match="two frozensets, or two tuples beside a nested one, that share"): + AutoSerializer().serialize(value) @pytest.mark.parametrize("frozen", [False, True]) def test_a_set_over_the_bound_is_refused_before_any_hash(self, counts: dict[str, int], frozen: bool) -> None: @@ -327,10 +348,10 @@ def test_a_set_at_the_bound_hashes_each_element_twice(self, counts: dict[str, in {(i, str(i)) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, {frozenset({i, -i - 10}) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, {datetime(2026, 1, 1) + timedelta(seconds=i) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, - {*range(_MAX_NON_STRING_SET_ELEMENTS - 3), True, None, 1.5}, + {*range(2, _MAX_NON_STRING_SET_ELEMENTS - 1), True, None, 1.5}, # True == 1, so no 1 here {*range(_MAX_NON_STRING_SET_ELEMENTS), *map(str, range(5000)), *(b"%d" % i for i in range(5000))}, {*range(_MAX_NON_STRING_SET_ELEMENTS), *(_Str(i) for i in range(5000))}, - set(itertools.product(range(-2, 3), repeat=3)), + {((i, -i), (frozenset({i}),), str(i)) for i in range(_MAX_NON_STRING_SET_ELEMENTS)}, ], ids=[ "ints", @@ -343,7 +364,7 @@ def test_a_set_at_the_bound_hashes_each_element_twice(self, counts: dict[str, in "mixed", "strs", "str subclass", - "offset tuples", + "nested tuples", ], ) @pytest.mark.parametrize("kind", [set, frozenset]) From 4fe0f43fb3f1592b1a05d5075483e1398f2ee1f5 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 00:52:01 +1100 Subject: [PATCH 05/11] fix(serializers): hold tuples to the frozenset rule only when one holds a frozenset Review found that refusing same-hash tuples whenever a tuple in the set held any tuple refused common honest data: grid edges stored both ways, such as ((0, -1), (0, -2)) and ((0, -2), (0, -1)), share a hash because hash(-1) == hash(-2). Tuples that hold only tuples and scalars compare in time linear in their size, so they keep the limit of 8 per hash. Only a frozenset, at any depth inside a tuple, makes a comparison recurse into set lookups; the check now walks the tuples a level at a time in C passes to find one. Also from review: the read error no longer calls every refused entry forged (an earlier cachekit version may have written it), the docs name the int exemption and the lossy nanosecond round trip, and the tests split and rename to say what they check. --- docs/error-codes.md | 2 +- docs/serializers/auto.md | 4 +- src/cachekit/serializers/auto_serializer.py | 50 +++++++++++++++----- tests/unit/test_auto_serializer_new_types.py | 36 +++++++++----- 4 files changed, 63 insertions(+), 29 deletions(-) diff --git a/docs/error-codes.md b/docs/error-codes.md index 3e11cce1..5447b144 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -493,7 +493,7 @@ export CACHEKIT_CONNECTION_POOL_SIZE=100 **Exception**: none raised to a `@cache`-decorated caller: the result is returned but not stored in the backend. The exception is `@cache(interop=...)`: there an unsupported return value raises `InteropError` (a `ValueError` subclass, `from cachekit.interop import InteropError`) by design, so a cross-SDK entry is never silently skipped. Calling the serializer directly raises `TypeError`, e.g. `StandardSerializer does not support custom classes (Python-specific types). ...` or `StandardSerializer does not support pandas DataFrames or Series (Python-specific types). ...` -**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value, or two frozensets that share one (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). +**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value, or two frozensets, or two tuples holding frozensets, that share one (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). **When it occurs**: ```python diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index 81ac1db1..e948d94c 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,11 +69,11 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value. No two of its frozensets may share a hash, and no two of its tuples may once any of its tuples holds a tuple or frozenset. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value (a set of ints alone excepted, see below). No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, or a set whose elements read back equal (a pandas `Timestamp` loses its nanoseconds), raises `SerializationError`, which a cached read treats as a miss. The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. -Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance: two frozensets compare by looking up each element of one in the other, and each lookup can compare frozensets one level down, so the cost of nested frozensets on one hash multiplies with every level. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1,), (-2,), ((0,),)}`, where a tuple holds a tuple. +Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples holding frozensets on one hash. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 5x the time of an honest set of the same size. To cache more elements, cache them as a list. diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 1d890607..a5db09dc 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -86,14 +86,13 @@ # hash(-2)). That bounds the number of __eq__ calls but not their cost: two frozensets compare by looking up each # element of one in the other, and each lookup can compare same-hash frozensets one level down, so the cost # multiplies with every level of nesting. Two frozensets may therefore never share a hash, nor may two tuples once -# any tuple in the set holds a tuple or frozenset (the path to a nested one); a flat tuple compares in time linear -# in its length. At these bounds, the costliest forged set found (1024 ints on one merged chain) decodes in about 5x +# a frozenset sits anywhere inside a tuple of the set; a tuple holding no frozenset compares in time linear in its +# size. At these bounds, the costliest forged set found (1024 ints on one merged chain) decodes in about 5x # the time of an honest set of the same size. _auto_default applies the same bounds to what it writes, so a refused # set fails loudly at write instead of missing on every read. _MAX_NON_STRING_SET_ELEMENTS = 1024 _MAX_SAME_HASH_SET_ELEMENTS = 8 _STR_TYPES = frozenset({str, bytes}) -_CONTAINERS = (tuple, frozenset) # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( @@ -375,6 +374,36 @@ def _auto_default(obj: Any) -> Any: raise TypeError(f"Object of type {type(obj).__name__} is not MessagePack serializable") +def _frozenset_in_tuples(values: list[Any], types: list[type]) -> bool: + """Whether a frozenset sits at any depth inside a tuple among ``values``. + + Walked a level at a time in C passes: linear in the tuples' size, and no recursion to overflow. + """ + while True: + values = list(chain.from_iterable(compress(values, map(issubclass, types, repeat(tuple))))) + if not values: + return False + types = list(map(type, values)) + if any(issubclass(t, frozenset) for t in set(types)): + return True + + +def _frozenset_comparison(values: list[Any], types: list[type]) -> bool: + """Whether set() over ``values`` could compare two frozensets: two of them, or two tuples holding them, on one hash. + + Distinct types are few, and a type hashes by identity, so set(types) is safe. + """ + kinds = set(types) + checked: list[type] = [frozenset] if any(issubclass(t, frozenset) for t in kinds) else [] + if any(issubclass(t, tuple) for t in kinds) and _frozenset_in_tuples(values, types): + checked.append(tuple) + for kind in checked: + hashes = sorted(map(hash, compress(values, map(issubclass, types, repeat(kind))))) + if any(map(operator.eq, hashes, hashes[1:])): + return True + return False + + def _set_bound_error(values: list[Any], types: list[type], exposed: int) -> str | None: """Why set() over ``values`` could be quadratic, or None (see _MAX_NON_STRING_SET_ELEMENTS). @@ -386,15 +415,9 @@ def _set_bound_error(values: list[Any], types: list[type], exposed: int) -> str # an all-int set needs no hash check, and sorting an honest one's hashes would cost more than building it. if types.count(int) == len(types): return None - # Sorted, not tallied in a dict or set: a table keyed by the hashes is the very build being bounded. Distinct - # types are few, and a type hashes by identity, so set(types) is safe. - if any(issubclass(t, _CONTAINERS) for t in set(types)): - held = set(map(type, chain.from_iterable(compress(values, map(issubclass, types, repeat(tuple)))))) - # A tuple holding a tuple or frozenset is a path to a nested frozenset, so then every tuple counts too. - nesting = _CONTAINERS if any(issubclass(t, _CONTAINERS) for t in held) else frozenset - nested = sorted(map(hash, compress(values, map(issubclass, types, repeat(nesting))))) - if any(map(operator.eq, nested, nested[1:])): - return "two frozensets, or two tuples beside a nested one, that share one hash" + # Hashes are sorted, never tallied in a dict or set: a table keyed by them is the very build being bounded. + if _frozenset_comparison(values, types): + return "two frozensets, or two tuples holding frozensets, that share one hash" # Fewer exposed elements cannot fill one hash past the bound: a str or bytes hash is secret, so only the # others' hashes are sorted. if exposed <= _MAX_SAME_HASH_SET_ELEMENTS: @@ -412,7 +435,8 @@ def _check_untrusted_set(values: list[Any]) -> None: reason = _set_bound_error(values, types, len(types) - types.count(str) - types.count(bytes)) if reason: raise SerializationError( - f"Invalid set format: {reason}: a hash-flood bound, not a framing error. The entry is likely forged." + f"Invalid set format: {reason}: a hash-flood bound, not a framing error. Either the entry is forged, " + "or a cachekit version without this bound wrote it." ) diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index eedba734..f1f81b95 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -225,9 +225,10 @@ class _Str(str): class TestAutoSerializerSetHashFloodBound: - """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, and at most - _MAX_SAME_HASH_SET_ELEMENTS on one hash; either refusal comes before set() builds the set, so a forged one never - pays for a quadratic build. The writer refuses the same sets, so an honest one fails at write, not every read.""" + """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, at most + _MAX_SAME_HASH_SET_ELEMENTS on one hash, and no two frozensets (or tuples holding them) on one hash. Each refusal + comes before set() builds the set, so a forged one never pays for a quadratic build. The writer refuses the same + sets, so an honest one fails at write, not on every read.""" @pytest.fixture def counts(self, monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: @@ -301,18 +302,24 @@ def test_frozensets_on_one_hash_are_refused_before_any_comparison( same-hash frozensets one level down, so nested same-hash frozensets cost more with every level. Even two on one hash are refused, bare or in a tuple, before set() compares them.""" inner = [{"__set__": True, "value": [_uuid_marker(i)], "frozen": True} for i in _SAME_HASH[:2]] - with pytest.raises(SerializationError, match="two frozensets, or two tuples beside a nested one, that share"): + with pytest.raises(SerializationError, match="two frozensets, or two tuples holding frozensets, that share"): AutoSerializer(enable_integrity_checking=False).deserialize(_set_doc([wrap(m) for m in inner], frozen)) assert counts["eq"] == 0 - def test_the_writer_refuses_frozensets_on_one_hash(self) -> None: + def test_the_writer_refuses_same_hash_frozensets_and_tuples_holding_them(self) -> None: assert hash(frozenset({-1})) == hash(frozenset({-2})) - flat_pair = {(-1,), (-2,)} # flat tuples may share a hash, until one tuple in the set nests - assert AutoSerializer().deserialize(*AutoSerializer().serialize(flat_pair)) == flat_pair - nested = ({frozenset({-1}), frozenset({-2})}, {(frozenset({-1}),), (frozenset({-2}),)}, {((-1,),), ((-2,),)}) - for value in (*nested, flat_pair | {((0,),)}): - with pytest.raises(TypeError, match="two frozensets, or two tuples beside a nested one, that share"): - AutoSerializer().serialize(value) + serializer = AutoSerializer() + # Tuples holding only tuples and scalars compare in linear time, so they may share a hash. + edges = {((0, -1), (0, -2)), ((0, -2), (0, -1)), (-1, ((1, 2),)), (-2, ((1, 2),))} + assert serializer.deserialize(*serializer.serialize(edges)) == edges + for value in ( + {frozenset({-1}), frozenset({-2})}, + {(-1, frozenset()), (-2, frozenset())}, + {(((frozenset({-1}),),),), (((frozenset({-2}),),),)}, # however deep the frozenset sits + {(-1, "x"), (-2, "x"), (0, (frozenset(),))}, # once any tuple holds one, every tuple counts + ): + with pytest.raises(TypeError, match="two frozensets, or two tuples holding frozensets, that share"): + serializer.serialize(value) @pytest.mark.parametrize("frozen", [False, True]) def test_a_set_over_the_bound_is_refused_before_any_hash(self, counts: dict[str, int], frozen: bool) -> None: @@ -375,10 +382,13 @@ def test_every_set_shape_at_the_bound_round_trips(self, value: set, kind: type) assert result == value @pytest.mark.parametrize("kind", [set, frozenset]) - def test_the_writer_refuses_a_set_its_reader_would_refuse(self, kind: type) -> None: + def test_the_writer_refuses_a_set_over_the_count(self, kind: type) -> None: with pytest.raises(TypeError, match=rf"cannot store this {kind.__name__}: it holds more than 1024 elements"): AutoSerializer().serialize({"v": [kind(range(_MAX_NON_STRING_SET_ELEMENTS + 1))]}) - # str and bytes subclasses read back as str and bytes, so the writer does not count them either. + + @pytest.mark.parametrize("kind", [set, frozenset]) + def test_the_writer_does_not_count_str_subclasses(self, kind: type) -> None: + """They read back as str, which the reader does not count.""" value = kind({*range(_MAX_NON_STRING_SET_ELEMENTS), *(_Str(i) for i in range(5000))}) serializer = AutoSerializer() assert serializer.deserialize(*serializer.serialize(value)) == value From 75695c1909af33ef024468819ef4555d3c8dfa58 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 00:53:30 +1100 Subject: [PATCH 06/11] docs(serializers): note the set bound's string exemption needs hash randomization --- docs/serializers/auto.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index e948d94c..5c8a4fca 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -71,7 +71,7 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value (a set of ints alone excepted, see below). No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, or a set whose elements read back equal (a pandas `Timestamp` loses its nanoseconds), raises `SerializationError`, which a cached read treats as a miss. -The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. +The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. That holds while Python's hash randomization is on, its default: a process run with `PYTHONHASHSEED` set to a fixed value hashes strings and bytes predictably, so a forged entry could then flood a set of them too. Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples holding frozensets on one hash. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. From 8d18e0bcda1b455de9c3b39817c5e84cbf9fdb8e Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 01:11:59 +1100 Subject: [PATCH 07/11] fix(serializers): say exactly which sets the frozenset rule refuses The refusal message, docstring and docs named only tuples holding frozensets, but once any tuple in a set holds one, every pair of same-hash tuples is refused. The docs also claimed every set whose elements read back equal is refused, though up to 8 still collapse without an error. Count exposed elements through one helper. --- docs/error-codes.md | 2 +- docs/serializers/auto.md | 8 ++++---- src/cachekit/serializers/auto_serializer.py | 18 ++++++++++++------ tests/unit/test_auto_serializer_new_types.py | 11 ++++++++--- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/docs/error-codes.md b/docs/error-codes.md index 5447b144..07a1ee90 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -493,7 +493,7 @@ export CACHEKIT_CONNECTION_POOL_SIZE=100 **Exception**: none raised to a `@cache`-decorated caller: the result is returned but not stored in the backend. The exception is `@cache(interop=...)`: there an unsupported return value raises `InteropError` (a `ValueError` subclass, `from cachekit.interop import InteropError`) by design, so a cross-SDK entry is never silently skipped. Calling the serializer directly raises `TypeError`, e.g. `StandardSerializer does not support custom classes (Python-specific types). ...` or `StandardSerializer does not support pandas DataFrames or Series (Python-specific types). ...` -**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value, or two frozensets, or two tuples holding frozensets, that share one (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). +**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value, or two frozensets that share one, or two tuples that share one in a set where a tuple holds a frozenset (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). **When it occurs**: ```python diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index 5c8a4fca..32aa2859 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,13 +69,13 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value (a set of ints alone excepted, see below). No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, or a set whose elements read back equal (a pandas `Timestamp` loses its nanoseconds), raises `SerializationError`, which a cached read treats as a miss. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value (a set of ints alone excepted, see below). No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. So does a set where more than 8 elements read back equal: pandas `Timestamp`s that differ only in nanoseconds read back as one `datetime`, and up to 8 of them collapse into one element without an error. -The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose element hashes that make one read quadratic in the set's size, either by giving many elements one hash or by giving them distinct hashes whose probe sequences in the set's table merge. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. That holds while Python's hash randomization is on, its default: a process run with `PYTHONHASHSEED` set to a fixed value hashes strings and bytes predictably, so a forged entry could then flood a set of them too. +The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose colliding element hashes that make one read quadratic in the set's size. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. That holds while Python's hash randomization is on, its default: a process run with `PYTHONHASHSEED` set to a fixed value hashes strings and bytes predictably, so a forged entry could then flood a set of them too. -Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples holding frozensets on one hash. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. +Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples on one hash in a set whose tuples hold a frozenset. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. -The reader counts the elements and sorts their hashes before it builds the set, so a refused set is never built. A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 5x the time of an honest set of the same size. To cache more elements, cache them as a list. +A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 5x the time of an honest set of the same size. To cache more elements, cache them as a list. ## When to Use diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index a5db09dc..5de7c1b2 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -342,7 +342,7 @@ def _auto_default(obj: Any) -> Any: types = list(map(type, value)) # Counted as the reader will see them (a str or bytes subclass reads back as str or bytes), but only when the # count could matter: below it a str subclass counts as exposed, which can only make the writer stricter. - exposed = len(types) - types.count(str) - types.count(bytes) + exposed = _exposed_count(types) if exposed > _MAX_NON_STRING_SET_ELEMENTS: exposed = sum(not isinstance(x, (str, bytes)) for x in value) reason = _set_bound_error(value, types, exposed) @@ -374,6 +374,11 @@ def _auto_default(obj: Any) -> Any: raise TypeError(f"Object of type {type(obj).__name__} is not MessagePack serializable") +def _exposed_count(types: list[type]) -> int: + """How many of the values these are the types of are neither str nor bytes, counted in C by list.count.""" + return len(types) - types.count(str) - types.count(bytes) + + def _frozenset_in_tuples(values: list[Any], types: list[type]) -> bool: """Whether a frozenset sits at any depth inside a tuple among ``values``. @@ -388,8 +393,9 @@ def _frozenset_in_tuples(values: list[Any], types: list[type]) -> bool: return True -def _frozenset_comparison(values: list[Any], types: list[type]) -> bool: - """Whether set() over ``values`` could compare two frozensets: two of them, or two tuples holding them, on one hash. +def _may_compare_frozensets(values: list[Any], types: list[type]) -> bool: + """Whether set() over ``values`` could compare two frozensets: two of them share a hash, or two tuples do while a + frozenset sits inside one of the set's tuples. Distinct types are few, and a type hashes by identity, so set(types) is safe. """ @@ -416,8 +422,8 @@ def _set_bound_error(values: list[Any], types: list[type], exposed: int) -> str if types.count(int) == len(types): return None # Hashes are sorted, never tallied in a dict or set: a table keyed by them is the very build being bounded. - if _frozenset_comparison(values, types): - return "two frozensets, or two tuples holding frozensets, that share one hash" + if _may_compare_frozensets(values, types): + return "two frozensets, or two tuples in a set where a tuple holds a frozenset, that share one hash" # Fewer exposed elements cannot fill one hash past the bound: a str or bytes hash is secret, so only the # others' hashes are sorted. if exposed <= _MAX_SAME_HASH_SET_ELEMENTS: @@ -432,7 +438,7 @@ def _check_untrusted_set(values: list[Any]) -> None: """Refuse a stored set's elements if set() over them could be quadratic, before set() builds it.""" # Counted by list.count, which runs in C: a Python loop over every element would slow honest sets several-fold. types = list(map(type, values)) - reason = _set_bound_error(values, types, len(types) - types.count(str) - types.count(bytes)) + reason = _set_bound_error(values, types, _exposed_count(types)) if reason: raise SerializationError( f"Invalid set format: {reason}: a hash-flood bound, not a framing error. Either the entry is forged, " diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index f1f81b95..566ae7ea 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -226,7 +226,8 @@ class _Str(str): class TestAutoSerializerSetHashFloodBound: """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, at most - _MAX_SAME_HASH_SET_ELEMENTS on one hash, and no two frozensets (or tuples holding them) on one hash. Each refusal + _MAX_SAME_HASH_SET_ELEMENTS on one hash, and no two frozensets on one hash (nor two tuples, once a frozenset sits in a + tuple of the set). Each refusal comes before set() builds the set, so a forged one never pays for a quadratic build. The writer refuses the same sets, so an honest one fails at write, not on every read.""" @@ -302,7 +303,9 @@ def test_frozensets_on_one_hash_are_refused_before_any_comparison( same-hash frozensets one level down, so nested same-hash frozensets cost more with every level. Even two on one hash are refused, bare or in a tuple, before set() compares them.""" inner = [{"__set__": True, "value": [_uuid_marker(i)], "frozen": True} for i in _SAME_HASH[:2]] - with pytest.raises(SerializationError, match="two frozensets, or two tuples holding frozensets, that share"): + with pytest.raises( + SerializationError, match="two frozensets, or two tuples in a set where a tuple holds a frozenset, that share" + ): AutoSerializer(enable_integrity_checking=False).deserialize(_set_doc([wrap(m) for m in inner], frozen)) assert counts["eq"] == 0 @@ -318,7 +321,9 @@ def test_the_writer_refuses_same_hash_frozensets_and_tuples_holding_them(self) - {(((frozenset({-1}),),),), (((frozenset({-2}),),),)}, # however deep the frozenset sits {(-1, "x"), (-2, "x"), (0, (frozenset(),))}, # once any tuple holds one, every tuple counts ): - with pytest.raises(TypeError, match="two frozensets, or two tuples holding frozensets, that share"): + with pytest.raises( + TypeError, match="two frozensets, or two tuples in a set where a tuple holds a frozenset, that share" + ): serializer.serialize(value) @pytest.mark.parametrize("frozen", [False, True]) From 93f4941c0d34a6a83b1bf4f2ac03a11b55371a7c Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 01:19:58 +1100 Subject: [PATCH 08/11] fix(serializers): name the lossy round trip in the set refusal message A set the writer accepted can still be refused on read when its elements only become equal once read back: pandas Timestamps that differ only in nanoseconds read back as one datetime. The read error now names that cause beside a forged entry and an older writer, so an operator does not chase a forgery. The code comment, test docstring and docs say the same, and the docs name the frozenset case. --- docs/serializers/auto.md | 4 ++-- src/cachekit/serializers/auto_serializer.py | 14 +++++++++----- tests/unit/test_auto_serializer_new_types.py | 8 ++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index 32aa2859..ead2f1e4 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,13 +69,13 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value (a set of ints alone excepted, see below). No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. So does a set where more than 8 elements read back equal: pandas `Timestamp`s that differ only in nanoseconds read back as one `datetime`, and up to 8 of them collapse into one element without an error. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value (a set of ints alone excepted, see below). No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. So does a set whose elements cross these bounds only once they read back equal: pandas `Timestamp`s that differ only in nanoseconds read back as one `datetime`, so more than 8 of them, or two frozensets of them, are refused, while up to 8 bare ones collapse into one element without an error. The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose colliding element hashes that make one read quadratic in the set's size. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. That holds while Python's hash randomization is on, its default: a process run with `PYTHONHASHSEED` set to a fixed value hashes strings and bytes predictably, so a forged entry could then flood a set of them too. Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples on one hash in a set whose tuples hold a frozenset. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. -A set of ints alone skips the hash sort: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 5x the time of an honest set of the same size. To cache more elements, cache them as a list. +A set of ints alone skips the same-hash check: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 5x the time of an honest set of the same size. To cache more elements, cache them as a list. ## When to Use diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 5de7c1b2..8f7b0e38 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -89,7 +89,8 @@ # a frozenset sits anywhere inside a tuple of the set; a tuple holding no frozenset compares in time linear in its # size. At these bounds, the costliest forged set found (1024 ints on one merged chain) decodes in about 5x # the time of an honest set of the same size. _auto_default applies the same bounds to what it writes, so a refused -# set fails loudly at write instead of missing on every read. +# set fails loudly at write instead of missing on every read, unless its elements only become equal when read back +# (datetimes that differ below a microsecond). _MAX_NON_STRING_SET_ELEMENTS = 1024 _MAX_SAME_HASH_SET_ELEMENTS = 8 _STR_TYPES = frozenset({str, bytes}) @@ -375,7 +376,10 @@ def _auto_default(obj: Any) -> Any: def _exposed_count(types: list[type]) -> int: - """How many of the values these are the types of are neither str nor bytes, counted in C by list.count.""" + """Count of the types that are neither exactly str nor exactly bytes. + + list.count runs in C: a Python loop over every element would slow honest sets several-fold. + """ return len(types) - types.count(str) - types.count(bytes) @@ -436,13 +440,13 @@ def _set_bound_error(values: list[Any], types: list[type], exposed: int) -> str def _check_untrusted_set(values: list[Any]) -> None: """Refuse a stored set's elements if set() over them could be quadratic, before set() builds it.""" - # Counted by list.count, which runs in C: a Python loop over every element would slow honest sets several-fold. types = list(map(type, values)) reason = _set_bound_error(values, types, _exposed_count(types)) if reason: raise SerializationError( - f"Invalid set format: {reason}: a hash-flood bound, not a framing error. Either the entry is forged, " - "or a cachekit version without this bound wrote it." + f"Invalid set format: {reason}: a hash-flood bound, not a framing error. Either the entry is forged, a " + "cachekit version without this bound wrote it, or its elements read back equal (such as datetimes that " + "differ below a microsecond)." ) diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index 566ae7ea..af6a8919 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -226,10 +226,10 @@ class _Str(str): class TestAutoSerializerSetHashFloodBound: """A stored set holds at most _MAX_NON_STRING_SET_ELEMENTS elements that are neither str nor bytes, at most - _MAX_SAME_HASH_SET_ELEMENTS on one hash, and no two frozensets on one hash (nor two tuples, once a frozenset sits in a - tuple of the set). Each refusal - comes before set() builds the set, so a forged one never pays for a quadratic build. The writer refuses the same - sets, so an honest one fails at write, not on every read.""" + _MAX_SAME_HASH_SET_ELEMENTS on one hash, and no two frozensets on one hash (nor two tuples, once a frozenset sits in + a tuple of the set). Each refusal comes before set() builds the set, so a forged one never pays for a quadratic + build. The writer refuses the same sets, so an honest one fails at write, not on every read (unless its elements + only become equal when read back, as datetimes that differ below a microsecond do).""" @pytest.fixture def counts(self, monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: From d688ed01c53edc8791edc00e4d847d43c9a1753b Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 01:26:58 +1100 Subject: [PATCH 09/11] test(serializers): pin the set bound's refusal of Timestamps that read back equal Scope the read error's read-back clause to shared-hash refusals and name the pandas case: stdlib datetimes cannot differ below a microsecond. --- src/cachekit/serializers/auto_serializer.py | 8 ++++---- tests/unit/test_auto_serializer_new_types.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 8f7b0e38..e983e7df 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -89,8 +89,8 @@ # a frozenset sits anywhere inside a tuple of the set; a tuple holding no frozenset compares in time linear in its # size. At these bounds, the costliest forged set found (1024 ints on one merged chain) decodes in about 5x # the time of an honest set of the same size. _auto_default applies the same bounds to what it writes, so a refused -# set fails loudly at write instead of missing on every read, unless its elements only become equal when read back -# (datetimes that differ below a microsecond). +# set fails loudly at write instead of missing on every read, unless its elements only share a hash once read back +# (pandas Timestamps that differ only in nanoseconds read back as one datetime). _MAX_NON_STRING_SET_ELEMENTS = 1024 _MAX_SAME_HASH_SET_ELEMENTS = 8 _STR_TYPES = frozenset({str, bytes}) @@ -445,8 +445,8 @@ def _check_untrusted_set(values: list[Any]) -> None: if reason: raise SerializationError( f"Invalid set format: {reason}: a hash-flood bound, not a framing error. Either the entry is forged, a " - "cachekit version without this bound wrote it, or its elements read back equal (such as datetimes that " - "differ below a microsecond)." + "cachekit version without this bound wrote it, or, for a shared hash, its elements read back equal (such " + "as pandas Timestamps that differ only in nanoseconds)." ) diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index af6a8919..8de210d9 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -229,7 +229,7 @@ class TestAutoSerializerSetHashFloodBound: _MAX_SAME_HASH_SET_ELEMENTS on one hash, and no two frozensets on one hash (nor two tuples, once a frozenset sits in a tuple of the set). Each refusal comes before set() builds the set, so a forged one never pays for a quadratic build. The writer refuses the same sets, so an honest one fails at write, not on every read (unless its elements - only become equal when read back, as datetimes that differ below a microsecond do).""" + only share a hash once read back, as pandas Timestamps that differ only in nanoseconds do).""" @pytest.fixture def counts(self, monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: @@ -263,6 +263,16 @@ def test_a_same_hash_set_is_refused_before_any_comparison( # Hashed once each, by the sort; set() would have compared each to every earlier one. assert counts == {"hash": len(_SAME_HASH), "eq": 0} + def test_timestamps_that_read_back_equal_are_refused_on_read_past_the_bounds(self) -> None: + """The writer sees distinct nanosecond Timestamps; the reader sees equal datetimes, so equal hashes.""" + pd = pytest.importorskip("pandas") + stamps = [pd.Timestamp("2026-01-01") + pd.Timedelta(i, "ns") for i in range(9)] + serializer = AutoSerializer() + for value in (set(stamps), {frozenset(stamps[:1]), frozenset(stamps[1:2])}): + with pytest.raises(SerializationError, match="for a shared hash, its elements read back equal"): + serializer.deserialize(*serializer.serialize(value)) + assert serializer.deserialize(*serializer.serialize(set(stamps[:8]))) == {datetime(2026, 1, 1)} + def test_eight_elements_may_share_a_hash_but_not_nine(self) -> None: """hash(-1) == hash(-2), so 3-D offsets over -1 and -2 put 8 honest tuples on one hash; 4-D offsets put 16.""" serializer = AutoSerializer() From 21ef90d3418c97c15c2fae0c48ef89904ab28e38 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 04:05:21 +1100 Subject: [PATCH 10/11] fix(serializers): bound empty-string and duplicate-int set floods Review found two shapes that passed the set bound. "" and b"" hash to 0 in every process, and a stored list may repeat them at 1 byte a copy, counted toward neither bound, so each copy walked whatever forged ints sat on hash 0's probe path. A stored set may now hold each at most once, as an honest one does. And the all-int shortcut assumed at most 13 ints share a hash, which holds for a live set but not for a stored list that repeats an int: copies pinned the table below its resize and each walked a long probe chain. The shortcut is gone, so every element's hash is checked; a run of more than 8 equal hashes, copies included, is refused. The checks now run only when their types are present, and the sort-and-compare lives in one helper. Sorting the hashes is the guard's main cost on honest sets of ints and floats. The docs drop the shortcut and the worst-case figure, name the empty values, and note that an aware datetime at fold=1 can come back equal to another element. --- docs/error-codes.md | 2 + docs/serializers/auto.md | 8 +- src/cachekit/serializers/auto_serializer.py | 102 +++++++++---------- tests/unit/test_auto_serializer_new_types.py | 40 +++++--- 4 files changed, 79 insertions(+), 73 deletions(-) diff --git a/docs/error-codes.md b/docs/error-codes.md index 07a1ee90..afdd2092 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -549,6 +549,8 @@ def get_json_data(): **Message** (logged): `L2 cache decrypt/integrity failure (corruption) for ...: ...`, or `(envelope_shape)` in place of `(corruption)` for the envelope-shape refusal described below +With `serializer='auto'`, `Invalid set format: ... a hash-flood bound, not a framing error.` is logged the same way: a stored set or frozenset is over the [Set Size Bound](serializers/auto.md#set-size-bound). A forged entry, an entry written by a cachekit version without the bound, or a set whose elements read back equal (pandas `Timestamp`s that differ only in nanoseconds) causes it; the function recomputes as for any corrupt entry. + **Exception**: none under `@cache` — cachekit attempts to evict the entry (best effort; a failed delete is logged and the entry stays) and the function recomputes. Calling a serializer's `deserialize()` directly raises `SerializationError` (there is no separate `DeserializationError` class). **Cause**: Cached data is corrupted, or was written by an incompatible serializer/config. This is corruption *detection*, not tamper detection: the plaintext checksum is unkeyed xxHash3-64, which anyone with backend write access can recompute. Tamper detection requires encryption — see *Decryption failed* above. diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index ead2f1e4..4b1067e8 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,13 +69,15 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, and at most 8 elements that share one hash value (a set of ints alone excepted, see below). No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. So does a set whose elements cross these bounds only once they read back equal: pandas `Timestamp`s that differ only in nanoseconds read back as one `datetime`, so more than 8 of them, or two frozensets of them, are refused, while up to 8 bare ones collapse into one element without an error. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, at most 8 elements that share one hash value, and the empty string `""` and empty bytes `b""` at most once each. No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. -The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose colliding element hashes that make one read quadratic in the set's size. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies. That holds while Python's hash randomization is on, its default: a process run with `PYTHONHASHSEED` set to a fixed value hashes strings and bytes predictably, so a forged entry could then flood a set of them too. +A set whose elements change their hash on the round trip can cross these bounds only once read back, so the writer accepts it and every read refuses it. pandas `Timestamp`s that differ only in nanoseconds read back as one `datetime`, so more than 8 of them, or two frozensets of them, are refused, while up to 8 bare ones collapse into one element without an error. An aware `datetime` at `fold=1` reads back with its offset but without `fold`, so it can come back equal to another element of the set. + +The count takes every element that is not a string or bytes, whatever its type: an int, a float, a bool, `None`, a UUID, a tuple, a frozenset or a datetime. Ints, floats, UUIDs and tuples hash without a random seed, so a forged entry could choose colliding element hashes that make one read quadratic in the set's size. A stored set is a list, so it can also repeat an element, and every copy counts toward both bounds. Strings and bytes hash under a per-process secret, so a set of them is bounded only by the size bounds every read applies, except `""` and `b""`, which hash to 0 in every process. The secret holds while Python's hash randomization is on, its default: a process run with `PYTHONHASHSEED` set to a fixed value hashes strings and bytes predictably, so a forged entry could then flood a set of them too. Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples on one hash in a set whose tuples hold a frozenset. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. -A set of ints alone skips the same-hash check: at most 13 ints in MessagePack's integer range share a hash, and they compare cheaply. Within the bounds, the costliest forged set found, 1,024 ints whose probe sequences merge, decodes in about 5x the time of an honest set of the same size. To cache more elements, cache them as a list. +Checking a set costs a sort of its elements' hashes on every read, which slows decoding a large set of ints or floats several-fold; sets of strings skip it. To cache more elements than the bounds admit, cache them as a list. ## When to Use diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index e983e7df..1719a989 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -29,6 +29,7 @@ import logging import operator import sys +from collections.abc import Iterable from datetime import date, datetime, time from functools import cached_property from importlib.util import find_spec @@ -75,25 +76,13 @@ # never travel in an envelope. serialize() always writes the field, so an absent one is rot too. _HEADER_FORMATS = _ENVELOPE_FORMATS | {"numpy", "arrow"} -# Bounds on one set or frozenset, so a forged entry cannot make set() quadratic. str and bytes hash under a -# per-process secret; an int, float, UUID or tuple hashes without one, so a forged entry can choose its elements' -# hashes, in two ways (interop.py's _MAX_NON_STRING_KEYS explains both for dicts; sets probe the same way). -# Distinct hashes whose probe sequences merge make each insert walk one shared chain with no __eq__ at all; the -# count of elements that are neither str nor bytes bounds that. Sharing one hash makes each insert call __eq__ on -# every earlier element of that hash: _set_bound_error refuses more than _MAX_SAME_HASH_SET_ELEMENTS elements on -# one hash, so an insert makes fewer __eq__ calls than that (an all-int set is exempt: at most 13 ints share a hash -# and they compare in C). 8 admits honest collisions such as 3-D offset tuples over -1 and -2 (hash(-1) == -# hash(-2)). That bounds the number of __eq__ calls but not their cost: two frozensets compare by looking up each -# element of one in the other, and each lookup can compare same-hash frozensets one level down, so the cost -# multiplies with every level of nesting. Two frozensets may therefore never share a hash, nor may two tuples once -# a frozenset sits anywhere inside a tuple of the set; a tuple holding no frozenset compares in time linear in its -# size. At these bounds, the costliest forged set found (1024 ints on one merged chain) decodes in about 5x -# the time of an honest set of the same size. _auto_default applies the same bounds to what it writes, so a refused -# set fails loudly at write instead of missing on every read, unless its elements only share a hash once read back -# (pandas Timestamps that differ only in nanoseconds read back as one datetime). +# Bounds on one stored set or frozenset, so a forged entry cannot make set() quadratic by choosing its elements' +# hashes (why, and what each bound admits: docs/serializers/auto.md#set-size-bound). _set_bound_error applies them to +# the reader's list and to what _auto_default writes. _MAX_NON_STRING_SET_ELEMENTS = 1024 _MAX_SAME_HASH_SET_ELEMENTS = 8 _STR_TYPES = frozenset({str, bytes}) +_CONTAINERS = (tuple, frozenset) # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( @@ -340,13 +329,10 @@ def _auto_default(obj: Any) -> Any: # NEW: set/frozenset support (type-safe roundtrip) if isinstance(obj, (set, frozenset)): value = list(obj) - types = list(map(type, value)) - # Counted as the reader will see them (a str or bytes subclass reads back as str or bytes), but only when the - # count could matter: below it a str subclass counts as exposed, which can only make the writer stricter. - exposed = _exposed_count(types) - if exposed > _MAX_NON_STRING_SET_ELEMENTS: - exposed = sum(not isinstance(x, (str, bytes)) for x in value) - reason = _set_bound_error(value, types, exposed) + # Counted as the reader will see them (a str or bytes subclass reads back as str or bytes) only when the count + # could matter. Below it a str subclass counts as exposed, which can only make the writer stricter. + recount = sum(not isinstance(x, (str, bytes)) for x in value) if len(value) > _MAX_NON_STRING_SET_ELEMENTS else None + reason = _set_bound_error(value, list(map(type, value)), recount) if reason: raise TypeError( f"AutoSerializer cannot store this {type(obj).__name__}: it holds {reason}, which its reader refuses " @@ -375,14 +361,6 @@ def _auto_default(obj: Any) -> Any: raise TypeError(f"Object of type {type(obj).__name__} is not MessagePack serializable") -def _exposed_count(types: list[type]) -> int: - """Count of the types that are neither exactly str nor exactly bytes. - - list.count runs in C: a Python loop over every element would slow honest sets several-fold. - """ - return len(types) - types.count(str) - types.count(bytes) - - def _frozenset_in_tuples(values: list[Any], types: list[type]) -> bool: """Whether a frozenset sits at any depth inside a tuple among ``values``. @@ -397,51 +375,63 @@ def _frozenset_in_tuples(values: list[Any], types: list[type]) -> bool: return True -def _may_compare_frozensets(values: list[Any], types: list[type]) -> bool: - """Whether set() over ``values`` could compare two frozensets: two of them share a hash, or two tuples do while a - frozenset sits inside one of the set's tuples. +def _hash_run_over(values: Iterable[Any], limit: int) -> bool: + """Whether more than ``limit`` of ``values`` share one hash. - Distinct types are few, and a type hashes by identity, so set(types) is safe. + The hashes are sorted, never tallied in a dict or set: a table keyed by them is the very build being bounded. + """ + hashes = sorted(map(hash, values)) + return any(map(operator.eq, hashes, hashes[limit:])) + + +def _empties(values: list[Any], types: list[type], kind: type) -> int: + """How many of ``values`` are an empty ``kind`` (str or bytes), counted in C passes.""" + return sum(map(operator.not_, compress(values, map(operator.is_, types, repeat(kind))))) + + +def _may_compare_frozensets(values: list[Any], types: list[type], kinds: set[type]) -> bool: + """Whether set() over ``values`` could compare two frozensets: two of them share a hash, or two tuples do while a + frozenset sits inside one of the set's tuples. ``kinds`` is set(types). """ - kinds = set(types) checked: list[type] = [frozenset] if any(issubclass(t, frozenset) for t in kinds) else [] if any(issubclass(t, tuple) for t in kinds) and _frozenset_in_tuples(values, types): checked.append(tuple) - for kind in checked: - hashes = sorted(map(hash, compress(values, map(issubclass, types, repeat(kind))))) - if any(map(operator.eq, hashes, hashes[1:])): - return True - return False + return any(_hash_run_over(compress(values, map(issubclass, types, repeat(kind))), 1) for kind in checked) -def _set_bound_error(values: list[Any], types: list[type], exposed: int) -> str | None: - """Why set() over ``values`` could be quadratic, or None (see _MAX_NON_STRING_SET_ELEMENTS). +def _set_bound_error(values: list[Any], types: list[type], exposed: int | None = None) -> str | None: + """Why set() over ``values`` could be quadratic, or None (see docs/serializers/auto.md#set-size-bound). - ``types`` holds each value's type; ``exposed`` is how many values are neither str nor bytes. + ``types`` holds each value's type. ``exposed`` is how many values are neither str nor bytes, by default counted + from ``types``. Every pass runs in C, and a pass only runs when its types are present: a Python loop over every + element would slow honest sets several-fold. """ + # Distinct types are few, and a type hashes by identity, so this set is safe to build. + kinds = set(types) + strings = str in kinds or bytes in kinds + if exposed is None: + exposed = len(types) - types.count(str) - types.count(bytes) if strings else len(types) if exposed > _MAX_NON_STRING_SET_ELEMENTS: return f"more than {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes" - # At most 13 ints in msgpack's int range share a hash (k * (2**61 - 1) for k in -4..8), and they compare in C: - # an all-int set needs no hash check, and sorting an honest one's hashes would cost more than building it. - if types.count(int) == len(types): - return None - # Hashes are sorted, never tallied in a dict or set: a table keyed by them is the very build being bounded. - if _may_compare_frozensets(values, types): + # Every other str and bytes hashes under a per-process secret, but "" and b"" hash to 0 in every process, and a + # stored list can repeat them for 1 byte a copy: each copy would walk whatever forged elements pile on hash 0's + # probe path. An honest set holds each at most once. + if any(kind in kinds and _empties(values, types, kind) > 1 for kind in (str, bytes)): + return 'the empty str or bytes ("" or b"") more than once' + if any(issubclass(t, _CONTAINERS) for t in kinds) and _may_compare_frozensets(values, types, kinds): return "two frozensets, or two tuples in a set where a tuple holds a frozenset, that share one hash" - # Fewer exposed elements cannot fill one hash past the bound: a str or bytes hash is secret, so only the - # others' hashes are sorted. + # Fewer exposed elements cannot put more than the bound on one hash (at most one "" and one b"" join them). if exposed <= _MAX_SAME_HASH_SET_ELEMENTS: return None - hashes = sorted(map(hash, compress(values, map(operator.not_, map(_STR_TYPES.__contains__, types))))) - if any(map(operator.eq, hashes, hashes[_MAX_SAME_HASH_SET_ELEMENTS:])): + others = compress(values, map(operator.not_, map(_STR_TYPES.__contains__, types))) if strings else values + if _hash_run_over(others, _MAX_SAME_HASH_SET_ELEMENTS): return f"more than {_MAX_SAME_HASH_SET_ELEMENTS} elements that share one hash" return None def _check_untrusted_set(values: list[Any]) -> None: """Refuse a stored set's elements if set() over them could be quadratic, before set() builds it.""" - types = list(map(type, values)) - reason = _set_bound_error(values, types, _exposed_count(types)) + reason = _set_bound_error(values, list(map(type, values))) if reason: raise SerializationError( f"Invalid set format: {reason}: a hash-flood bound, not a framing error. Either the entry is forged, a " diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index 8de210d9..06b85b58 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -288,21 +288,38 @@ def test_eight_elements_may_share_a_hash_but_not_nine(self) -> None: } def test_ints_floats_and_other_types_on_one_hash_count_together(self) -> None: - """No type is exempt: 202 distinct floats can share one hash, and a UUID's Python __eq__ runs against each. - Only an all-int set skips the check: at most 13 ints in msgpack's range share a hash, and they compare in C.""" + """No type is exempt: 202 distinct floats can share one hash, and a UUID's Python __eq__ runs against each.""" p = 2**61 - 1 serializer = AutoSerializer(enable_integrity_checking=False) - all_int = [k * p for k in range(-4, 9)] - assert {hash(k) for k in all_int} == {0} - assert serializer.deserialize(_set_doc(all_int, False)) == set(all_int) for mixed in ( [*(2.0 ** (61 * k) for k in range(9))], # hash 1 - [*(k * p for k in range(-4, 5)), 0.5], # hash 0, and a float, so not an all-int set + [*(k * p for k in range(-4, 5))], # hash 0 [*(2.0 ** (61 * k) for k in range(5)), *(_uuid_marker(1 + j * p) for j in range(4))], # hash 1 ): for frozen in (False, True): with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): - serializer.deserialize(_set_doc(mixed, frozen)) + serializer.deserialize(_set_doc([*mixed, "a"], frozen)) # a str too: the sort then masks + + @pytest.mark.parametrize("frozen", [False, True]) + def test_duplicates_count_toward_the_hash_bound(self, frozen: bool) -> None: + """A stored list can repeat an element, which the set dedups only after each copy has walked its probe path, + so copies count like any elements on one hash. An honest writer never repeats one.""" + serializer = AutoSerializer(enable_integrity_checking=False) + with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): + serializer.deserialize(_set_doc([*range(100), *[25] * 8], frozen)) + assert serializer.deserialize(_set_doc([*range(100), *[25] * 7], frozen)) == set(range(100)) + + @pytest.mark.parametrize("frozen", [False, True]) + @pytest.mark.parametrize("empty", ["", b""]) + def test_an_empty_str_or_bytes_may_appear_once(self, frozen: bool, empty: str | bytes) -> None: + """Both hash to 0 in every process, so copies of one would all walk hash 0's probe path, and they count + toward neither bound.""" + assert hash(empty) == 0 + serializer = AutoSerializer(enable_integrity_checking=False) + with pytest.raises(SerializationError, match=r"the empty str or bytes \(\"\" or b\"\"\) more than once"): + serializer.deserialize(_set_doc([0, empty, empty], frozen)) + honest = {0, "a", "", b""} + assert AutoSerializer().deserialize(*AutoSerializer().serialize(honest)) == honest @pytest.mark.parametrize("frozen", [False, True]) @pytest.mark.parametrize("wrap", [lambda m: m, lambda m: {"__tuple__": True, "value": [m]}], ids=["bare", "in a tuple"]) @@ -349,6 +366,8 @@ def test_the_bound_counts_only_elements_that_are_neither_str_nor_bytes(self) -> assert serializer.deserialize(_set_doc([*strings, *range(1024)], False)) == {*strings, *range(1024)} with pytest.raises(SerializationError, match="more than 1024 elements"): serializer.deserialize(_set_doc([*strings, *range(1025)], False)) + only_bytes = [b"%d" % i for i in range(2000)] + assert serializer.deserialize(_set_doc([*only_bytes, *range(1024)], False)) == {*only_bytes, *range(1024)} def test_a_set_at_the_bound_hashes_each_element_twice(self, counts: dict[str, int]) -> None: value = frozenset(UUID(int=i) for i in range(_MAX_NON_STRING_SET_ELEMENTS)) @@ -401,13 +420,6 @@ def test_the_writer_refuses_a_set_over_the_count(self, kind: type) -> None: with pytest.raises(TypeError, match=rf"cannot store this {kind.__name__}: it holds more than 1024 elements"): AutoSerializer().serialize({"v": [kind(range(_MAX_NON_STRING_SET_ELEMENTS + 1))]}) - @pytest.mark.parametrize("kind", [set, frozenset]) - def test_the_writer_does_not_count_str_subclasses(self, kind: type) -> None: - """They read back as str, which the reader does not count.""" - value = kind({*range(_MAX_NON_STRING_SET_ELEMENTS), *(_Str(i) for i in range(5000))}) - serializer = AutoSerializer() - assert serializer.deserialize(*serializer.serialize(value)) == value - class TestAutoSerializerComplexNesting: """Test nested complex objects with new types.""" From ae597051a8022ed6bd4cc2483dc09f7fa16cab42 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 8 Nov 2026 04:33:57 +1100 Subject: [PATCH 11/11] fix(serializers): count set elements in C before the writer's recount The writer ran its Python isinstance recount on every set of more than 1024 elements, all-string sets included. It now counts the types in C first and recounts only when that count is over the bound. A set of only str or only bytes takes its own branch: nothing is exposed, and list.count finds a repeated empty value. The per-hash bound counts only elements that are neither str nor bytes, and its refusal says so. Docs: a refused stored set reaches the @cache log only as SerializationError (the log carries the class, not the text); the cost note covers reads and writes and string sets; fold=1 is named beside the Timestamp cause. --- docs/error-codes.md | 4 +-- docs/serializers/auto.md | 4 +-- src/cachekit/serializers/auto_serializer.py | 33 ++++++++++++-------- tests/unit/test_auto_serializer_new_types.py | 24 ++++++++++---- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/docs/error-codes.md b/docs/error-codes.md index afdd2092..7cf3395a 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -493,7 +493,7 @@ export CACHEKIT_CONNECTION_POOL_SIZE=100 **Exception**: none raised to a `@cache`-decorated caller: the result is returned but not stored in the backend. The exception is `@cache(interop=...)`: there an unsupported return value raises `InteropError` (a `ValueError` subclass, `from cachekit.interop import InteropError`) by design, so a cross-SDK entry is never silently skipped. Calling the serializer directly raises `TypeError`, e.g. `StandardSerializer does not support custom classes (Python-specific types). ...` or `StandardSerializer does not support pandas DataFrames or Series (Python-specific types). ...` -**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 that share one hash value, or two frozensets that share one, or two tuples that share one in a set where a tuple holds a frozenset (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). +**Cause**: The default serializer handles `None`, `bool`, `int`, `float`, `str`, `bytes`, `list`, `tuple`, `dict`, `datetime`, `date` and `time`. Custom classes, dataclasses and DataFrames are not supported. `serializer='auto'` also refuses a set or frozenset holding more than 1,024 elements that are neither strings nor bytes, or more than 8 of those that share one hash value, or two frozensets that share one, or two tuples that share one in a set where a tuple holds a frozenset (`AutoSerializer cannot store this set: it holds ...`): see [Set Size Bound](serializers/auto.md#set-size-bound). **When it occurs**: ```python @@ -549,7 +549,7 @@ def get_json_data(): **Message** (logged): `L2 cache decrypt/integrity failure (corruption) for ...: ...`, or `(envelope_shape)` in place of `(corruption)` for the envelope-shape refusal described below -With `serializer='auto'`, `Invalid set format: ... a hash-flood bound, not a framing error.` is logged the same way: a stored set or frozenset is over the [Set Size Bound](serializers/auto.md#set-size-bound). A forged entry, an entry written by a cachekit version without the bound, or a set whose elements read back equal (pandas `Timestamp`s that differ only in nanoseconds) causes it; the function recomputes as for any corrupt entry. +With `serializer='auto'`, a stored set or frozenset over the [Set Size Bound](serializers/auto.md#set-size-bound) is handled the same way. Under `@cache` the log line ends in `SerializationError` and names no bound, because the log carries only the exception's class; the full text, `Invalid set format: ... a hash-flood bound, not a framing error. ...`, appears only when you call `deserialize()` directly. A forged entry, an entry written by a cachekit version without the bound, or a set whose elements read back equal (pandas `Timestamp`s that differ only in nanoseconds, or an aware `datetime` at `fold=1`) causes it; the function recomputes as for any corrupt entry. **Exception**: none under `@cache` — cachekit attempts to evict the entry (best effort; a failed delete is logged and the entry stays) and the function recomputes. Calling a serializer's `deserialize()` directly raises `SerializationError` (there is no separate `DeserializationError` class). diff --git a/docs/serializers/auto.md b/docs/serializers/auto.md index 4b1067e8..ee9f8884 100644 --- a/docs/serializers/auto.md +++ b/docs/serializers/auto.md @@ -69,7 +69,7 @@ Without pyarrow, DataFrames use the columnar MessagePack format. If pyarrow is i ## Set Size Bound -A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, at most 8 elements that share one hash value, and the empty string `""` and empty bytes `b""` at most once each. No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. +A set or frozenset holds at most 1,024 elements that are neither strings nor bytes, at most 8 of those that share one hash value, and the empty string `""` and empty bytes `b""` at most once each. No two of its frozensets may share a hash, and no two of its tuples may once a frozenset sits anywhere inside one of its tuples. Writing a set over any of these bounds raises `TypeError`; under `@cache` the result is returned but not cached, and the failure is logged (see [Serialization unsupported type](../error-codes.md#serialization-unsupported-type)). Reading one, which a forged entry or an earlier cachekit version can hold, raises `SerializationError`, which a cached read treats as a miss. A set whose elements change their hash on the round trip can cross these bounds only once read back, so the writer accepts it and every read refuses it. pandas `Timestamp`s that differ only in nanoseconds read back as one `datetime`, so more than 8 of them, or two frozensets of them, are refused, while up to 8 bare ones collapse into one element without an error. An aware `datetime` at `fold=1` reads back with its offset but without `fold`, so it can come back equal to another element of the set. @@ -77,7 +77,7 @@ The count takes every element that is not a string or bytes, whatever its type: Honest elements can share a hash: `hash(-1) == hash(-2)`, so the 3-D offsets over -1 and -2, such as `(-1, -2, -1)`, put 8 tuples on one hash, the most the bound admits; 4-D offsets put 16 there and are refused. Frozensets get no such allowance, because comparing two of them costs more with every level they nest. So `{frozenset({-1}), frozenset({-2})}` is refused, and so is `{(-1, frozenset()), (-2, frozenset())}`, two tuples on one hash in a set whose tuples hold a frozenset. Tuples that hold only tuples and scalars keep the allowance: `{((0, -1), (0, -2)), ((0, -2), (0, -1))}` round-trips. -Checking a set costs a sort of its elements' hashes on every read, which slows decoding a large set of ints or floats several-fold; sets of strings skip it. To cache more elements than the bounds admit, cache them as a list. +Checking a set adds cost to every read and every write of it. For a set of more than 8 ints, floats, UUIDs or other elements that are not strings, sorting their hashes dominates, and it can make decoding a large set several times slower. A set of only strings or only bytes pays a smaller per-element check. To cache more elements than the bounds admit, cache them as a list. ## When to Use diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 1719a989..6963bcc2 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -82,7 +82,7 @@ _MAX_NON_STRING_SET_ELEMENTS = 1024 _MAX_SAME_HASH_SET_ELEMENTS = 8 _STR_TYPES = frozenset({str, bytes}) -_CONTAINERS = (tuple, frozenset) +_EMPTY_TWICE = 'the empty str or bytes ("" or b"") more than once' # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( @@ -329,10 +329,13 @@ def _auto_default(obj: Any) -> Any: # NEW: set/frozenset support (type-safe roundtrip) if isinstance(obj, (set, frozenset)): value = list(obj) - # Counted as the reader will see them (a str or bytes subclass reads back as str or bytes) only when the count - # could matter. Below it a str subclass counts as exposed, which can only make the writer stricter. - recount = sum(not isinstance(x, (str, bytes)) for x in value) if len(value) > _MAX_NON_STRING_SET_ELEMENTS else None - reason = _set_bound_error(value, list(map(type, value)), recount) + types = list(map(type, value)) + exposed = len(types) - types.count(str) - types.count(bytes) + if exposed > _MAX_NON_STRING_SET_ELEMENTS: + # A str or bytes subclass reads back as str or bytes: recount as the reader will see them. Below the + # bound a subclass counts as exposed, which can only make the writer stricter. + exposed = sum(not isinstance(x, (str, bytes)) for x in value) + reason = _set_bound_error(value, types, exposed) if reason: raise TypeError( f"AutoSerializer cannot store this {type(obj).__name__}: it holds {reason}, which its reader refuses " @@ -384,7 +387,7 @@ def _hash_run_over(values: Iterable[Any], limit: int) -> bool: return any(map(operator.eq, hashes, hashes[limit:])) -def _empties(values: list[Any], types: list[type], kind: type) -> int: +def _empty_count(values: list[Any], types: list[type], kind: type) -> int: """How many of ``values`` are an empty ``kind`` (str or bytes), counted in C passes.""" return sum(map(operator.not_, compress(values, map(operator.is_, types, repeat(kind))))) @@ -408,24 +411,28 @@ def _set_bound_error(values: list[Any], types: list[type], exposed: int | None = """ # Distinct types are few, and a type hashes by identity, so this set is safe to build. kinds = set(types) + if len(kinds) == 1 and kinds <= _STR_TYPES: + # All str or all bytes, the most common large set: nothing is exposed, and list.count finds the empty value. + return _EMPTY_TWICE if values.count(types[0]()) > 1 else None strings = str in kinds or bytes in kinds if exposed is None: exposed = len(types) - types.count(str) - types.count(bytes) if strings else len(types) if exposed > _MAX_NON_STRING_SET_ELEMENTS: return f"more than {_MAX_NON_STRING_SET_ELEMENTS} elements that are neither str nor bytes" # Every other str and bytes hashes under a per-process secret, but "" and b"" hash to 0 in every process, and a - # stored list can repeat them for 1 byte a copy: each copy would walk whatever forged elements pile on hash 0's - # probe path. An honest set holds each at most once. - if any(kind in kinds and _empties(values, types, kind) > 1 for kind in (str, bytes)): - return 'the empty str or bytes ("" or b"") more than once' - if any(issubclass(t, _CONTAINERS) for t in kinds) and _may_compare_frozensets(values, types, kinds): + # stored list can repeat them for 1 or 2 bytes a copy: each copy would walk whatever forged elements pile on hash + # 0's probe path. An honest set holds each at most once. + if any(kind in kinds and _empty_count(values, types, kind) > 1 for kind in (str, bytes)): + return _EMPTY_TWICE + if _may_compare_frozensets(values, types, kinds): return "two frozensets, or two tuples in a set where a tuple holds a frozenset, that share one hash" - # Fewer exposed elements cannot put more than the bound on one hash (at most one "" and one b"" join them). + # The per-hash bound counts the elements that are neither str nor bytes (only one "" and one b"" can join their + # run on hash 0), so fewer of them than the bound cannot exceed it. if exposed <= _MAX_SAME_HASH_SET_ELEMENTS: return None others = compress(values, map(operator.not_, map(_STR_TYPES.__contains__, types))) if strings else values if _hash_run_over(others, _MAX_SAME_HASH_SET_ELEMENTS): - return f"more than {_MAX_SAME_HASH_SET_ELEMENTS} elements that share one hash" + return f"more than {_MAX_SAME_HASH_SET_ELEMENTS} elements that are neither str nor bytes and share one hash" return None diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index 06b85b58..0a855049 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -258,7 +258,10 @@ def test_a_same_hash_set_is_refused_before_any_comparison( self, counts: dict[str, int], frozen: bool, element: Callable[[int], dict] ) -> None: data = _set_doc([element(i) for i in _SAME_HASH], frozen) - with pytest.raises(SerializationError, match="more than 8 elements that share one hash: a hash-flood bound"): + with pytest.raises( + SerializationError, + match="more than 8 elements that are neither str nor bytes and share one hash: a hash-flood bound", + ): AutoSerializer(enable_integrity_checking=False).deserialize(data) # Hashed once each, by the sort; set() would have compared each to every earlier one. assert counts == {"hash": len(_SAME_HASH), "eq": 0} @@ -279,9 +282,9 @@ def test_eight_elements_may_share_a_hash_but_not_nine(self) -> None: offsets = set(itertools.product(range(-2, 3), repeat=3)) assert len({hash(t) for t in itertools.product((-1, -2), repeat=3)}) == 1 assert serializer.deserialize(*serializer.serialize(offsets)) == offsets - with pytest.raises(TypeError, match="it holds more than 8 elements that share one hash"): + with pytest.raises(TypeError, match="it holds more than 8 elements that are neither str nor bytes and share one hash"): serializer.serialize(set(itertools.product(range(-2, 3), repeat=4))) - with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): + with pytest.raises(SerializationError, match="more than 8 elements that are neither str nor bytes and share one hash"): serializer.deserialize(_set_doc([_uuid_marker(i) for i in _SAME_HASH[:9]], False), None) assert serializer.deserialize(_set_doc([_uuid_marker(i) for i in _SAME_HASH[:8]], False), None) == { UUID(int=i) for i in _SAME_HASH[:8] @@ -297,15 +300,19 @@ def test_ints_floats_and_other_types_on_one_hash_count_together(self) -> None: [*(2.0 ** (61 * k) for k in range(5)), *(_uuid_marker(1 + j * p) for j in range(4))], # hash 1 ): for frozen in (False, True): - with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): - serializer.deserialize(_set_doc([*mixed, "a"], frozen)) # a str too: the sort then masks + with pytest.raises( + SerializationError, match="more than 8 elements that are neither str nor bytes and share one hash" + ): + serializer.deserialize( + _set_doc([*mixed, "a"], frozen) + ) # a str forces the branch that filters strings out before the sort @pytest.mark.parametrize("frozen", [False, True]) def test_duplicates_count_toward_the_hash_bound(self, frozen: bool) -> None: """A stored list can repeat an element, which the set dedups only after each copy has walked its probe path, so copies count like any elements on one hash. An honest writer never repeats one.""" serializer = AutoSerializer(enable_integrity_checking=False) - with pytest.raises(SerializationError, match="more than 8 elements that share one hash"): + with pytest.raises(SerializationError, match="more than 8 elements that are neither str nor bytes and share one hash"): serializer.deserialize(_set_doc([*range(100), *[25] * 8], frozen)) assert serializer.deserialize(_set_doc([*range(100), *[25] * 7], frozen)) == set(range(100)) @@ -318,6 +325,11 @@ def test_an_empty_str_or_bytes_may_appear_once(self, frozen: bool, empty: str | serializer = AutoSerializer(enable_integrity_checking=False) with pytest.raises(SerializationError, match=r"the empty str or bytes \(\"\" or b\"\"\) more than once"): serializer.deserialize(_set_doc([0, empty, empty], frozen)) + # All str or all bytes takes its own branch, which counts the empty value with list.count. + a, b = ("a", "b") if empty == "" else (b"a", b"b") + with pytest.raises(SerializationError, match="more than once"): + serializer.deserialize(_set_doc([a, empty, b, empty], frozen)) + assert serializer.deserialize(_set_doc([a, empty, b], frozen)) == {a, empty, b} honest = {0, "a", "", b""} assert AutoSerializer().deserialize(*AutoSerializer().serialize(honest)) == honest