diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index de90e254c..7d51d2990 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -825,12 +825,14 @@ ctypedef struct user_filters_udata: int input_cdtype int output_cdtype int32_t chunkshape + int32_t itemsize # of both dtypes ctypedef struct filler_udata: char* py_func uintptr_t inputs_id int output_cdtype int32_t chunkshape + int32_t itemsize # of the output dtype ctypedef struct udf_udata: char* py_func @@ -4793,6 +4795,7 @@ cdef class SChunk: postf_udata.input_cdtype = dtype_input.num postf_udata.output_cdtype = dtype_output.num postf_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize + postf_udata.itemsize = dtype_input.itemsize postparams.user_data = postf_udata dparams.postparams = postparams @@ -4847,6 +4850,7 @@ cdef class SChunk: fill_udata.inputs_id = inputs_id fill_udata.output_cdtype = np.dtype(dtype_output).num fill_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize + fill_udata.itemsize = np.dtype(dtype_output).itemsize preparams.user_data = fill_udata cparams.preparams = preparams @@ -4881,6 +4885,7 @@ cdef class SChunk: pref_udata.input_cdtype = dtype_input.num pref_udata.output_cdtype = dtype_output.num pref_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize + pref_udata.itemsize = dtype_input.itemsize preparams.user_data = pref_udata cparams.preparams = preparams @@ -5002,9 +5007,18 @@ cdef class SChunk: blosc2_schunk_free(schunk_ptr) +# The arrays that general_postfilter, general_filler and general_prefilter hand to Python have +# size // typesize items of the callback's dtype, so they only fit buffers whose typesize is the +# dtype's itemsize. Other buffers (C-Blosc2 before Blosc/c-blosc2#857 passes vlmetalayer content +# to the postfilter, and a chunk can have another typesize than its SChunk) get a ValueError, raised +# before anything is written and re-raised by the caller after the C call (_raise_callback_error()). + # postfilter cdef int general_postfilter(blosc2_postfilter_params *params) with gil: cdef user_filters_udata *udata = params.user_data + if params.typesize != udata.itemsize: + raise ValueError(f"postfilter: data typesize {params.typesize} does not match " + f"dtype itemsize {udata.itemsize}") cdef int nd = 1 cdef np.npy_intp dims = params.size // params.typesize input = np.PyArray_SimpleNewFromData(nd, &dims, udata.input_cdtype, params.input) @@ -5018,6 +5032,9 @@ cdef int general_postfilter(blosc2_postfilter_params *params) with gil: # filler cdef int general_filler(blosc2_prefilter_params *params): cdef filler_udata *udata = params.user_data + if params.output_typesize != udata.itemsize: + raise ValueError(f"filler: data typesize {params.output_typesize} does not match " + f"dtype itemsize {udata.itemsize}") cdef int nd = 1 cdef np.npy_intp dims = params.output_size // params.output_typesize @@ -5666,6 +5683,10 @@ def nelem_from_inputs(inputs_tuple, nelem=None): # prefilter cdef int general_prefilter(blosc2_prefilter_params *params): cdef user_filters_udata *udata = params.user_data + # Typesize check: see general_postfilter + if params.output_typesize != udata.itemsize: + raise ValueError(f"prefilter: data typesize {params.output_typesize} does not match " + f"dtype itemsize {udata.itemsize}") cdef int nd = 1 cdef np.npy_intp dims = params.output_size // params.output_typesize diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 33452acae..ef3c0d945 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1538,7 +1538,8 @@ def postfilter(self, input_dtype: np.dtype, output_dtype: np.dtype = None) -> No * `nthreads` must be 1 when decompressing. * The :paramref:`input_dtype` itemsize must be the same as the - :paramref:`output_dtype` itemsize. + :paramref:`output_dtype` itemsize, and the typesize of the chunks: + decompressing a chunk with another typesize fails. See Also -------- @@ -1640,6 +1641,7 @@ def filler(self, inputs_tuple: tuple[tuple], schunk_dtype: np.dtype, nelem: int Notes ----- * Compression `nthreads` must be 1 when using this. + * The :paramref:`schunk_dtype` itemsize must be the typesize of the SChunk. * This does not need to be removed from the created `SChunk` instance. See Also @@ -1674,16 +1676,19 @@ def initialize(func): raise ValueError("Cannot apply a filler to a non empty SChunk") nelem_ = blosc2_ext.nelem_from_inputs(inputs_tuple, nelem) super(SChunk, self)._set_filler(func, id(inputs_tuple), schunk_dtype) - chunksize = self.chunksize - written_nbytes = 0 - nbytes = nelem_ * self.typesize - while written_nbytes < nbytes: - chunk = np.zeros(chunksize // self.typesize, dtype=schunk_dtype) - self.append_data(chunk) - written_nbytes += chunksize - if (nbytes - written_nbytes) < self.chunksize: - chunksize = nbytes - written_nbytes - self.remove_prefilter(func.__name__) + try: + chunksize = self.chunksize + written_nbytes = 0 + nbytes = nelem_ * self.typesize + while written_nbytes < nbytes: + chunk = np.zeros(chunksize // self.typesize, dtype=schunk_dtype) + self.append_data(chunk) + written_nbytes += chunksize + if (nbytes - written_nbytes) < self.chunksize: + chunksize = nbytes - written_nbytes + finally: + # Also after a failed append: the filler only keeps the id of inputs_tuple + self.remove_prefilter(func.__name__) def exec_func(*args): func(*args) @@ -1719,7 +1724,8 @@ def prefilter(self, input_dtype: np.dtype, output_dtype: np.dtype = None) -> Non * `nthreads` must be 1 when compressing. * The :paramref:`input_dtype` itemsize must be the same as the - :paramref:`output_dtype` itemsize. + :paramref:`output_dtype` itemsize, and the typesize of the SChunk: + compressing with another typesize fails. See Also -------- diff --git a/tests/test_postfilters.py b/tests/test_postfilters.py index 6ed66bbc2..677861cb0 100644 --- a/tests/test_postfilters.py +++ b/tests/test_postfilters.py @@ -86,3 +86,82 @@ def postf3(input, output, offset): assert np.array_equal(data, res) blosc2.remove_urlpath(urlpath) + + +# The postfilters below only record their calls and never touch their arrays, so a call with +# arrays that do not fit the buffers cannot overrun them, with or without the typesize guard. +# Each test runs with the GIL kept and released around the C calls. + + +@pytest.fixture(params=[False, True], ids=["gil-held", "gil-released"]) +def releasegil(request): + old = blosc2.set_releasegil(request.param) + yield + blosc2.set_releasegil(old) + + +@pytest.mark.parametrize("typesize", [1, 8]) +def test_postfilter_wrong_typesize(releasegil, typesize): + # int32 arrays of size // typesize items would overrun a typesize 1 buffer, and cover only + # half of a typesize 8 one: the postfilter refuses such chunks + data = np.arange(1000, dtype=np.int32) + schunk = blosc2.SChunk( + chunksize=data.nbytes, + data=data, + cparams={"typesize": typesize, "nthreads": 1}, + dparams={"nthreads": 1}, + ) + calls = [] + + @schunk.postfilter(np.int32) + def postf_wrong_typesize(input, output, offset): + calls.append(input.size) + + with pytest.raises( + ValueError, match=f"postfilter: data typesize {typesize} does not match dtype itemsize 4" + ): + schunk.decompress_chunk(0) + assert calls == [] + + schunk.remove_postfilter("postf_wrong_typesize") + assert np.array_equal(np.frombuffer(schunk.decompress_chunk(0), dtype=np.int32), data) + + +def test_postfilter_vlmeta(releasegil): + # The postfilter is for chunk data, not for the msgpack content (typesize 1) of a vlmetalayer + data = np.arange(2000, dtype=np.int32) + schunk = blosc2.SChunk( + chunksize=4000, + data=data, + contiguous=False, + cparams={"typesize": 4, "nthreads": 1}, + dparams={"nthreads": 1}, + ) + schunk.vlmeta["info"] = {"answer": 42} + calls = [] + + @schunk.postfilter(np.int32) + def postf_vlmeta(input, output, offset): + calls.append(input.size) + + # C-Blosc2 before Blosc/c-blosc2#857 decompresses vlmetalayers with the postfilter, which then + # refuses them; with it the content is read without the postfilter. Either way the callback + # never sees the content. + refused = "postfilter: data typesize 1 does not match dtype itemsize 4" + try: + info = schunk.vlmeta["info"] + except ValueError as e: + info = str(e) + assert info in ({"answer": 42}, refused) + assert calls == [] + + # Serializing an in-memory sparse super-chunk copies its vlmetalayers the same way + try: + copy = blosc2.schunk_from_cframe(schunk.to_cframe(), copy=True) + info = copy.vlmeta["info"] + except ValueError as e: + info = str(e) + assert info in ({"answer": 42}, refused) + + schunk.remove_postfilter("postf_vlmeta") + assert schunk.vlmeta["info"] == {"answer": 42} diff --git a/tests/test_prefilters.py b/tests/test_prefilters.py index d2da4365f..3fb77274a 100644 --- a/tests/test_prefilters.py +++ b/tests/test_prefilters.py @@ -207,3 +207,61 @@ def pref3(input, output, offset): assert np.array_equal(new_data, res) blosc2.remove_urlpath(urlpath) + + +# The prefilters below only record their calls and never touch their arrays, so a call with +# arrays that do not fit the buffers cannot overrun them, with or without the typesize guard. +# Each test runs with the GIL kept and released around the C calls. + + +@pytest.fixture(params=[False, True], ids=["gil-held", "gil-released"]) +def releasegil(request): + old = blosc2.set_releasegil(request.param) + yield + blosc2.set_releasegil(old) + + +@pytest.mark.parametrize("typesize", [1, 8]) +def test_prefilter_wrong_typesize(releasegil, typesize): + # int32 arrays of size // typesize items would overrun a typesize 1 buffer, and cover only + # half of a typesize 8 one: the prefilter refuses such chunks + data = np.arange(1000, dtype=np.int32) + schunk = blosc2.SChunk(chunksize=data.nbytes, cparams={"typesize": typesize, "nthreads": 1}) + calls = [] + + @schunk.prefilter(np.int32) + def pref_wrong_typesize(input, output, offset): + calls.append(input.size) + + with pytest.raises( + ValueError, match=f"prefilter: data typesize {typesize} does not match dtype itemsize 4" + ): + schunk.append_data(data) + assert calls == [] + assert schunk.nchunks == 0 + + schunk.remove_prefilter("pref_wrong_typesize") + schunk.append_data(data) + assert np.array_equal(np.frombuffer(schunk.decompress_chunk(0), dtype=np.int32), data) + + +@pytest.mark.parametrize("typesize", [1, 8]) +def test_filler_wrong_typesize(releasegil, typesize): + # Same for a filler, which then must not stay set: it keeps only the id of its inputs + data = np.arange(1000, dtype=np.int32) + schunk = blosc2.SChunk(chunksize=data.nbytes, cparams={"typesize": typesize, "nthreads": 1}) + calls = [] + + with pytest.raises( + ValueError, match=f"filler: data typesize {typesize} does not match dtype itemsize 4" + ): + + @schunk.filler(((data, np.int32),), np.int32) + def fill_wrong_typesize(inputs_tuple, output, offset): + calls.append(output.size) + + assert calls == [] + assert schunk.nchunks == 0 + assert "fill_wrong_typesize" not in blosc2.prefilter_funcs + schunk.append_data(data) + assert schunk.nchunks == 1