diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index fc7cf36b0..8d20705b5 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -515,7 +515,11 @@ cdef extern from "blosc2.h": int64_t blosc2_schunk_fill_special(blosc2_schunk *schunk, int64_t nitems, int special_value, int32_t chunksize); - int64_t blosc2_schunk_append_buffer(blosc2_schunk *schunk, void *src, int32_t nbytes) + int64_t blosc2_schunk_append_buffer(blosc2_schunk *schunk, const void *src, int32_t nbytes) nogil + int64_t blosc2_schunk_insert_buffer(blosc2_schunk *schunk, int64_t nchunk, const void *src, + int32_t nbytes) nogil + int64_t blosc2_schunk_update_buffer(blosc2_schunk *schunk, int64_t nchunk, const void *src, + int32_t nbytes) nogil int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int64_t nchunk, void *dest, int32_t nbytes) int blosc2_schunk_get_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t ** chunk, @@ -1399,6 +1403,55 @@ cdef int _raise_callback_error() except -1: SystemError.""" return -1 if PyErr_Occurred() != NULL else 0 + +cdef enum _write_op: + _WRITE_APPEND + _WRITE_INSERT + _WRITE_UPDATE + + +cdef inline int64_t _store_buffer_nogil(blosc2_schunk *schunk, _write_op op, int64_t nchunk, + const void *src, int32_t nbytes) noexcept nogil: + if op == _WRITE_APPEND: + return blosc2_schunk_append_buffer(schunk, src, nbytes) + if op == _WRITE_INSERT: + return blosc2_schunk_insert_buffer(schunk, nchunk, src, nbytes) + return blosc2_schunk_update_buffer(schunk, nchunk, src, nbytes) + + +cdef int64_t _store_buffer(blosc2_schunk *schunk, _write_op op, int64_t nchunk, + const void *src, Py_ssize_t nbytes) except? -1: + """Compress `nbytes` at `src` with the super-chunk's context and append it (`nchunk` is + ignored then), insert it at `nchunk` or replace chunk `nchunk` with it; c-blosc2 reuses its + compression buffer for frames. Returns the number of chunks, or a negative c-blosc2 error + code.""" + cdef int64_t rc + if nbytes > BLOSC2_MAX_BUFFERSIZE: + raise ValueError(f"Cannot compress more than {BLOSC2_MAX_BUFFERSIZE} bytes into one chunk") + if _can_compress_without_gil(schunk.cctx): + with nogil: + rc = _store_buffer_nogil(schunk, op, nchunk, src, nbytes) + else: + rc = _store_buffer_nogil(schunk, op, nchunk, src, nbytes) + return rc + + +cdef int64_t _store_data(blosc2_schunk *schunk, _write_op op, int64_t nchunk, data, + str error) except -1: + """_store_buffer() for an object exposing the buffer protocol; raises RuntimeError(error) + if c-blosc2 fails.""" + cdef Py_buffer buf + cdef int64_t rc + PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE) + try: + rc = _store_buffer(schunk, op, nchunk, buf.buf, buf.len) + finally: + PyBuffer_Release(&buf) + if rc < 0: + _raise_callback_error() + raise RuntimeError(error) + return rc + def get_blocksize(): return blosc1_get_blocksize() @@ -1995,14 +2048,10 @@ cdef class SChunk: if chunksize > INT_MAX: raise ValueError("Maximum chunksize allowed is 2^31 - 1") self.schunk.chunksize = chunksize - cdef const uint8_t[:] typed_view cdef int64_t index cdef Py_buffer buf cdef uint8_t *buf_ptr - cdef int comp_size - cdef int32_t csize - cdef uint8_t* chunk - cdef int32_t len_chunk + cdef int64_t len_chunk if data is not None: PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE) buf_ptr = buf.buf @@ -2013,26 +2062,11 @@ cdef class SChunk: if i == (nchunks - 1): len_chunk = len_data - i * chunksize index = i * chunksize - csize = (len_chunk + BLOSC2_MAX_OVERHEAD) - chunk = malloc(csize) - self.schunk.current_nchunk = i - comp_size = _compress_ctx(self.schunk.cctx, buf_ptr + index, len_chunk, chunk, csize) - if comp_size < 0: - free(chunk) + rc = _store_buffer(self.schunk, _WRITE_APPEND, i, buf_ptr + index, len_chunk) + if rc < 0: PyBuffer_Release(&buf) _raise_callback_error() - raise RuntimeError("Could not compress the data") - elif comp_size == 0: - free(chunk) - PyBuffer_Release(&buf) - raise RuntimeError("The result could not fit") - chunk = realloc(chunk, comp_size) - _check_comp_length('chunk', comp_size) - nchunks_ = blosc2_schunk_append_chunk(self.schunk, chunk, False) - if nchunks_ != (i + 1): - free(chunk) - PyBuffer_Release(&buf) - raise RuntimeError("An error occurred while appending the chunks") + raise RuntimeError("Could not compress and append the chunk") PyBuffer_Release(&buf) @property @@ -2180,28 +2214,8 @@ cdef class SChunk: @_handle_locked def append_data(self, data): - cdef Py_buffer buf - PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE) - cdef int size - cdef int32_t len_chunk = (buf.len + BLOSC2_MAX_OVERHEAD) - cdef uint8_t* chunk = malloc(len_chunk) - self.schunk.current_nchunk = self.schunk.nchunks - size = _compress_ctx(self.schunk.cctx, buf.buf, buf.len, chunk, len_chunk) - PyBuffer_Release(&buf) - if size < 0: - free(chunk) - _raise_callback_error() - raise RuntimeError("Could not compress the data") - elif size == 0: - free(chunk) - raise RuntimeError("The result could not fit") - chunk = realloc(chunk, size) - _check_comp_length('chunk', size) - rc = blosc2_schunk_append_chunk(self.schunk, chunk, False) - if rc < 0: - free(chunk) - raise RuntimeError("Could not append the chunk") - return rc + return _store_data(self.schunk, _WRITE_APPEND, self.schunk.nchunks, data, + "Could not compress and append the chunk") @_handle_locked def fill_special(self, nitems, special_value, value): @@ -2409,32 +2423,9 @@ cdef class SChunk: return rc @_handle_locked - def insert_data(self, nchunk, data, copy): - cdef blosc2_context *cctx - cdef Py_buffer buf - PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE) - cdef int size - cdef int32_t len_chunk = (buf.len + BLOSC2_MAX_OVERHEAD) - cdef uint8_t* chunk = malloc(len_chunk) - self.schunk.current_nchunk = nchunk # prefilter needs this value to be set - size = _compress_ctx(self.schunk.cctx, buf.buf, buf.len, chunk, len_chunk) - PyBuffer_Release(&buf) - if size <= 0: - free(chunk) - if size < 0: - _raise_callback_error() - raise RuntimeError("Could not compress the data") - raise RuntimeError("The result could not fit ") - - chunk = realloc(chunk, size) - _check_comp_length('chunk', size) - rc = blosc2_schunk_insert_chunk(self.schunk, nchunk, chunk, copy) - # Without a copy, the schunk owns the chunk only if the insert succeeded - if copy or rc < 0: - free(chunk) - if rc < 0: - raise RuntimeError("Could not insert the desired chunk") - return rc + def insert_data(self, int64_t nchunk, data, copy=False): + return _store_data(self.schunk, _WRITE_INSERT, nchunk, data, + "Could not compress and insert the chunk") @_handle_locked def update_chunk(self, nchunk, chunk): @@ -2456,32 +2447,9 @@ cdef class SChunk: return None @_handle_locked - def update_data(self, nchunk, data, copy): - cdef Py_buffer buf - PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE) - cdef int size - cdef int32_t len_chunk = (buf.len + BLOSC2_MAX_OVERHEAD) - cdef uint8_t* chunk = malloc(len_chunk) - self.schunk.current_nchunk = nchunk # prefilter needs this value to be set - size = _compress_ctx(self.schunk.cctx, buf.buf, buf.len, chunk, len_chunk) - - PyBuffer_Release(&buf) - if size <= 0: - free(chunk) - if size < 0: - _raise_callback_error() - raise RuntimeError("Could not compress the data") - raise RuntimeError("The result could not fit ") - - chunk = realloc(chunk, size) - _check_comp_length('chunk', size) - rc = blosc2_schunk_update_chunk(self.schunk, nchunk, chunk, copy) - # Without a copy, the schunk owns the chunk only if the update succeeded - if copy or rc < 0: - free(chunk) - if rc < 0: - raise RuntimeError("Could not update the desired chunk") - return rc + def update_data(self, int64_t nchunk, data, copy=False): + return _store_data(self.schunk, _WRITE_UPDATE, nchunk, data, + "Could not compress and update the chunk") # This is used internally for prefiltering @_handle_locked @@ -2539,93 +2507,64 @@ cdef class SChunk: cdef int64_t nbytes_copy = min(nbytes, buf.len - buf_pos) cdef int64_t data_start cdef uint8_t *data - cdef uint8_t *chunk - cdef int32_t alloc_len cdef int32_t chunk_nbytes cdef int32_t chunksize - cdef int comp_rc - if buf.len < nbytes: - raise ValueError("Not enough data for writing the slice") - - if stop > nitems: - # Increase SChunk's size - if start < nitems: - rc = blosc2_schunk_set_slice_buffer(self.schunk, start, nitems, buf.buf) - if rc < 0: - PyBuffer_Release(&buf) - _raise_callback_error() - raise RuntimeError("Error while setting the slice") - buf_pos = (nitems - start) * self.schunk.typesize - if self.schunk.nbytes % self.schunk.chunksize != 0: - # Update last chunk before appending any other - if stop * self.schunk.typesize >= self.schunk.chunksize * self.schunk.nchunks: - chunk_nbytes = self.schunk.chunksize - nbytes_copy = min(nbytes_copy, self.schunk.chunksize * self.schunk.nchunks - nitems * self.schunk.typesize) - else: - chunk_nbytes = (stop * self.schunk.typesize) % self.schunk.chunksize - data = malloc(chunk_nbytes) - rc = blosc2_schunk_decompress_chunk(self.schunk, self.schunk.nchunks - 1, data, chunk_nbytes) - if rc < 0: - free(data) - PyBuffer_Release(&buf) - _raise_callback_error() - raise RuntimeError("Error while decompressing the chunk") - data_start = self.schunk.nbytes - (self.schunk.nchunks - 1) * self.schunk.chunksize - memcpy(data + data_start, buf_ptr + buf_pos, nbytes_copy) - chunk = malloc(chunk_nbytes + BLOSC2_MAX_OVERHEAD) - self.schunk.current_nchunk = self.schunk.nchunks - 1 - comp_rc = _compress_ctx(self.schunk.cctx, data, chunk_nbytes, chunk, - chunk_nbytes + BLOSC2_MAX_OVERHEAD) - free(data) - if comp_rc < 0: - free(chunk) - PyBuffer_Release(&buf) - _raise_callback_error() - raise RuntimeError("Error while compressing the data") - elif comp_rc == 0: - free(chunk) - PyBuffer_Release(&buf) - raise RuntimeError("The result could not fit") - rc = blosc2_schunk_update_chunk(self.schunk, self.schunk.nchunks - 1, chunk, True) - free(chunk) - if rc < 0: - PyBuffer_Release(&buf) - raise RuntimeError("Error while updating the chunk") - buf_pos += nbytes_copy - # Append data if needed - if buf_pos < buf.len: - nappends = int(stop * self.schunk.typesize / self.schunk.chunksize - self.schunk.nchunks) - if (stop * self.schunk.typesize) % self.schunk.chunksize != 0: - nappends += 1 - for i in range(nappends): - if (self.schunk.nchunks + 1) * self.schunk.chunksize <= stop * self.schunk.typesize: - chunksize = self.schunk.chunksize + cdef int64_t rc + try: + if buf.len < nbytes: + raise ValueError("Not enough data for writing the slice") + + if stop > nitems: + # Increase SChunk's size + if start < nitems: + rc = blosc2_schunk_set_slice_buffer(self.schunk, start, nitems, buf.buf) + if rc < 0: + _raise_callback_error() + raise RuntimeError("Error while setting the slice") + buf_pos = (nitems - start) * self.schunk.typesize + if self.schunk.nbytes % self.schunk.chunksize != 0: + # Update last chunk before appending any other + if stop * self.schunk.typesize >= self.schunk.chunksize * self.schunk.nchunks: + chunk_nbytes = self.schunk.chunksize + nbytes_copy = min(nbytes_copy, self.schunk.chunksize * self.schunk.nchunks - nitems * self.schunk.typesize) else: - chunksize = (stop * self.schunk.typesize) % self.schunk.chunksize - alloc_len = (chunksize + BLOSC2_MAX_OVERHEAD) - chunk = malloc(alloc_len) - self.schunk.current_nchunk = self.schunk.nchunks - comp_rc = _compress_ctx(self.schunk.cctx, buf_ptr + buf_pos, chunksize, chunk, alloc_len) - if comp_rc < 0: - free(chunk) - PyBuffer_Release(&buf) + chunk_nbytes = (stop * self.schunk.typesize) % self.schunk.chunksize + data = malloc(chunk_nbytes) + rc = blosc2_schunk_decompress_chunk(self.schunk, self.schunk.nchunks - 1, data, chunk_nbytes) + if rc < 0: + free(data) _raise_callback_error() - raise RuntimeError("Error while compressing the chunk") - elif comp_rc == 0: - free(chunk) - PyBuffer_Release(&buf) - raise RuntimeError("The result could not fit") - chunk = realloc(chunk, comp_rc) - _check_comp_length('chunk', comp_rc) - rc = blosc2_schunk_append_chunk(self.schunk, chunk, False) + raise RuntimeError("Error while decompressing the chunk") + data_start = self.schunk.nbytes - (self.schunk.nchunks - 1) * self.schunk.chunksize + memcpy(data + data_start, buf_ptr + buf_pos, nbytes_copy) + try: + rc = _store_buffer(self.schunk, _WRITE_UPDATE, self.schunk.nchunks - 1, data, chunk_nbytes) + finally: + free(data) if rc < 0: - free(chunk) - PyBuffer_Release(&buf) - raise RuntimeError("Error while appending the chunk") - buf_pos += chunksize - else: - rc = blosc2_schunk_set_slice_buffer(self.schunk, start, stop, buf.buf) - PyBuffer_Release(&buf) + _raise_callback_error() + raise RuntimeError("Error while compressing and updating the chunk") + buf_pos += nbytes_copy + # Append data if needed + if buf_pos < buf.len: + nappends = int(stop * self.schunk.typesize / self.schunk.chunksize - self.schunk.nchunks) + if (stop * self.schunk.typesize) % self.schunk.chunksize != 0: + nappends += 1 + for i in range(nappends): + if (self.schunk.nchunks + 1) * self.schunk.chunksize <= stop * self.schunk.typesize: + chunksize = self.schunk.chunksize + else: + chunksize = (stop * self.schunk.typesize) % self.schunk.chunksize + rc = _store_buffer(self.schunk, _WRITE_APPEND, self.schunk.nchunks, buf_ptr + buf_pos, + chunksize) + if rc < 0: + _raise_callback_error() + raise RuntimeError("Error while compressing and appending the chunk") + buf_pos += chunksize + else: + rc = blosc2_schunk_set_slice_buffer(self.schunk, start, stop, buf.buf) + finally: + PyBuffer_Release(&buf) _check_rc(rc, "Error while setting the slice") @_handle_locked diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 5116f708c..96c2cbae6 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1027,7 +1027,8 @@ def insert_data(self, nchunk: int, data: object, copy: bool) -> int: data: bytes object The data that will be compressed and inserted as a chunk. copy: bool - Whether to make an internal copy of the chunk to insert it or not. + Kept for backward compatibility; it has no effect. The chunk compressed + from :paramref:`data` stays internal, so it never needs a copy. Returns ------- @@ -1161,7 +1162,8 @@ def update_data(self, nchunk: int, data: object, copy: bool) -> int: data: bytes object The data to be compressed and will replace the old chunk. copy: bool - Whether to make an internal copy of the chunk before updating it. + Kept for backward compatibility; it has no effect. The chunk compressed + from :paramref:`data` stays internal, so it never needs a copy. Returns ------- @@ -3003,15 +3005,10 @@ def open( # noqa: C901 container must enable it) and cannot be combined with `mmap_mode`. The ``BLOSC_LOCKING`` environment variable enables it globally. For more info, see :class:`blosc2.Storage`. - cparams: :class:`blosc2.CParams` or dict - The compression parameters for new and updated chunks, which are the same that can - be used in the :func:`~blosc2.compress2` function. Except for a super-chunk opened in - mode 'w' (which starts empty), they update the stored ones: a dict changes only the - parameters it has (new filters without filters_meta get zero metas), a - :class:`blosc2.CParams` all of them; nthreads defaults to :attr:`blosc2.nthreads`. - Typesize and blocksize cannot be changed (they fix the layout of the stored data): other - values raise a ValueError, while the defaults of a :class:`blosc2.CParams` (typesize 8, - blocksize 0) are ignored. + cparams: dict + A dictionary with the compression parameters, which are the same that can be + used in the :func:`~blosc2.compress2` function. + Typesize and blocksize cannot be changed. dparams: dict A dictionary with the decompression parameters, which are the same that can be used in the :func:`~blosc2.decompress2` function. diff --git a/tests/test_schunk_data_writes.py b/tests/test_schunk_data_writes.py new file mode 100644 index 000000000..887689daf --- /dev/null +++ b/tests/test_schunk_data_writes.py @@ -0,0 +1,177 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# append_data(), insert_data(), update_data(), SChunk(data=...) and set_slice() compress through +# c-blosc2's blosc2_schunk_{append,insert,update}_buffer(), which write frames from a compression +# buffer reused by the calling thread. + +import threading + +import numpy as np +import pytest + +import blosc2 + +CHUNK_LEN = 20_000 +STORAGES = [ + {"contiguous": False}, + {"contiguous": True}, + {"contiguous": True, "urlpath": "data_writes.b2frame"}, + {"contiguous": False, "urlpath": "data_writes_s.b2frame"}, +] + + +@pytest.fixture(params=STORAGES, ids=["memory", "cframe", "disk-cframe", "disk-sframe"]) +def storage(request): + blosc2.remove_urlpath(request.param.get("urlpath")) + yield request.param + blosc2.remove_urlpath(request.param.get("urlpath")) + + +@pytest.fixture(params=[False, True], ids=["gil", "nogil"]) +def releasegil(request): + old = blosc2.set_releasegil(request.param) + yield request.param + blosc2.set_releasegil(old) + + +def new_schunk(storage, nthreads=2, **kwargs): + return blosc2.SChunk( + chunksize=CHUNK_LEN * 4, + cparams={"typesize": 4, "nthreads": nthreads}, + dparams={"nthreads": nthreads}, + **storage, + **kwargs, + ) + + +def chunks_of(schunk): + return [np.frombuffer(schunk.decompress_chunk(i), dtype=np.int32) for i in range(schunk.nchunks)] + + +def test_data_writes(storage, releasegil): + schunk = new_schunk(storage) + expected = [] + for i in range(4): + expected.append(np.arange(CHUNK_LEN, dtype=np.int32) * (i + 1)) + assert schunk.append_data(expected[-1]) == i + 1 + expected.insert(2, np.full(CHUNK_LEN, -3, dtype=np.int32)) + assert schunk.insert_data(2, expected[2], copy=False) == 5 + expected.insert(5, np.full(CHUNK_LEN, -5, dtype=np.int32)) + assert schunk.insert_data(5, expected[5], copy=True) == 6 + # Several rounds of updates, so that frames get the reused buffer with other contents each time + for rnd in range(3): + for n in reversed(range(len(expected))): + expected[n] = np.arange(CHUNK_LEN, dtype=np.int32) + 1000 * n + 100_000 * rnd + assert schunk.update_data(n, expected[n], copy=bool(rnd % 2)) == len(expected) + for got, exp in zip(chunks_of(schunk), expected, strict=True): + np.testing.assert_array_equal(got, exp) + assert schunk.nbytes == len(expected) * CHUNK_LEN * 4 + if "urlpath" in storage: + reopened = blosc2.open(storage["urlpath"]) + for got, exp in zip(chunks_of(reopened), expected, strict=True): + np.testing.assert_array_equal(got, exp) + + +def test_data_writes_reach_prefilter(storage): + # The prefilter computes each item from its position, so it must see the right chunk number. + # With the GIL held only: Python prefilters crash with set_releasegil(True). + schunk = new_schunk(storage, nthreads=1) + + @schunk.prefilter(np.int32, np.int32) + def from_position(input, output, offset): + output[:] = offset + np.arange(output.size, dtype=np.int32) + + zeros = np.zeros(CHUNK_LEN, dtype=np.int32) + for _ in range(3): + schunk.append_data(zeros) + schunk.update_data(2, zeros, copy=False) + schunk.insert_data(1, zeros, copy=False) + schunk.remove_prefilter("from_position") + # The chunks written at positions 0, 1 (inserted), 1 (moved up) and 2 (updated) + for got, pos in zip(chunks_of(schunk), [0, 1, 1, 2], strict=True): + np.testing.assert_array_equal(got, np.arange(pos * CHUNK_LEN, (pos + 1) * CHUNK_LEN)) + + +def test_schunk_data_and_set_slice(storage, releasegil): + data = np.arange(3 * CHUNK_LEN + 123, dtype=np.int32) + schunk = new_schunk(storage, data=data) + assert schunk.nchunks == 4 + # Grows the partial last chunk, then appends whole and partial chunks + more = -np.arange(2 * CHUNK_LEN + 7, dtype=np.int32) + schunk[data.size : data.size + more.size] = more + expected = np.concatenate([data, more]) + out = np.empty_like(expected) + schunk.get_slice(0, expected.size, out=out) + np.testing.assert_array_equal(out, expected) + + +def test_failed_data_writes_change_nothing(storage): + schunk = new_schunk(storage, data=np.arange(2 * CHUNK_LEN, dtype=np.int32)) + before = chunks_of(schunk) + buffer = np.ones(CHUNK_LEN, dtype=np.int32) + with pytest.raises(RuntimeError): + schunk.update_data(2, buffer, copy=False) + with pytest.raises(RuntimeError): + schunk.insert_data(3, buffer, copy=False) + assert schunk.nchunks == 2 + assert schunk.nbytes == 2 * CHUNK_LEN * 4 + for got, exp in zip(chunks_of(schunk), before, strict=True): + np.testing.assert_array_equal(got, exp) + + +@pytest.mark.parametrize("locking", [False, True]) +def test_insert_after_another_handle_appended(tmp_path, locking): + # The position is checked against the file, not this handle's possibly stale view of it + urlpath = str(tmp_path / "two_handles.b2frame") + a = new_schunk({"contiguous": True, "urlpath": urlpath, "mode": "w", "locking": locking}) + a.append_data(np.zeros(CHUNK_LEN, dtype=np.int32)) + b = blosc2.open(urlpath, mode="a", locking=locking) + assert b.nchunks == 1 + a.append_data(np.ones(CHUNK_LEN, dtype=np.int32)) + assert b.insert_data(2, np.full(CHUNK_LEN, 7, dtype=np.int32), copy=False) == 3 + reopened = blosc2.open(urlpath) + for got, value in zip(chunks_of(reopened), [0, 1, 7], strict=True): + np.testing.assert_array_equal(got, np.full(CHUNK_LEN, value, dtype=np.int32)) + + +@pytest.mark.skipif(blosc2.IS_WASM, reason="no threads on wasm32") +def test_data_writes_from_threads(tmp_path): + # Each thread compresses into its own reused buffer; frames in memory and on disk + old = blosc2.set_releasegil(True) + try: + nthreads = 4 + schunks = [ + new_schunk( + {"contiguous": True, "urlpath": str(tmp_path / f"t{i}.b2frame")} + if i % 2 + else {"contiguous": True} + ) + for i in range(nthreads) + ] + errors = [] + + def write(i): + try: + for n in range(6): + schunks[i].append_data(np.full(CHUNK_LEN, 100 * i + n, dtype=np.int32)) + for n in range(6): + schunks[i].update_data(n, np.full(CHUNK_LEN, -(100 * i + n), dtype=np.int32), copy=False) + except Exception as e: # pragma: no cover - reported below + errors.append(e) + + threads = [threading.Thread(target=write, args=(i,)) for i in range(nthreads)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not errors + for i, schunk in enumerate(schunks): + for n, got in enumerate(chunks_of(schunk)): + np.testing.assert_array_equal(got, np.full(CHUNK_LEN, -(100 * i + n), dtype=np.int32)) + finally: + blosc2.set_releasegil(old)