From 31482777368fab796b14eb34bcd9d117ab6dbe7f Mon Sep 17 00:00:00 2001 From: August Vi Date: Sun, 18 Oct 2026 10:56:56 -0700 Subject: [PATCH] Add Disk.with_options(on_error='miss') so unreadable entries are deleted and treated as missing instead of raising. --- CHANGELOG.md | 1 + diskcache_community/core.py | 161 +++++++++- diskcache_community/persistent.py | 41 ++- docs/tutorial.rst | 35 +++ tests/test_entry_errors.py | 491 ++++++++++++++++++++++++++++++ 5 files changed, 720 insertions(+), 9 deletions(-) create mode 100644 tests/test_entry_errors.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e42cb47..27b2f92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ diskcache-community forked from DiskCache at upstream commit ebfa37c (3 March 20 ### Features +- `Disk.with_options(on_error='miss')` returns a disk type, on any disk class including `SafeDisk` and the classes `with_types` returns, whose caches delete an entry they find but cannot read back (a corrupt or stale pickle, a refused type, a path outside the cache, an undecodable text file) and treat it as missing: `get` returns its default, `cache[key]` raises `KeyError`, `pop`, `pull`, `peek` and `peekitem` move on, and iteration skips unreadable keys. `Index.items()` and `values()` skip such entries too. The first such entry per cache is logged as a warning on the `diskcache_community` logger; a read-only cache does not delete it, and its `peek` and `peekitem` raise. A missing value file, `MemoryError` and `RecursionError` are never treated as unreadable. The default, `on_error='raise'`, keeps raising as before, and the option is a class attribute that is never persisted (PR #25). - `configure(disk)` sets the disk type of every `Cache`, `FanoutCache`, `Deque`, `Index` and `DjangoCache` created afterwards without a disk, and `default_disk()` returns the type in effect. Without `configure`, the environment decides: `DISKCACHE_DISK=safe` selects `SafeDisk`, with `DISKCACHE_SAFE_TYPES` and `DISKCACHE_SAFE_MODULES` passed to `SafeDisk.with_types`, and `DISKCACHE_DISK` also takes `default` or the import path of a disk class. A disk passed explicitly always wins, so libraries that choose their own disk keep it; caches that already exist keep theirs, and nothing is persisted. The `disk` arguments now default to `None`, meaning that type (PR #24). ### Packaging diff --git a/diskcache_community/core.py b/diskcache_community/core.py index 101722b..72d0e03 100644 --- a/diskcache_community/core.py +++ b/diskcache_community/core.py @@ -13,6 +13,7 @@ import inspect import io import json +import logging import os import os.path as op import pathlib @@ -194,6 +195,8 @@ def _makedirs_private(directory): class Disk: """Cache key and value serialization for SQLite database and files.""" + on_error = 'raise' + def __init__(self, directory, min_file_size=0, pickle_protocol=0): """Initialize disk instance. @@ -206,6 +209,40 @@ def __init__(self, directory, min_file_size=0, pickle_protocol=0): self.min_file_size = min_file_size self.pickle_protocol = pickle_protocol + @classmethod + def with_options(cls, *, on_error): + """Return a subclass of this disk type with the given options. + + `on_error` says what a cache does with an entry that it finds but + cannot read back: a pickle that is corrupt, refers to a class that no + longer exists, or is refused by :class:`SafeDisk`, or a value file + that is not valid text. ``'raise'``, the default, raises the error. + ``'miss'`` deletes the entry and treats it as missing, so ``get`` + returns its default and ``cache[key]`` raises :exc:`KeyError`; the + first such entry of each cache is logged as a warning on the + ``diskcache_community`` logger. A read-only cache returns the default + from ``get`` without deleting the entry, while its ``peek`` and + ``peekitem`` raise, since they cannot move past it. ``len`` and + ``in`` count an unreadable entry until a read deletes it. A missing + value file, :exc:`MemoryError` and :exc:`RecursionError` are never + treated as unreadable entries. The option is a class attribute and + is never persisted. + + >>> MissDisk = Disk.with_options(on_error='miss') + >>> MissDisk.on_error + 'miss' + + :param str on_error: ``'raise'`` or ``'miss'`` + :return: subclass to pass as ``disk=`` + :raises ValueError: if `on_error` is not one of the two + + """ + if on_error not in ('raise', 'miss'): + raise ValueError( + "on_error must be 'raise' or 'miss', not %r" % (on_error,) + ) + return _derive_disk(cls, {'on_error': on_error}) + def hash(self, key): """Compute portable hash for `key`. @@ -990,6 +1027,8 @@ def remove(self, file_path): super().remove(file_path) +_LOGGER = logging.getLogger('diskcache_community') + ENV_DISK = 'DISKCACHE_DISK' ENV_SAFE_TYPES = 'DISKCACHE_SAFE_TYPES' ENV_SAFE_MODULES = 'DISKCACHE_SAFE_MODULES' @@ -1425,6 +1464,13 @@ def after_fork_child(self): # Every live Cache, so a child process can forget inherited connections. _CACHES: 'weakref.WeakSet[Cache]' = weakref.WeakSet() +# Caches that have logged an unreadable entry as a warning already. +_UNREADABLE_LOGGED: 'weakref.WeakSet[Cache]' = weakref.WeakSet() + +# Errors that on_error='miss' never treats as an unreadable entry: a missing +# value file is already a miss, and the others say nothing about the entry. +_NOT_UNREADABLE = (OSError, MemoryError, RecursionError) + # Connections inherited from a parent process, kept open on purpose. _INHERITED_CONNECTIONS: list[sqlite3.Connection] = [] @@ -2331,6 +2377,66 @@ def decr(self, key, delta=1, default=0, retry=False): """ return self.incr(key, -delta, default, retry) + def _unreadable(self, error): + """Return True when `error`, raised reading an entry, means a miss. + + That is when the disk's `on_error` is ``'miss'``. The first such + entry of the cache is logged as a warning and later ones at debug + level. + + """ + if getattr(self._disk, 'on_error', 'raise') != 'miss' or isinstance( + error, _NOT_UNREADABLE + ): + return False + + logged = self in _UNREADABLE_LOGGED + _UNREADABLE_LOGGED.add(self) + _LOGGER.log( + logging.DEBUG if logged else logging.WARNING, + 'treating an unreadable entry in %r as missing: %s: %s', + self._directory, + type(error).__name__, + error, + ) + return True + + def _remove_unreadable_file(self, filename): + """Remove the value file of an unreadable entry. + + A disk may refuse to remove the path, as SafeDisk does for a path + outside the cache directory, and that path is then left alone. + + """ + if filename is not None: + with cl.suppress(OSError, ValueError): + self._disk.remove(filename) + + def _drop_unreadable(self, rowid, mode, filename, db_value, retry): + """Delete the unreadable entry at `rowid` unless it has changed.""" + if self._read_only: + return + + with self._transact(retry) as (sql, _): + cursor = sql( + 'DELETE FROM Cache WHERE rowid = ? AND mode = ?' + ' AND filename IS ? AND value IS ?', + (rowid, mode, filename, db_value), + ) + deleted = cursor.rowcount + + if deleted: + self._remove_unreadable_file(filename) + + def _read_key(self, key, raw): + """Return the key of a row, or ENOVAL for one read as a miss.""" + try: + return self._disk.get(key, raw) + except Exception as error: # pylint: disable=broad-except + if not self._unreadable(error): + raise + return ENOVAL + def get( self, key, @@ -2407,6 +2513,11 @@ def get( except IOError: # Key was deleted before we could retrieve result. return default + except Exception as error: # pylint: disable=broad-except + if not self._unreadable(error): + raise + self._drop_unreadable(rowid, mode, filename, db_value, retry) + return default else: # Slow path, transaction required. cache_hit = ( @@ -2443,6 +2554,15 @@ def get( if statistics: sql(cache_miss) return default + except Exception as error: # pylint: disable=broad-except + if not self._unreadable(error): + raise + self._drop_unreadable( + rowid, mode, filename, db_value, retry + ) + if statistics: + sql(cache_miss) + return default if statistics: sql(cache_hit) @@ -2556,6 +2676,12 @@ def pop( except IOError: # Key was deleted before we could retrieve result. return default + except Exception as error: # pylint: disable=broad-except + if not self._unreadable(error): + raise + self._remove_unreadable_file(filename) + filename = None + return default finally: if filename is not None: self._disk.remove(filename) @@ -2828,6 +2954,12 @@ def pull( except IOError: # Key was deleted before we could retrieve result. continue + except Exception as error: # pylint: disable=broad-except + if not self._unreadable(error): + raise + self._remove_unreadable_file(name) + name = None + continue finally: if name is not None: self._disk.remove(name) @@ -2945,6 +3077,11 @@ def peek( except IOError: # Key was deleted before we could retrieve result. continue + except Exception as error: # pylint: disable=broad-except + if self._read_only or not self._unreadable(error): + raise + self._drop_unreadable(rowid, mode, name, db_value, retry) + continue break if expire_time and tag: @@ -3019,13 +3156,17 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): else: break - key = self._disk.get(db_key, raw) - try: + key = self._disk.get(db_key, raw) value = self._disk.fetch(mode, name, db_value, False) except IOError: # Key was deleted before we could retrieve result. continue + except Exception as error: # pylint: disable=broad-except + if self._read_only or not self._unreadable(error): + raise + self._drop_unreadable(rowid, mode, name, db_value, retry) + continue break if expire_time and tag: @@ -3485,7 +3626,7 @@ def iterkeys(self, reverse=False, tag=ENOVAL): """ sql = self._sql limit = 100 - _disk_get = self._disk.get + _read_key = self._read_key select = 'SELECT key, raw FROM Cache' args = () @@ -3513,7 +3654,9 @@ def iterkeys(self, reverse=False, tag=ENOVAL): else: return - yield _disk_get(key, raw) + result = _read_key(key, raw) + if result is not ENOVAL: + yield result while True: rows = sql(iterate, args + (key, raw, key, limit)).fetchall() @@ -3522,7 +3665,9 @@ def iterkeys(self, reverse=False, tag=ENOVAL): break for key, raw in rows: - yield _disk_get(key, raw) + result = _read_key(key, raw) + if result is not ENOVAL: + yield result def _iter(self, ascending=True): sql = self._sql @@ -3535,7 +3680,7 @@ def _iter(self, ascending=True): bound = max_rowid + 1 limit = 100 - _disk_get = self._disk.get + _read_key = self._read_key rowid = 0 if ascending else bound select = ( 'SELECT rowid, key, raw FROM Cache' @@ -3555,7 +3700,9 @@ def _iter(self, ascending=True): break for rowid, key, raw in rows: - yield _disk_get(key, raw) + result = _read_key(key, raw) + if result is not ENOVAL: + yield result def __iter__(self): """Iterate keys in cache including expired items.""" diff --git a/diskcache_community/persistent.py b/diskcache_community/persistent.py index 2bb477a..a2c5ca9 100644 --- a/diskcache_community/persistent.py +++ b/diskcache_community/persistent.py @@ -16,6 +16,43 @@ from .core import ENOVAL, Cache +class _IndexValuesView(ValuesView): + """Values view of an Index that skips items read as missing. + + An item can disappear between listing its key and reading its value: + another process deleted it, or the disk treats an unreadable value as + missing. + + """ + + def __iter__(self): + _cache = self._mapping._cache # pylint: disable=protected-access + for key in self._mapping: + value = _cache.get(key, ENOVAL, retry=True) + if value is not ENOVAL: + yield value + + def __contains__(self, value): + return any(item is value or item == value for item in self) + + +class _IndexItemsView(ItemsView): + """Items view of an Index that skips items read as missing.""" + + def __iter__(self): + _cache = self._mapping._cache # pylint: disable=protected-access + for key in self._mapping: + value = _cache.get(key, ENOVAL, retry=True) + if value is not ENOVAL: + yield key, value + + def __contains__(self, item): + key, value = item + _cache = self._mapping._cache # pylint: disable=protected-access + stored = _cache.get(key, ENOVAL, retry=True) + return stored is not ENOVAL and (stored is value or stored == value) + + def _make_compare(seq_op, doc): """Make compare method with Sequence semantics.""" @@ -1082,7 +1119,7 @@ def values(self): :return: values view """ - return ValuesView(self) + return _IndexValuesView(self) def items(self): """Set-like object providing a view of index items. @@ -1096,7 +1133,7 @@ def items(self): :return: items view """ - return ItemsView(self) + return _IndexItemsView(self) __hash__ = None # type: ignore diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 061b4c3..0627438 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1052,6 +1052,41 @@ wins, then :func:`configure `, then the environment, then :class:`Disk `. Caches that already exist keep their disk, and the choice is never stored in the cache directory. +.. _tutorial-unreadable-entries: + +Unreadable entries +.................. + +An entry can exist and still not be readable: its pickle is corrupt, refers to +a class that a library upgrade removed, or is refused by :class:`SafeDisk +`. By default reading it raises that error. +:meth:`Disk.with_options ` returns a +disk type that deletes such an entry and treats it as missing instead, which +suits a cache whose entries can always be computed again: + +.. code-block:: python + + from diskcache_community import Cache, SafeDisk + + AppDisk = SafeDisk.with_types(Point).with_options(on_error='miss') + cache = Cache('/tmp/mycache', disk=AppDisk) + +``get`` then returns its default, ``cache[key]`` raises :exc:`KeyError`, +``pop``, ``pull``, ``peek`` and ``peekitem`` move on to the next item, and +iteration, including the items and values of an :class:`Index +`, skips entries that cannot be read. ``len`` and +``in`` still count an unreadable entry until a read deletes it. The first +unreadable entry of each cache is logged as a warning on the +``diskcache_community`` logger. A read-only cache returns the default from +``get`` without deleting the entry, and its ``peek`` and ``peekitem`` raise, +since they cannot move past it. A value file that has disappeared is a miss +in either mode, as before. + +The entry is deleted for every process that shares the directory, so give +every reader and writer of a shared cache the same disk: a reader whose +allowlist is narrower than a writer's would delete entries the writer can +still read. + .. _tutorial-read-only: Read-Only Caches diff --git a/tests/test_entry_errors.py b/tests/test_entry_errors.py new file mode 100644 index 0000000..66f681d --- /dev/null +++ b/tests/test_entry_errors.py @@ -0,0 +1,491 @@ +"""Test Disk.with_options(on_error='miss').""" + +import logging +import os +import os.path as op +import pickle +import shutil +import sqlite3 +import tempfile + +import pytest + +import diskcache_community as dc +from diskcache_community.core import MODE_PICKLE, MODE_TEXT +from tests.safedisk_models import models + +pytestmark = pytest.mark.filterwarnings('ignore', category=dc.EmptyDirWarning) + +MissDisk = dc.Disk.with_options(on_error='miss') +CORRUPT = b'\x80\x05\x95garbage' +STALE = b'\x80\x04\x8c\x0cno_such_mod1\x8c\x03Cls\x93)\x81.' + + +def _execute(path): + with open(path, 'w', encoding='utf-8') as writer: + writer.write('executed') + return 'executed' + + +class Poison: + def __init__(self, path): + self.path = path + + def __reduce__(self): + return (_execute, (self.path,)) + + +@pytest.fixture +def directory(): + directory = tempfile.mkdtemp(prefix='diskcache-') + yield directory + shutil.rmtree(directory, ignore_errors=True) + + +def corrupt(cache, key, data=CORRUPT): + """Replace the value of `key` with the pickle bytes `data`, inline.""" + cache._sql( + 'UPDATE Cache SET mode = ?, value = ?, filename = NULL' + ' WHERE key = ? AND raw = 1', + (MODE_PICKLE, sqlite3.Binary(data), key), + ) + + +def count(cache): + ((rows,),) = cache._sql('SELECT COUNT(*) FROM Cache').fetchall() + return rows + + +def test_with_options_derives_a_subclass(): + assert issubclass(MissDisk, dc.Disk) + assert MissDisk.on_error == 'miss' + assert dc.Disk.on_error == 'raise' + assert 'on_error' in repr(MissDisk) + assert MissDisk.__bases__ == (dc.Disk,) + + +@pytest.mark.parametrize('value', ['ignore', None, True, 'MISS']) +def test_with_options_rejects_unknown_values(value): + with pytest.raises(ValueError, match='on_error'): + dc.Disk.with_options(on_error=value) + + +def test_with_options_combines_with_with_types(): + first = dc.SafeDisk.with_types(models.Point).with_options(on_error='miss') + second = dc.SafeDisk.with_options(on_error='miss').with_types(models.Pair) + + assert first.on_error == 'miss' + assert first.safe_types == frozenset({models.Point}) + assert second.on_error == 'miss' + assert second.safe_types == frozenset({models.Pair}) + assert first.__bases__ == second.__bases__ == (dc.SafeDisk,) + + +def test_default_still_raises(directory): + with dc.Cache(directory) as cache: + cache['key'] = {'a': 1} + corrupt(cache, 'key') + with pytest.raises(pickle.UnpicklingError): + cache.get('key') + assert count(cache) == 1 + + +@pytest.mark.parametrize('data', [CORRUPT, STALE]) +def test_get_treats_unreadable_value_as_missing(directory, data): + with dc.Cache(directory, disk=MissDisk) as cache: + cache['key'] = {'a': 1} + cache['other'] = {'b': 2} + corrupt(cache, 'key', data) + + assert cache.get('key', default='missing') == 'missing' + assert count(cache) == 1 + assert 'key' not in cache + with pytest.raises(KeyError): + cache['key'] # pylint: disable=pointless-statement + assert cache['other'] == {'b': 2} + + cache['key'] = {'a': 3} + assert cache['key'] == {'a': 3} + + +def test_get_metadata_default(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + cache.set('key', {'a': 1}, tag='t') + corrupt(cache, 'key') + assert cache.get('key', 'd', expire_time=True, tag=True) == ( + 'd', + None, + None, + ) + + +def test_unreadable_file_value(directory): + with dc.Cache(directory, disk=MissDisk, disk_min_file_size=0) as cache: + cache['key'] = {'a': 1} + ((filename,),) = cache._sql( + 'SELECT filename FROM Cache WHERE key = ?', ('key',) + ).fetchall() + full_path = op.join(directory, filename) + with open(full_path, 'wb') as writer: + writer.write(CORRUPT) + + assert cache.get('key') is None + assert count(cache) == 0 + assert not op.exists(full_path) + + +def test_unreadable_text_file(directory): + with dc.Cache(directory, disk=MissDisk, disk_min_file_size=0) as cache: + cache['key'] = 'text' + ((mode, filename),) = cache._sql( + 'SELECT mode, filename FROM Cache WHERE key = ?', ('key',) + ).fetchall() + assert mode == MODE_TEXT + with open(op.join(directory, filename), 'wb') as writer: + writer.write(b'\xff\xfe\xfa') + + assert cache.get('key') is None + assert count(cache) == 0 + + +def test_missing_file_unchanged(directory): + with dc.Cache(directory, disk=MissDisk, disk_min_file_size=0) as cache: + cache['key'] = {'a': 1} + ((filename,),) = cache._sql( + 'SELECT filename FROM Cache WHERE key = ?', ('key',) + ).fetchall() + os.remove(op.join(directory, filename)) + + assert cache.get('key') is None + assert count(cache) == 1 + + +def test_statistics_and_eviction_policy_paths(directory): + with dc.Cache( + directory, + disk=MissDisk, + statistics=True, + eviction_policy='least-recently-used', + ) as cache: + cache['key'] = {'a': 1} + corrupt(cache, 'key') + + assert cache.get('key') is None + assert cache.stats() == (0, 1) + assert count(cache) == 0 + + +def test_safedisk_refusal_is_a_miss(directory): + path = op.join(directory, 'marker') + disk = dc.SafeDisk.with_types(models.Point).with_options(on_error='miss') + + with dc.Cache(directory) as cache: + cache['shape'] = models.Shape('line', []) + cache['point'] = models.Point(1, 2) + cache['poison'] = 'value' + corrupt(cache, 'poison', pickle.dumps(Poison(path))) + + with dc.Cache(directory, disk=disk) as cache: + assert cache.get('shape') is None + assert cache.get('poison') is None + assert cache['point'] == models.Point(1, 2) + assert count(cache) == 1 + + assert not op.exists(path) + + +def test_safedisk_path_escape_is_a_miss_and_keeps_the_file(directory): + outside = op.join(directory, 'outside.val') + with open(outside, 'wb') as writer: + writer.write(pickle.dumps({'secret': 1})) + + cache_dir = op.join(directory, 'cache') + disk = dc.SafeDisk.with_options(on_error='miss') + + with dc.Cache(cache_dir) as cache: + cache['key'] = {'a': 1} + cache._sql( + 'UPDATE Cache SET mode = ?, value = NULL, filename = ?' + ' WHERE key = ?', + (MODE_PICKLE, outside, 'key'), + ) + + with dc.Cache(cache_dir, disk=disk) as cache: + assert cache.get('key') is None + assert count(cache) == 0 + + assert op.exists(outside) + + +def test_pop(directory): + with dc.Cache(directory, disk=MissDisk, disk_min_file_size=0) as cache: + cache['key'] = {'a': 1} + ((filename,),) = cache._sql( + 'SELECT filename FROM Cache WHERE key = ?', ('key',) + ).fetchall() + with open(op.join(directory, filename), 'wb') as writer: + writer.write(CORRUPT) + + assert cache.pop('key', default='gone') == 'gone' + assert count(cache) == 0 + assert not op.exists(op.join(directory, filename)) + + +def test_pull_skips_unreadable_items(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + first = cache.push({'n': 1}) + cache.push({'n': 2}) + cache._sql( + 'UPDATE Cache SET value = ? WHERE key = ?', + (sqlite3.Binary(CORRUPT), first), + ) + + assert cache.pull()[1] == {'n': 2} + assert cache.pull() == (None, None) + assert count(cache) == 0 + + +def test_peek_skips_and_deletes_unreadable_items(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + first = cache.push({'n': 1}) + cache.push({'n': 2}) + cache._sql( + 'UPDATE Cache SET value = ? WHERE key = ?', + (sqlite3.Binary(CORRUPT), first), + ) + + assert cache.peek()[1] == {'n': 2} + assert count(cache) == 1 + + +def test_peekitem_skips_unreadable_values_and_keys(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + cache['first'] = {'n': 1} + cache[('tuple', 'key')] = {'n': 2} + cache['last'] = {'n': 3} + corrupt(cache, 'last') + + assert cache.peekitem() == (('tuple', 'key'), {'n': 2}) + cache._sql( + 'UPDATE Cache SET key = ? WHERE raw = 0', (sqlite3.Binary(STALE),) + ) + assert cache.peekitem() == ('first', {'n': 1}) + assert count(cache) == 1 + + +def test_iteration_skips_unreadable_keys(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + cache['a'] = 1 + cache[('b',)] = 2 + cache['c'] = 3 + cache._sql( + 'UPDATE Cache SET key = ? WHERE raw = 0', (sqlite3.Binary(STALE),) + ) + + assert list(cache) == ['a', 'c'] + assert list(reversed(cache)) == ['c', 'a'] + assert sorted(cache.iterkeys()) == ['a', 'c'] + assert len(cache) == 3 + + +def test_iteration_raises_by_default(directory): + with dc.Cache(directory) as cache: + cache[('b',)] = 2 + cache._sql( + 'UPDATE Cache SET key = ? WHERE raw = 0', (sqlite3.Binary(STALE),) + ) + with pytest.raises(ModuleNotFoundError): + list(cache) + + +def test_read_only_cache_does_not_delete(directory): + with dc.Cache(directory) as cache: + cache['key'] = {'a': 1} + cache.push({'n': 1}) + corrupt(cache, 'key') + cache._sql( + 'UPDATE Cache SET value = ? WHERE raw = 1 AND key != ?', + (sqlite3.Binary(CORRUPT), 'key'), + ) + + with dc.Cache(directory, disk=MissDisk, read_only=True) as cache: + assert cache.get('key') is None + assert count(cache) == 2 + with pytest.raises(pickle.UnpicklingError): + cache.peek() + with pytest.raises(pickle.UnpicklingError): + cache.peekitem() + + +def test_logged_once_as_warning(directory, caplog): + with dc.Cache(directory, disk=MissDisk) as cache: + cache['a'] = {'a': 1} + cache['b'] = {'b': 2} + corrupt(cache, 'a') + corrupt(cache, 'b') + + with caplog.at_level(logging.DEBUG, logger='diskcache_community'): + cache.get('a') + cache.get('b') + + levels = [record.levelno for record in caplog.records] + assert levels == [logging.WARNING, logging.DEBUG] + assert 'UnpicklingError' in caplog.records[0].getMessage() + + +def test_memoize_recomputes(directory): + calls = [] + + with dc.Cache(directory, disk=MissDisk) as cache: + + @cache.memoize() + def square(number): + calls.append(number) + return {'value': number * number} + + assert square(3) == {'value': 9} + cache._sql('UPDATE Cache SET value = ?', (sqlite3.Binary(CORRUPT),)) + assert square(3) == {'value': 9} + assert calls == [3, 3] + + +def test_containers(directory): + with dc.FanoutCache(op.join(directory, 'fanout'), disk=MissDisk) as f: + f['key'] = {'a': 1} + shard = f._shards[f._hash('key') % f._count] + corrupt(shard, 'key') + assert f.get('key', 'missing') == 'missing' + + index = dc.Index(op.join(directory, 'index'), disk=MissDisk) + index['key'] = {'a': 1} + corrupt(index.cache, 'key') + assert index.get('key') is None + index.cache.close() + + +def test_django(directory): + pytest.importorskip('django') + django_cache = dc.DjangoCache(directory, {'DISK': MissDisk, 'SHARDS': 1}) + try: + django_cache.set('key', {'a': 1}) + shard = django_cache._cache._shards[0] + ((key,),) = shard._sql('SELECT key FROM Cache').fetchall() + corrupt(shard, key) + assert django_cache.get('key', 'missing') == 'missing' + finally: + django_cache.close() + + +def test_custom_disk_errors_are_misses(directory): + class Refused(Exception): + pass + + class StrictDisk(dc.Disk): + def fetch(self, mode, filename, value, read): + if mode == MODE_PICKLE: + raise Refused('no pickles here') + return super().fetch(mode, filename, value, read) + + with dc.Cache(directory, disk=StrictDisk) as cache: + cache['key'] = {'a': 1} + with pytest.raises(Refused): + cache['key'] # pylint: disable=pointless-statement + + disk = StrictDisk.with_options(on_error='miss') + + with dc.Cache(directory, disk=disk) as cache: + assert cache.get('key') is None + assert count(cache) == 0 + + +def test_cache_with_options_pickles(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + copy = pickle.loads(pickle.dumps(cache)) + try: + assert type(copy.disk).on_error == 'miss' + finally: + copy.close() + + +def test_option_not_persisted(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + cache['key'] = 1 + names = {name for (name,) in cache._sql('SELECT key FROM Settings')} + + assert not any('error' in name for name in names) + + +def test_default_raises_on_every_path(directory): + with dc.Cache(directory, statistics=True) as cache: + cache['key'] = {'a': 1} + corrupt(cache, 'key') + with pytest.raises(pickle.UnpicklingError): + cache.get('key') + with pytest.raises(pickle.UnpicklingError): + cache.pop('key') + + with dc.Cache(directory + '-queue') as cache: + first = cache.push({'n': 1}) + cache._sql( + 'UPDATE Cache SET value = ? WHERE key = ?', + (sqlite3.Binary(CORRUPT), first), + ) + with pytest.raises(pickle.UnpicklingError): + cache.pull() + + +def test_eviction_policy_path_without_statistics(directory): + with dc.Cache( + directory, disk=MissDisk, eviction_policy='least-frequently-used' + ) as cache: + cache['key'] = {'a': 1} + corrupt(cache, 'key') + assert cache.get('key') is None + assert count(cache) == 0 + + +def test_changed_entry_is_not_deleted(directory): + with dc.Cache(directory, disk=MissDisk) as cache: + cache['key'] = {'a': 1} + ((rowid, mode, filename, value),) = cache._sql( + 'SELECT rowid, mode, filename, value FROM Cache' + ).fetchall() + + cache._drop_unreadable(rowid, mode, filename, b'something else', False) + + assert count(cache) == 1 + assert cache['key'] == {'a': 1} + + +def test_index_views_skip_unreadable_values(directory): + index = dc.Index(directory, disk=MissDisk) + index.update({'a': {'n': 1}, 'b': {'n': 2}, 'c': {'n': 3}}) + corrupt(index.cache, 'b') + + assert list(index.items()) == [('a', {'n': 1}), ('c', {'n': 3})] + assert ('a', {'n': 1}) in index.items() + assert ('b', {'n': 2}) not in index.items() + index['b'] = {'n': 2} + corrupt(index.cache, 'b') + assert list(index.values()) == [{'n': 1}, {'n': 3}] + assert {'n': 3} in index.values() + assert {'n': 2} not in index.values() + index.cache.close() + + +def test_memory_and_recursion_errors_are_not_misses(directory): + class FailingDisk(dc.Disk): + error = MemoryError + + def fetch(self, mode, filename, value, read): + raise self.error('not about the entry') + + for error in (MemoryError, RecursionError): + disk = type( + 'FailingDisk', (FailingDisk,), {'error': error} + ).with_options(on_error='miss') + with dc.Cache(op.join(directory, error.__name__), disk=disk) as c: + c['key'] = {'a': 1} + with pytest.raises(error): + c.get('key') + assert count(c) == 1