From 764297b9588f3ea8a4534978fb2f791836699c2a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 21 Oct 2026 22:50:32 +0300 Subject: [PATCH] gh-74876: Add copyreg.Proxy A copyreg.Proxy is pickled as the wrapped object, which must be strongly referenced elsewhere in the same pickle. It replaces the native support of weak references in pickle: weakref.ref, WeakMethod, WeakSet, WeakValueDictionary and WeakKeyDictionary are now reduced using Proxy. The copy module supports Proxy too. Co-Authored-By: Claude Opus 5.5 --- Doc/library/copyreg.rst | 24 +++++ Doc/library/weakref.rst | 10 ++- Doc/whatsnew/3.16.rst | 11 +++ Lib/_weakrefset.py | 17 ++-- Lib/copy.py | 8 +- Lib/copyreg.py | 47 +++++++++- Lib/pickle.py | 50 ++++------- Lib/test/pickletester.py | 28 ++++++ Lib/test/test_copy.py | 18 ++++ Lib/test/test_weakref.py | 58 ++++++++++--- Lib/test/test_weakset.py | 19 ++-- Lib/weakref.py | 79 ++++++----------- ...6-10-22-07-23-25.gh-issue-74876.fO5FoR.rst | 5 ++ Modules/_pickle.c | 87 +++++-------------- 14 files changed, 279 insertions(+), 182 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-07-23-25.gh-issue-74876.fO5FoR.rst diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index d59936029da69d..a5d4a98676eeed 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -39,6 +39,30 @@ Such constructors may be factory functions or class instances. object or subclass of :class:`pickle.Pickler` can also be used for declaring reduction functions. + +.. class:: Proxy(obj) + + A wrapper for an object which is pickled weakly. + Reduction functions can use it to pickle objects + to which the reconstructed object only holds weak references. + For example, a weak reference is reduced to + ``(weakref.ref, (Proxy(referent),))``. + + A proxy is pickled as the wrapped object *obj*, + which is unpickled in its place. + *obj* must also be strongly referenced elsewhere in the same pickle; + otherwise pickling raises :exc:`pickle.PicklingError`. + + :func:`copy.copy` of a proxy returns *obj*, + and :func:`copy.deepcopy` returns the deep copy of *obj*. + + .. attribute:: obj + + The wrapped object. + + .. versionadded:: next + + Example ------- diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 56c06ae609a215..cb62af5717528c 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -276,8 +276,10 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method. A :class:`!WeakSet` can be pickled if the elements are strongly referenced elsewhere in the same pickle (see :class:`ref`). - A copy or a deep copy shares the elements with the original; - user-defined attributes are copied or deep-copied. + A shallow copy contains the same elements as the original. + A deep copy refers to the deep copies of the elements, + so they must be deep-copied together with it; + its user-defined attributes are deep-copied. .. versionchanged:: next :class:`!WeakSet` can now be pickled and copied. @@ -314,7 +316,9 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method. A :class:`!WeakMethod` can be copied, and pickled if the object the method is bound to is strongly referenced elsewhere in the same pickle (see :class:`ref`). - A copy or a deep copy is an equal weak method to the same object. + A copy is an equal weak method bound to the same object. + A deep copy is bound to the deep copy of the object, + so it must be deep-copied together with it. .. versionadded:: 3.4 diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index b29170b5b0b93d..d4bea7d25061a5 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -193,6 +193,17 @@ concurrent.futures (Contributed by xzmeng and Serhiy Storchaka in :gh:`108518`.) +copyreg +------- + +* Add :class:`copyreg.Proxy`, a wrapper that is pickled as the wrapped object. + Reduction functions can use it to pickle objects + to which the reconstructed object only holds weak references. + The wrapped object must be strongly referenced elsewhere in the same pickle; + otherwise pickling raises :exc:`pickle.PicklingError`. + (Contributed by Serhiy Storchaka in :gh:`74876`.) + + csv --- diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index c462e63f430ff7..c1b24488c9d413 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -3,6 +3,7 @@ # by abc.py to load everything else at startup. from _weakref import ref +from copyreg import Proxy from types import GenericAlias __all__ = ['WeakSet'] @@ -44,14 +45,13 @@ def __contains__(self, item): return wr in self.data def __reduce__(self): - # Pickle the elements as weak references; data and _remove are - # rebuilt by the reconstructor. Copying shares the elements. + # The elements must be strongly referenced elsewhere in the pickle. + # data and _remove are rebuilt by the constructor. dict, slots = self.__getstate__() for name in ('data', '_remove'): slots.pop(name, None) state = (dict, slots) if dict or slots else None - return (_restore_weakset, (self.__class__, [ref(item) for item in self]), - state) + return (self.__class__, ([Proxy(item) for item in self],), state) def add(self, item): self.data.add(ref(item, self._remove)) @@ -62,6 +62,8 @@ def clear(self): def copy(self): return self.__class__(self) + __copy__ = copy + def pop(self): while True: try: @@ -156,10 +158,3 @@ def __repr__(self): return repr(self.data) __class_getitem__ = classmethod(GenericAlias) - - -def _restore_weakset(cls, refs): - self = cls.__new__(cls) - WeakSet.__init__(self) - self.update(item for r in refs for item in [r()] if item is not None) - return self diff --git a/Lib/copy.py b/Lib/copy.py index f7fd680657b85a..56f66ce38737f4 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -51,7 +51,7 @@ class instances). import types import weakref -from copyreg import dispatch_table +from copyreg import dispatch_table, Proxy class Error(Exception): pass @@ -71,6 +71,8 @@ def copy(x): return x if cls in _copy_builtin_containers: return cls.copy(x) + if cls is Proxy: + return x.obj if issubclass(cls, type): @@ -221,6 +223,10 @@ def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) d[types.MethodType] = _deepcopy_method +def _deepcopy_proxy(x, memo, deepcopy=deepcopy): + return deepcopy(x.obj, memo) +d[Proxy] = _deepcopy_proxy + del d def _keep_alive(x, memo): diff --git a/Lib/copyreg.py b/Lib/copyreg.py index a5e8add4a554d7..348d98692fc485 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -4,7 +4,7 @@ C, not for instances of user-defined classes. """ -__all__ = ["pickle", "constructor", +__all__ = ["pickle", "constructor", "Proxy", "add_extension", "remove_extension", "clear_extension_cache"] dispatch_table = {} @@ -23,6 +23,36 @@ def constructor(object): if not callable(object): raise TypeError("constructors must be callable") +class Proxy: + """Wrapper for an object that is pickled as a weak reference. + + Pickling a Proxy pickles the wrapped object, which is unpickled in + its place. Like the referent of a weak reference, the wrapped object + must also be strongly referenced elsewhere in the same pickle, + otherwise pickling fails. This lets a reduction function pickle + objects that the reconstructed object will only refer to weakly. + """ + + __slots__ = ('obj',) + + def __init__(self, obj): + self.obj = obj + + def __repr__(self): + return f'{self.__class__.__qualname__}({self.obj!r})' + +class _Reducer: + # An object that is pickled and copied as the result of a call: + # reduce is a (callable, args) pair returned from __reduce__(). + + __slots__ = ('_reduce',) + + def __init__(self, reduce): + self._reduce = reduce + + def __reduce__(self): + return self._reduce + # Example: provide pickling support for complex numbers. def pickle_complex(c): @@ -41,6 +71,21 @@ def pickle_super(obj): pickle(super, pickle_super) +from _weakref import ref as _ref + +# All dead weak references are interchangeable, so they are pickled as +# references to this set. It is unpickled as a new set which dies with +# the unpickler's memo, and all dead references as the same reference. +_dead_referent = set() + +def pickle_weakref(r): + obj = r() + if obj is None: + return _ref, (_dead_referent,) + return _ref, (Proxy(obj),) + +pickle(_ref, pickle_weakref) + # Support for pickling new-style objects def _reconstructor(cls, base, state): diff --git a/Lib/pickle.py b/Lib/pickle.py index 64a77734205917..fac26022879d48 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -23,7 +23,7 @@ """ from types import FunctionType -from copyreg import dispatch_table +from copyreg import dispatch_table, Proxy from copyreg import _extension_registry, _inverted_registry, _extension_cache from itertools import batched from functools import partial @@ -32,7 +32,6 @@ from struct import pack, unpack import io import codecs -import weakref import _compat_pickle __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", @@ -487,7 +486,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True, self._write_large_bytes = self.framer.write_large_bytes self.memo = {} # Ids of objects reached so far only through weak references; any - # left at the end of dump() is an error. See save_weakref. + # left at the end of dump() is an error. See save_proxy. self._weak_only = set() self.proto = int(protocol) self.bin = protocol >= 1 @@ -1286,39 +1285,20 @@ def save_type(self, obj): dispatch[FunctionType] = save_global dispatch[type] = save_type - def save_weakref(self, obj): - # Reconstruct as weakref.ref(referent). The referent is saved - # *weakly* (no anchoring, see save()) and tagged weak-only only after + def save_proxy(self, obj): + # A Proxy is pickled as its wrapped object, saved *weakly* (no + # anchoring, see save()). The object is tagged weak-only only after # its whole subtree, so a strong cycle inside it cannot anchor it. - referent = obj() - if referent is None: - # All dead references are interchangeable: pickle by name as the - # weakref._dead_ref singleton. - self.write(GLOBAL + b'weakref\n_dead_ref\n') - return - - save = self.save - write = self.write - save(weakref.ref) - # A referent already in the memo is anchored or already tracked; - # otherwise it is tagged below, until a strong save of it. - anchored = id(referent) in self.memo - # Build the 1-tuple (referent,); TUPLE1 needs protocol 2. - if self.proto < 2: - write(MARK) - save(referent, weak=True) - if not anchored and id(referent) in self.memo: - self._weak_only.add(id(referent)) - write(TUPLE1 if self.proto >= 2 else TUPLE) - write(REDUCE) - # The referent's subtree may have pickled this very (cached) weakref - # already; reuse the memoized one. - if id(obj) in self.memo: - write(POP + self.get(self.memo[id(obj)][0])) - else: - self.memoize(obj) - - dispatch[weakref.ReferenceType] = save_weakref + # An object already in the memo is anchored or already tracked; + # otherwise it stays tagged until a strong save of it. + obj = obj.obj + i = id(obj) + anchored = i in self.memo + self.save(obj, weak=True) + if not anchored and i in self.memo: + self._weak_only.add(i) + + dispatch[Proxy] = save_proxy # Unpickling machinery diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 9c0679d1d13d23..1cf28fbbe04306 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1866,6 +1866,17 @@ def test_weakref_reused_pickler(self): obj2, wr2 = pickle.loads(f.getvalue()) self.assertIs(wr2(), obj2) + def test_proxy_without_strong_ref(self): + obj = Object() + for proto in protocols: + with self.subTest(proto=proto): + self.assertRaises(pickle.PicklingError, self.dumps, + copyreg.Proxy(obj), proto) + # Another Proxy is not a strong reference. + self.assertRaises(pickle.PicklingError, self.dumps, + [copyreg.Proxy(obj), copyreg.Proxy(obj)], + proto) + def test_bad_reduce_result(self): obj = REX([print, ()]) for proto in protocols: @@ -3684,6 +3695,23 @@ def test_dead_weakref(self): support.gc_collect() self.assertIsNone(a()) + def test_proxy(self): + # A Proxy is unpickled as the wrapped object. + obj = Object() + for proto in protocols: + with self.subTest(proto=proto): + x, y = self.loads(self.dumps([obj, copyreg.Proxy(obj)], proto)) + self.assertIs(type(x), Object) + self.assertIs(y, x) + # Identity holds regardless of pickling order. + x, y = self.loads(self.dumps([copyreg.Proxy(obj), obj], proto)) + self.assertIs(type(x), Object) + self.assertIs(y, x) + # Objects which are not memoized need no strong reference. + x = self.loads(self.dumps([copyreg.Proxy(None), + copyreg.Proxy(42)], proto)) + self.assertEqual(x, [None, 42]) + def test_newobj_overridden_new(self): # Test that Python class with C implemented __new__ is pickleable for proto in protocols: diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 2a12a6737df163..b8150457e00772 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -869,6 +869,24 @@ def test_copy_weakref(self): def test_deepcopy_weakref(self): self._check_weakref(copy.deepcopy) + def test_copy_proxy(self): + class C: + pass + x = C() + self.assertIs(copy.copy(copyreg.Proxy(x)), x) + + def test_deepcopy_proxy(self): + # A Proxy is replaced by the deep copy of the wrapped object. + class C: + pass + x = C() + y, z = copy.deepcopy([x, copyreg.Proxy(x)]) + self.assertIsNot(y, x) + self.assertIs(z, y) + z, y = copy.deepcopy([copyreg.Proxy(x), x]) + self.assertIsNot(y, x) + self.assertIs(z, y) + def _check_copy_weakdict(self, _dicttype): class C(object): pass diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 52ed3f117be569..676939d7bcd7da 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1386,18 +1386,34 @@ def test_pickle_without_strong_ref(self): self.assertRaises(pickle.PicklingError, pickle.dumps, r, proto) def test_copy(self): - # Copying rebuilds an equal weak method bound to the same instance - # (weak references are atomic under copy). + # A copy is an equal weak method bound to the same object. o = Object(1) r = weakref.WeakMethod(o.some_method) - for copyfunc in copy.copy, copy.deepcopy: - with self.subTest(copyfunc=copyfunc): - r2 = copyfunc(r) - self.assertIsInstance(r2, weakref.WeakMethod) - self.assertIsNot(r2, r) - self.assertEqual(r2, r) - self.assertIs(r2().__self__, o) - self.assertEqual(r2()(), 4) + r2 = copy.copy(r) + self.assertIsInstance(r2, weakref.WeakMethod) + self.assertIsNot(r2, r) + self.assertEqual(r2, r) + self.assertIs(r2().__self__, o) + self.assertEqual(r2()(), 4) + # A dead weak method cannot be recreated. + del o + gc_collect() + self.assertIs(copy.copy(r), r) + + def test_deepcopy(self): + # A deep copy is bound to the deep copy of the object, so it must be + # deep-copied together with it. + o = Object(1) + r = weakref.WeakMethod(o.some_method) + o2, r2 = copy.deepcopy((o, r)) + self.assertIsInstance(r2, weakref.WeakMethod) + self.assertIsNot(o2, o) + self.assertIs(r2().__self__, o2) + self.assertEqual(r2()(), 4) + # Otherwise the copy of the object is discarded. + r2 = copy.deepcopy(r) + gc_collect() + self.assertIsNone(r2()) class MappingTestCase(TestBase): @@ -1822,6 +1838,28 @@ def test_pickling_without_strong_ref(self): with self.assertRaises(pickle.PicklingError): pickle.dumps(d, proto) + def test_pickling_recursive(self): + # WeakValueDictionary containing itself. + d = weakref.WeakValueDictionary() + d[1] = d + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + dup = pickle.loads(pickle.dumps(d, proto)) + self.assertIsNot(dup, d) + self.assertEqual(list(dup.keys()), [1]) + self.assertIs(dup[1], dup) + + # WeakKeyDictionary containing itself. + o = Object(1) + d = weakref.WeakKeyDictionary() + d[o] = d + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + dup, o2 = pickle.loads(pickle.dumps((d, o), proto)) + self.assertIsNot(dup, d) + self.assertEqual(list(dup.keys()), [o2]) + self.assertIs(dup[o2], dup) + def check_popitem(self, klass, key1, value1, key2, value2): weakdict = klass() weakdict[key1] = value1 diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 60e1a154b3ec44..8c04d411d5afb9 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -461,23 +461,32 @@ def test_copying(self): s.x = ['x'] s.z = ['z'] + # A shallow copy contains the same elements, without the + # user-defined attributes (like copy()). dup = copy.copy(s) self.assertIsInstance(dup, cls) self.assertEqual(dup, s) self.assertIsNot(dup, s) - self.assertIs(dup.x, s.x) - self.assertIs(dup.z, s.z) + self.assertNotHasAttr(dup, 'x') + self.assertNotHasAttr(dup, 'z') self.assertNotHasAttr(dup, 'y') - dup = copy.deepcopy(s) + # A deep copy refers to the deep copies of the elements, so they + # must be deep-copied together with it. + dup, items = copy.deepcopy((s, self.items)) self.assertIsInstance(dup, cls) - self.assertEqual(dup, s) - self.assertIsNot(dup, s) + self.assertEqual(dup, WeakSet(items)) + self.assertEqual(set(map(id, dup)), set(map(id, items))) + self.assertNotEqual(set(map(id, dup)), set(map(id, s))) self.assertEqual(dup.x, s.x) self.assertIsNot(dup.x, s.x) self.assertEqual(dup.z, s.z) self.assertIsNot(dup.z, s.z) self.assertNotHasAttr(dup, 'y') + # Otherwise the copies of the elements are discarded. + dup = copy.deepcopy(s) + support.gc_collect() + self.assertEqual(len(dup), 0) def test_pickling(self): for cls in WeakSet, WeakSetWithSlots: diff --git a/Lib/weakref.py b/Lib/weakref.py index fc895d4033300b..a4f0c9b130b1f5 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -22,7 +22,7 @@ from _weakrefset import WeakSet import _collections_abc # Import after _weakref to avoid circular import. -import copyreg +from copyreg import Proxy, _Reducer import sys import itertools @@ -37,11 +37,6 @@ _collections_abc.MutableSet.register(WeakSet) -# Pickle reconstructs a reference to a dead object as this singleton -- all -# dead references are interchangeable. Born dead: nothing else holds the set. -_dead_ref = ref(set()) - - class WeakMethod(ref): """ A custom `weakref.ref` subclass which simulates a weak reference to @@ -72,6 +67,24 @@ def _cb(arg): self_wr = ref(self) return self + def __copy__(self): + meth = self() + if meth is None: + # A dead weak method cannot be recreated. + return self + return type(self)(meth) + + def __reduce__(self): + # The instance is pickled weakly; the bound method is rebuilt on + # unpickling by looking the function up on the instance. + # The callback is not preserved. + meth = self() + if meth is None: + raise TypeError( + "cannot pickle a weak reference to a dead object") + return self.__class__, (_Reducer((getattr, (Proxy(meth.__self__), + meth.__func__.__name__))),) + def __call__(self): obj = super().__call__() func = self._func_ref() @@ -96,24 +109,6 @@ def __ne__(self, other): __hash__ = ref.__hash__ -# Pickling support for weak methods. A WeakMethod is reduced to a weak -# reference to its instance; the bound method is rebuilt on unpickling by -# looking the function up on the instance. The callback is not preserved. - -def _restore_weakmethod(obj_ref, name): - return WeakMethod(getattr(obj_ref(), name)) - - -def _reduce_weakmethod(self): - meth = self() - if meth is None: - raise TypeError( - "cannot pickle a weak reference to a dead object") - return (_restore_weakmethod, (ref(meth.__self__), meth.__func__.__name__)) - -copyreg.pickle(WeakMethod, _reduce_weakmethod) - - class WeakValueDictionary(_collections_abc.MutableMapping): """Mapping class that references values weakly. @@ -187,14 +182,14 @@ def __deepcopy__(self, memo): return new def __reduce__(self): - # Pickle the values as weak references; data and _remove are rebuilt - # by the reconstructor. Copying uses __copy__/__deepcopy__ instead. + # The values must be strongly referenced elsewhere in the pickle. + # data and _remove are rebuilt by the reconstructor. dict, slots = self.__getstate__() for name in ('data', '_remove'): slots.pop(name, None) state = (dict, slots) if dict or slots else None - items = [(key, ref(o)) for key, o in self.items()] - return (_restore_weakvaluedict, (self.__class__, items), state) + items = ((key, Proxy(o)) for key, o in self.items()) + return (self.__class__, (), state, None, items) def get(self, key, default=None): try: @@ -313,16 +308,6 @@ def __ror__(self, other): return NotImplemented -def _restore_weakvaluedict(cls, items): - self = cls.__new__(cls) - WeakValueDictionary.__init__(self) - for key, r in items: - o = r() - if o is not None: - self[key] = o - return self - - class KeyedRef(ref): """Specialized reference that includes a key corresponding to the value. @@ -405,14 +390,14 @@ def __deepcopy__(self, memo): return new def __reduce__(self): - # Pickle the keys as weak references; data and _remove are rebuilt - # by the reconstructor. Copying uses __copy__/__deepcopy__ instead. + # The keys must be strongly referenced elsewhere in the pickle. + # data and _remove are rebuilt by the reconstructor. dict, slots = self.__getstate__() for name in ('data', '_remove'): slots.pop(name, None) state = (dict, slots) if dict or slots else None - items = [(ref(key), value) for key, value in self.items()] - return (_restore_weakkeydict, (self.__class__, items), state) + items = ((Proxy(key), value) for key, value in self.items()) + return (self.__class__, (), state, None, items) def get(self, key, default=None): return self.data.get(ref(key),default) @@ -500,16 +485,6 @@ def __ror__(self, other): return NotImplemented -def _restore_weakkeydict(cls, items): - self = cls.__new__(cls) - WeakKeyDictionary.__init__(self) - for r, value in items: - key = r() - if key is not None: - self[key] = value - return self - - class finalize: """Class for finalization of weakrefable objects diff --git a/Misc/NEWS.d/next/Library/2026-10-22-07-23-25.gh-issue-74876.fO5FoR.rst b/Misc/NEWS.d/next/Library/2026-10-22-07-23-25.gh-issue-74876.fO5FoR.rst new file mode 100644 index 00000000000000..1a809136d00dd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-07-23-25.gh-issue-74876.fO5FoR.rst @@ -0,0 +1,5 @@ +Add :class:`copyreg.Proxy`, a wrapper that is pickled as the wrapped object. +Reduction functions can use it to pickle objects +to which the reconstructed object only holds weak references. +The wrapped object must be strongly referenced elsewhere in the same pickle; +otherwise pickling raises :exc:`pickle.PicklingError`. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f37c0a5bba5541..f18d7c3dadf272 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -188,6 +188,9 @@ typedef struct { /* copyreg._inverted_registry, {code: (module_name, function_name)} */ PyObject *inverted_registry; + /* copyreg.Proxy */ + PyObject *Proxy_type; + /* Import mappings for compatibility with Python 2.x */ /* _compat_pickle.NAME_MAPPING, @@ -254,6 +257,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->extension_registry); Py_CLEAR(st->extension_cache); Py_CLEAR(st->inverted_registry); + Py_CLEAR(st->Proxy_type); Py_CLEAR(st->name_mapping_2to3); Py_CLEAR(st->import_mapping_2to3); Py_CLEAR(st->name_mapping_3to2); @@ -320,6 +324,9 @@ _Pickle_InitState(PickleState *st) "not %.200s", Py_TYPE(st->extension_cache)->tp_name); goto error; } + st->Proxy_type = PyObject_GetAttrString(copyreg, "Proxy"); + if (!st->Proxy_type) + goto error; Py_CLEAR(copyreg); /* Load the 2.x -> 3.x stdlib module mapping tables */ @@ -4658,75 +4665,26 @@ memo_clear_weak_only(PicklerObject *self, Py_ssize_t *value) } } -/* Pickle a weakref.ref as weakref.ref(referent). The referent is pickled - *weakly* (no anchoring) and tagged weak-only only after its whole subtree, +/* A copyreg.Proxy is pickled as its wrapped object, pickled *weakly* (no + anchoring). The object is tagged weak-only only after its whole subtree, so a strong cycle inside it cannot anchor it. See Lib/pickle.py. */ static int -save_weakref(PickleState *st, PicklerObject *self, PyObject *obj) +save_proxy(PickleState *st, PicklerObject *self, PyObject *obj) { - const char tuple1_op = TUPLE1; - const char tuple_op = TUPLE; - const char mark_op = MARK; - const char reduce_op = REDUCE; - const char pop_op = POP; - int status = -1; - - PyObject *referent; - if (PyWeakref_GetRef(obj, &referent) < 0) + PyObject *wrapped = PyObject_GetAttrString(obj, "obj"); + if (wrapped == NULL) return -1; - if (referent == NULL) { - /* All dead references are interchangeable: pickle by name as the - weakref._dead_ref singleton. */ - const char global_op = GLOBAL; - const char dead_ref_name[] = "weakref\n_dead_ref\n"; - if (_Pickler_Write(self, &global_op, 1) < 0 || - _Pickler_Write(self, dead_ref_name, sizeof(dead_ref_name) - 1) < 0) - { - return -1; - } - return 0; - } - - /* Py_TYPE(obj) is exactly weakref.ref, the reconstructor. */ - if (save(st, self, (PyObject *)Py_TYPE(obj), 0) < 0) - goto done; - /* A referent already in the memo is anchored or already tracked; + /* An object already in the memo is anchored or already tracked; otherwise it is tagged below, until a strong save of it. */ - int anchored = PyMemoTable_Get(self->memo, referent) != NULL; - - /* Build the 1-tuple (referent,); TUPLE1 needs protocol 2. */ - if (self->proto < 2 && _Pickler_Write(self, &mark_op, 1) < 0) - goto done; - - if (save_object(st, self, referent, 0, 1) < 0) - goto done; - - if (!anchored) { - Py_ssize_t *value = PyMemoTable_Get(self->memo, referent); + int anchored = PyMemoTable_Get(self->memo, wrapped) != NULL; + int status = save_object(st, self, wrapped, 0, 1); + if (status == 0 && !anchored) { + Py_ssize_t *value = PyMemoTable_Get(self->memo, wrapped); if (value != NULL) memo_set_weak_only(self, value); } - - if (_Pickler_Write(self, self->proto >= 2 ? &tuple1_op : &tuple_op, 1) < 0 || - _Pickler_Write(self, &reduce_op, 1) < 0) - goto done; - - /* The referent's subtree may have pickled this very (cached) weakref - already; reuse the memoized one. */ - if (PyMemoTable_Get(self->memo, obj)) { - if (_Pickler_Write(self, &pop_op, 1) < 0) - goto done; - if (memo_get(st, self, obj) < 0) - goto done; - } - else if (memo_put(st, self, obj) < 0) { - goto done; - } - status = 0; - - done: - Py_DECREF(referent); + Py_DECREF(wrapped); return status; } @@ -4778,8 +4736,8 @@ weak_only_type_names(PicklerObject *self) return NULL; } -/* `weak` is true only for save_weakref's immediate save of its referent; it - does not propagate to nested saves. All other callers go through save(). */ +/* `weak` is true only for save_proxy's immediate save of the wrapped object; + it does not propagate to nested saves. All other callers go through save(). */ static int save_object(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save, int weak) @@ -4850,8 +4808,8 @@ save_object(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save, return -1; } - if (PyWeakref_CheckRefExact(obj)) { - status = save_weakref(st, self, obj); + if (type == (PyTypeObject *)st->Proxy_type) { + status = save_proxy(st, self, obj); goto done; } @@ -8474,6 +8432,7 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->PicklingError); Py_VISIT(st->UnpicklingError); Py_VISIT(st->dispatch_table); + Py_VISIT(st->Proxy_type); Py_VISIT(st->extension_registry); Py_VISIT(st->extension_cache); Py_VISIT(st->inverted_registry);